o
    پiF                    @   s  d Z ddlZddlmZmZmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZmZ ddlZddlmZ ddlmZmZmZmZ ddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( dd	l)m*Z* dd
l+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2 g dZ3eG dd dZ4eG dd dZ5dde	e6df de	e7df de6de	e4df fddZ8				dde	e6df de	e7df de	e6df de6de
e
e4  f
dd Z9		!dd"e	e:e:f de6d#ee6e
e6 f d$e;de	e4df f
d%d&Z<d'ee4ee4 f de
e4 fd(d)Z=d*ee6 d+e6de6fd,d-Z>eG d.d/ d/Z?G d0d1 d1ej@ZAd2e:d3e6d4e6d5e6d6e	e6e6f d7e?deej@ fd8d9ZBG d:d; d;ej@ZCG d<d= d=ej@ZDG d>d? d?ej@ZEG d@dA dAej@ZFG dBdC dCej@ZGG dDdE dEej@ZHG dFdG dGej@ZIeJeCeDeEeFeGeHeIdHZKdIe:dJej@fdKdLZLdMee:ej@f fdNdOZMG dPdQ dQejNZO	R	R	S	dd3e6d4e6dTe:dUe:dVe:d7e?fdWdXZPddZd[ZQd\d] ZRd^ee:ef d_e4d`e5fdadbZSddeSfdce5dde7dee6dfee:ef dgee6 d7ee? dhee fdidjZTddce5dle;fdmdnZUG dodp dpej@ZVddqej@dre:dse;ddfdtduZWeJdi dve5e4dwddxdYddydze4dwdYd{dYddydze4d|d}d~dYdddze4d|dd~dYdddze4d|dd~ddddzfddddde5e4dwddxdYddydze4dwdYd{dYddydze4d|d}d~dYdddze4d|dd~dYdddze4d|dd~ddddzfddddde5e4dwdddYddydze4dwdddYddydze4d|dddYdddze4d|dYddYdddze4d|ddddddzfddddde5e8ddddddde5e8ddddddde5e8ddddddde5e8dddddde5e8dddddde5e8ddddddde5e8dddddde5e8ddddddde5e8dddddde5e8ddddddde5e8ddddddeJddddde5e4d|dYdddddze4d|dddYdddze4d|d}ddYdddze4d|dddYddydzfdxdddddde5e4ddddddyeJ de4d|dddYdddze4d|d}ddYdddze4d|dddYddydzfdxddddeJdkddde5e4d|dYdddddze4d|dYddYdddze4d|dYddYdddze4d|dYddYdddzfdddddde5e4d|dYdddddze4d|dYddYdddze4d|dYddYdddze4d|dYddYdddzfddddddde5e4d|dYdddddze4d|dYddYdddze4d|dYddYdddze4d|dYddYdddzfddddddde5e4d|dYdddddze4d|dYddYdddze4d|dYddYdddze4d|dYddYdddzfddddddde5e4d|dYdddddze4d|dYddYdddze4d|dYddYdddze4d|dYddYdddzfdddddeJddɍdʍde5e4d|dYdddddze4d|dddYdddze4d|dddYdddze4d|dYddYdddzfdddRdddde5e4d|dYdddddze4d|dddYdddze4d|dddYdddze4d|dYddYdddzfdddRdddde5e4d|dYdddddze4d|dddYdddze4d|dddYdddze4d|dYddYdddzfdddRddddύde5e4d|dYdddddze4d|dddYdddze4d|dddYdddze4d|dYddYdddzfdddRddddύde5e4d|dYdddddze4d|dddYdddze4d|dddYdddze4d|dYddYdddzfdddRddddύde5e4d|dddddӍe4d|dddYddӍe4d|d}ddYddӍe4d|dddYddӍfdddRddԍde5e4d|ddddddze4d|dddYdddze4d|d}ddYdddze4d|dddYdddzfddddddde5e4d|dYddYdddze4d|d}ddYdddze4d|dd{dYdddze4d|dYddYdddzfddRdRdddeJddۍeJdkdkd܍dݍ	de5e4d|dYddYdddze4d|d}ddYdddze4d|dd{dYdddze4d|dYddYdddzfddRdRdddeJddۍeJdkdkd܍dݍ	de5e4d|ddddddze4d|d}dxdYdddze4d|dddYdddze4d|dddYdddzfdddRdRdddeJddۍeJdkdkd܍d
de5e4d|ddddddze4d|d}dxdYdddze4d|dddYdddze4d|dddYdddzfdddRdRdddeJddۍeJdkdkd܍d
de5e4d|ddddddze4d|dd{dYdddze4d|dddYdddze4d|dddYdddzfdddRdRdddeJddۍeJdkdkd܍d
de5e4d|dYddYdddze4d|d}ddYdddze4d|dd{dYdddze4d|dYddYdddzfddRdRddeedddeJddۍeJdkdkd܍d
de5e4d|dYddYdddze4d|d}ddYdddze4d|dd{dYdddze4d|dYddYdddzfddRdRddeedddeJddۍeJdkdkd܍d
de5e4d|ddddddze4d|d}dxdYdddze4d|dddYdddze4d|dddYdddzfdddRdRddeedddeJddۍeJdkdkd܍dde5e9ddddddde5e9dddddde5e9dddddde5e9dddddde5e9ddddddde5e4d|dddddӍe4d|dddYddӍe4d|d}ddYddӍe4d|dddYddӍfddRdddddde5e4d|dddddӍe4d|dddYddӍe4d|d ddYddӍe4d|dddYddӍfddRdddddde5e4d|dddddӍe4d|d}ddYddӍe4d|dddYddӍe4d|d}ddYddӍfdddRdddddde5e4d|d}ddddӍe4d|dddYddӍe4d|dddYddӍe4d|dddYddӍfdddRdddddde5e4d|dddddӍe4d|d	ddYddӍe4d|d
ddYddӍe4d|dddYddӍfdddRdddddde5e4d|dddddӍe4d|dddYddӍe4d|d}ddYddӍe4d|dddYddӍfddRddddddde5e4ddddYdddze4ddddYdddze4dwddxdYdddze4d|dddYdddzfdddRddeJddۍdZXdD ]ZYeeXeY ddeXeYd < 	qR	ddee:ejZf deVde:dee:ejZf fddZ[dee:ejZf deVfdd Z\dd!e:d"e;deVfd#d$Z]dd%e:dee:ef fd&d'Z^dd%e:dee:ef fd(d)Z_e1i d*e^d+d,d-e^d+d,d.e^d+d/d0d1d2e^d+d3d4d5d6e^d+d3d4d5d7e^d+d3d4d5d8e^d+d3d4d5d9e^d+d3d4d5d:e^d+d3d4d5d;e^d+d3d4d5d<e^d+d3d4d5d=e^d+d3d4d5d>e^d+d3d4d5d?e^d+d3d4d@dAdydBdCe^d+dDdEd/d0dFdydGdHe_d+dIdFdydJdKe_d+dLdFdydJi dMe_d+dNdFdydJdOe_d+dPdFdydJdQe_d+dRdFdydJdSe_d+dTd/dUdVe_d+dWdFdydJdXe_d+dYdFdydJdZe_d+d[dFdydJd\e_d+d]dFdydJd^e_d+d_dFdydJd`e_d+dadFdydJdbe_d+dcdFdydJdde_d+dedfdgdgdhdidjdFdydk
dle_d+dmdfdgdgdjd@dydndoe_d+dpdgdgdqd@drdse_d+dtdgdgdjd@dydudve_d+dwdgdgdjd@dydudxe_dfdgdgdhdidqdFdyi dze_d+d{dfdgdgdqd@d|d}e_d+d~dgdgdqd@dydude^d+dddde^d+dddde^d+dddde^d+dddde^d+dddde_d+deedkdhdiddde_d+deedkdhdiddde_d+d~eedkdFdddde_d+deedkddddde_d+deedkddddde_d+deedkdhdiddde_d+deedkdhdiddde_d+deedkdhdiddde_d+deedhdidde_d+deedhdide_d+deedFdde_d+deeddde_d+deeddde_d+deedhdide_d+deedhdide_d+deedhdide_d/d0de_d+dfddqdddZ`e2ddeVfddvZae2ddeVfddZbe2ddeVfddZce2ddeVfddZde2ddeVfddZee2ddeVfddZfe2ddeVfddZge2ddeVfddZhe2ddeVfddZie2ddeVfddZje2ddeVfddZke2ddeVfddZle2ddeVfddZme2ddeVfddZne2ddeVfddZoe2ddeVfddZpe2ddeVfddZqe2ddeVfddZre2ddeVfddÄZse2ddeVfddĄZte2ddeVfddƄZue2ddeVfdd˄Zve2ddeVfdd̄Zwe2ddeVfdd΄Zxe2ddeVfddЄZye2ddeVfddфZze2ddeVfdd҄Z{e2ddeVfddՄZ|e2ddeVfddքZ}e2ddeVfddބZ~e2ddeVfdd߄Ze2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfddZe2ddeVfdːdZe2ddeVfd̐dZe2ddeVfd͐dZe2ddeVfdΐdτZe2ddeVfdАdфZe2ddeVfdҐdӄZe2ddeVfdԐdՄZe2ddeVfd֐dׄZe2ddeVfdؐdZe2ddeVfdِdZdS (  a   Bring-Your-Own-Blocks Network

A flexible network w/ dataclass based config for stacking those NN blocks.

This model is currently used to implement the following networks:

GPU Efficient (ResNets) - gernet_l/m/s (original versions called genet, but this was already used (by SENet author)).
Paper: `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
Code and weights: https://github.com/idstcv/GPU-Efficient-Networks, licensed Apache 2.0

RepVGG - repvgg_*
Paper: `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
Code and weights: https://github.com/DingXiaoH/RepVGG, licensed MIT

MobileOne - mobileone_*
Paper: `MobileOne: An Improved One millisecond Mobile Backbone` - https://arxiv.org/abs/2206.04040
Code and weights: https://github.com/apple/ml-mobileone, licensed MIT

In all cases the models have been modified to fit within the design of ByobNet. I've remapped
the original weights and verified accuracies.

For GPU Efficient nets, I used the original names for the blocks since they were for the most part
the same as original residual blocks in ResNe(X)t, DarkNet, and other existing models. Note also some
changes introduced in RegNet were also present in the stem and bottleneck blocks for this model.

A significant number of different network archs can be implemented here, including variants of the
above nets that include attention.

Hacked together by / copyright Ross Wightman, 2021.
    N)	dataclassfieldreplace)partial)TupleListDictOptionalUnionAnyCallableSequence)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDOPENAI_CLIP_MEANOPENAI_CLIP_STD)ClassifierHeadNormMlpClassifierHeadConvNormActBatchNormAct2dEvoNorm2dS0aAttentionPool2dRotAttentionPool2dDropPathAvgPool2dSamecreate_conv2dget_act_layerget_norm_act_layerget_attnmake_divisible	to_2tuple   )build_model_with_cfg)feature_take_indices)named_applycheckpoint_seq)generate_default_cfgsregister_model)ByobNetByoModelCfgByoBlockCfgcreate_byob_stemcreate_blockc                   @   s   e Zd ZU dZeeejf ed< e	ed< e	ed< dZ
e	ed< dZeee	ef  ed< d	Zeed
< dZee ed< dZeeeef  ed< dZee ed< dZeeeef  ed< dZeeeef  ed< dS )r*   zuBlock configuration for Bring-Your-Own-Blocks.

    Defines configuration for a single block or stage of blocks.
    typedc   sNgs      ?br
attn_layerattn_kwargsself_attn_layerself_attn_kwargsblock_kwargs)__name__
__module____qualname____doc__r
   strnnModule__annotations__intr1   r2   r	   r   r4   floatr5   r6   r   r   r7   r8   r9    rD   rD   G/home/ubuntu/.local/lib/python3.10/site-packages/timm/models/byobnet.pyr*   5   s   
 r*   c                   @   sV  e Zd ZU dZeeeeedf f df ed< dZe	ed< dZ
e	ed< dZee	 ed	< d
Zeeee eedf f ed< dZeed< dZeed< dZeed< dZeed< dZe	ed< dZe	ed< dZe	ed< dZee ed< dZe	ed< dZee	 ed< edd  d!Zeed"< dZee	 ed#< ed$d  d!Z eed%< ed&d  d!Z!e"e	e#f ed'< dS )(r)   zlModel configuration for Bring-Your-Own-Blocks network.

    Defines overall architecture configuration.
    .blocksconv1x1
downsample3x3	stem_typemaxpool	stem_pool    stem_chsr3   width_factorr   num_featuresTzero_init_lastFfixed_input_sizerelu	act_layer	batchnorm
norm_layer aa_layerNhead_hidden_size
classifier	head_typer5   c                   C      t  S NdictrD   rD   rD   rE   <lambda>f       zByoModelCfg.<lambda>)default_factoryr6   r7   c                   C   r\   r]   r^   rD   rD   rD   rE   r`   h   ra   r8   c                   C   r\   r]   r^   rD   rD   rD   rE   r`   i   ra   r9   )$r:   r;   r<   r=   r   r
   r*   rA   rH   r>   rJ   rL   r	   rN   rB   r   rO   rC   rP   rQ   boolrR   rT   rV   rX   rY   r[   r5   r   r6   r_   r7   r8   r9   r   r   rD   rD   rD   rE   r)   J   s*   
  ""r)            r!   r3   r3   r3   r3   r.   .wfgroupsreturnc                    s>   d}d dkrfdd t  fddt| ||D }|S )zCreate RepVGG block configuration.

    Args:
        d: Depth (number of blocks) per stage.
        wf: Width factor per stage.
        groups: Number of groups for grouped convolution.

    Returns:
        Tuple of block configurations.
    @            r   c                    s   |d d dkr|   S dS )Nr!   r0   r   rD   )chsidx)rj   rD   rE   r`   {       z_rep_vgg_bcfg.<locals>.<lambda>c                    s&   g | ]\}}}t d |||  dqS )rep)r-   r.   r/   r2   )r*   ).0r.   r/   ri   
group_sizerD   rE   
<listcomp>|      & z!_rep_vgg_bcfg.<locals>.<listcomp>)tuplezip)r.   ri   rj   r/   bcfgrD   )rw   rj   rE   _rep_vgg_bcfgl   s   r}   r0      
   r!   rD   	se_blocksnum_conv_branchesc                 C   s   d}t d|d |d  }|pdt|  }g }t| |||D ]R\} }}}g }	t| D ]@}
|| }t|d}i }|
| | krAd|d< |	tddd	|d	|d
|g7 }	|	tddd	|dtddd	i|d
|g7 }	|}q*||	g7 }q|S )a,  Create MobileOne block configuration.

    Args:
        d: Depth (number of blocks) per stage.
        wf: Width factor per stage.
        se_blocks: Number of SE blocks per stage.
        num_conv_branches: Number of conv branches.

    Returns:
        List of block configurations per stage.
    rl   rm   r   )r   )r   ser5   oner!   )r-   r.   r/   r2   r9   kernel_sizeNrD   )minlenr{   ranger_   r*   )r.   ri   r   r   r/   prev_cr|   wr   scfgiout_cbkakrD   rD   rE   _mobileone_bcfg   s*   
 
r   Ftypeseveryfirstc                 K   s   t | dksJ t|tr"tt|rdn|||d }|s"|d g}t| g }t|D ]}||v r6| d n| d }|td|dd|g7 }q,t|S )a\  Interleave 2 block types in stack.

    Args:
        types: Two block type names to interleave.
        d: Total depth of blocks.
        every: Interval for alternating blocks.
        first: Whether to start with alternate block.
        **kwargs: Additional block arguments.

    Returns:
        Tuple of interleaved block configurations.
    r0   r   r!   )r-   r.   NrD   )r   
isinstancerB   listr   setr*   rz   )r   r.   r   r   kwargsrF   r   
block_typerD   rD   rE   interleave_blocks   s   

r   stage_blocks_cfgc                    sF   t | ts| f} g }t| D ]\} | fddt jD 7 }q|S )zExpand block config into individual block instances.

    Args:
        stage_blocks_cfg: Block configuration(s) for a stage.

    Returns:
        List of individual block configurations.
    c                    s   g | ]}t  d dqS )r!   r.   )r   )ru   _cfgrD   rE   rx      s    z%expand_blocks_cfg.<locals>.<listcomp>)r   r   	enumerater   r.   )r   
block_cfgsr   rD   r   rE   expand_blocks_cfg   s   
	r   rw   channelsc                 C   s    | sdS ||  dksJ ||  S )zCalculate number of groups for grouped convolution.

    Args:
        group_size: Size of each group (1 for depthwise).
        channels: Number of channels.

    Returns:
        Number of groups.
    r!   r   rD   )rw   r   rD   rD   rE   
num_groups   s   
r   c                   @   sX   e Zd ZU dZeZeed< eZ	eed< e
jZeed< dZee ed< dZee ed< dS )LayerFnz&Container for layer factory functions.conv_norm_actnorm_actactNattn	self_attn)r:   r;   r<   r=   r   r   r   rA   r   r   r?   ReLUr   r   r	   r   rD   rD   rD   rE   r      s   
 r   c                       s^   e Zd ZdZ				ddedededed	ed
ee f fddZde	j
de	j
fddZ  ZS )DownsampleAvgz[Average pool downsampling module.

    AvgPool Downsampling as in 'D' ResNet variants.
    r!   FNin_chsout_chsstridedilation	apply_actlayersc           	         s   t t|   |pt }|dkr|nd}|dks|dkr3|dkr&|dkr&tntj}|d|ddd| _nt | _|j	||d|d| _
dS )aB  Initialize DownsampleAvg.

        Args:
            in_chs: Number of input channels.
            out_chs: Number of output channels.
            stride: Stride for downsampling.
            dilation: Dilation rate.
            apply_act: Whether to apply activation.
            layers: Layer factory functions.
        r!   r0   TF)	ceil_modecount_include_padr   N)superr   __init__r   r   r?   	AvgPool2dpoolIdentityr   conv)	selfr   r   r   r   r   r   
avg_strideavg_pool_fn	__class__rD   rE   r      s   

zDownsampleAvg.__init__xrk   c                 C   s   |  | |S )zoForward pass.

        Args:
            x: Input tensor.

        Returns:
            Output tensor.
        )r   r   r   r   rD   rD   rE   forward  s   	zDownsampleAvg.forward)r!   r!   FN)r:   r;   r<   r=   rB   rc   r	   r   r   torchTensorr   __classcell__rD   rD   r   rE   r      s(    	r   downsample_typer   r   r   r   r   c                 K   s   | dv sJ ||ks|dks|d |d kr<| sdS | dkr,t ||f||d d|S |j||fd||d d|S t S )a  Create shortcut connection for residual blocks.

    Args:
        downsample_type: Type of downsampling ('avg', 'conv1x1', or '').
        in_chs: Input channels.
        out_chs: Output channels.
        stride: Stride for downsampling.
        dilation: Dilation rates.
        layers: Layer factory functions.
        **kwargs: Additional arguments.

    Returns:
        Shortcut module or None.
    )avgrG   rW   r!   r   Nr   r   r   )r   r   r   )r   r   r?   r   )r   r   r   r   r   r   r   rD   rD   rE   create_shortcut!  s     r   c                       s   e Zd ZdZ												
ddededededeeef dee dedede	de	de
dedef fddZd de	fddZdd Z  ZS )!
BasicBlockz$ ResNet Basic Block - kxk + kxk
       r!   r!   r!   Nr3   r   TF        r   r   r   r   r   rw   bottle_ratiorH   	attn_last
linear_outr   
drop_blockdrop_path_ratec              	      s   t t|   |pt }t|| }t||}t|||||d|d| _|j|||||d d| _	|	s7|j
d u r;t n|
|| _
|j||||d ||dd| _|	rW|j
d u r[t n|
|| _|dkrit|nt | _|
rwt | _d S |jdd	| _d S )
NFr   r   r   r   r   r   r!   )r   rj   
drop_layerr   r   Tinplace)r   r   r   r   r   r   r   shortcutr   	conv1_kxkr   r?   r   	conv2_kxkr   r   	drop_pathr   )r   r   r   r   r   r   rw   r   rH   r   r   r   r   r   mid_chsrj   r   rD   rE   r   H  s"   

""$zBasicBlock.__init__rQ   c                 C   \   |r| j d urt| jjdd d urtj| jjj | j| j	fD ]}t
|dr+|  q d S Nweightreset_parametersr   getattrr   bnr?   initzeros_r   r   r   hasattrr   r   rQ   r   rD   rD   rE   init_weightsl     "
zBasicBlock.init_weightsc                 C   X   |}|  |}| |}| |}| |}| |}| jd ur'|| | }| |S r]   )r   r   r   r   r   r   r   r   r   r   rD   rD   rE   r   s     






zBasicBlock.forward)r   r!   r   Nr3   r   TFNNr   F)r:   r;   r<   r=   rB   r   r	   rC   r>   rc   r   r   r   r   r   r   rD   rD   r   rE   r   D  sT    
	
$r   c                       s   e Zd ZdZ														d d
ededededeeef dedee dede	de	de	de	de
dedef fddZd!de	fddZdd Z  ZS )"BottleneckBlockz4 ResNet-like Bottleneck Block - 1x1 - kxk - 1x1
    r   r!   r   r3   Nr   Fr   r   r   r   r   r   r   rw   rH   r   r   
extra_conv	bottle_inr   r   r   c              	      s<  t t|   |pt }t|r|n|| }t||}t|||||d|d| _|||d| _	|j|||||d ||d| _
|rN|j||||d |d| _nt | _|	sZ|jd u r^t n||| _|j||ddd| _|	ru|jd u ryt n||| _|dkrt|nt | _|
rt | _d S |jd	d
| _d S )NFr   r!   r   r   r   rj   r   )r   rj   r   r   Tr   )r   r   r   r   r   r   r   r   r   	conv1_1x1r   
conv2b_kxkr?   r   r   	conv3_1x1r   r   r   r   )r   r   r   r   r   r   r   rw   rH   r   r   r   r   r   r   r   r   rj   r   rD   rE   r     s.   



""$zBottleneckBlock.__init__rQ   c                 C   r   r   )r   r   r   r   r?   r   r   r   r   r   r   r   r   rD   rD   rE   r     r   zBottleneckBlock.init_weightsc                 C   sl   |}|  |}| |}| |}| |}| |}| |}| |}| jd ur1|| | }| |S r]   )	r   r   r   r   r   r   r   r   r   r   rD   rD   rE   r     s   








zBottleneckBlock.forward)r   r!   r   r3   Nr   FFFFNNr   r   r:   r;   r<   r=   rB   r   rC   r	   r>   rc   r   r   r   r   r   r   rD   rD   r   rE   r     s`    
	
,r   c                       s   e Zd ZdZ												
ddededededeeef dedee dede	de	de
dedef fddZd de	fddZdd Z  ZS )!	DarkBlocka
   DarkNet-like (1x1 + 3x3 w/ stride) block

    The GE-Net impl included a 1x1 + 3x3 block in their search space. It was not used in the feature models.
    This block is pretty much a DarkNet block (also DenseNet) hence the name. Neither DarkNet or DenseNet
    uses strides within the block (external 3x3 or maxpool downsampling is done in front of the block repeats).

    If one does want to use a lot of these blocks w/ stride, I'd recommend using the EdgeBlock (3x3 /w stride + 1x1)
    for more optimal compute.
    r   r!   r   r3   Nr   TFr   r   r   r   r   r   r   rw   rH   r   r   r   r   r   c              
      s   t t|   |pt }t|| }t||}t|||||d|d| _|||d| _	|	s2|j
d u r6t n|
|| _
|j|||||d ||dd| _|	rS|j
d u rWt n|
|| _|dkret|nt | _|
rst | _d S |jdd| _d S )	NFr   r!   r   r   r   rj   r   r   r   Tr   )r   r   r   r   r   r   r   r   r   r   r   r?   r   r   r   r   r   r   r   r   r   r   r   r   r   rw   rH   r   r   r   r   r   r   rj   r   rD   rE   r     s"   

""$zDarkBlock.__init__rQ   c                 C   r   r   r   r   rD   rD   rE   r     r   zDarkBlock.init_weightsc                 C   r   r]   )r   r   r   r   r   r   r   r   rD   rD   rE   r     r   zDarkBlock.forward)r   r!   r   r3   Nr   TFNNr   r   r   rD   rD   r   rE   r     sT    
	
$r   c                       s   e Zd ZdZ												dd
ededededeeef dedee dede	de	de
dedef fddZdde	fddZdd Z  ZS ) 	EdgeBlocka   EdgeResidual-like (3x3 + 1x1) block

    A two layer block like DarkBlock, but with the order of the 3x3 and 1x1 convs reversed.
    Very similar to the EfficientNet Edge-Residual block but this block it ends with activations, is
    intended to be used with either expansion or bottleneck contraction, and can use DW/group/non-grouped convs.

    FIXME is there a more common 3x3 + 1x1 conv block to name this after?
    r   r!   r   r3   Nr   Fr   r   r   r   r   r   r   rw   rH   r   r   r   r   r   c              	      s   t t|   |pt }t|| }t||}t|||||d|d| _|j|||||d ||d| _	|	s9|j
d u r=t n|
|| _
|j||ddd| _|	rT|j
d u rXt n|
|| _|dkrft|nt | _|
rtt | _d S |jdd	| _d S )
NFr   r   r   r!   r   r   Tr   )r   r   r   r   r   r   r   r   r   r   r   r?   r   	conv2_1x1r   r   r   r   r   r   rD   rE   r     s"   

""$zEdgeBlock.__init__rQ   c                 C   r   r   )r   r   r   r   r?   r   r   r   r   r   r   r   r   rD   rD   rE   r   3  r   zEdgeBlock.init_weightsc                 C   r   r]   )r   r   r   r   r   r   r   r   rD   rD   rE   r   :  r   zEdgeBlock.forward)r   r!   r   r3   Nr   FFNNr   r   r   rD   rD   r   rE   r     sT    
	
#r   c                       s   e Zd ZdZ											d$d
ededededeeef dedee dede	de
dedef fddZd%defddZdd Zdd Zdeejejf fd d!Zdeejejf fd"d#Z  ZS )&RepVggBlockzQ RepVGG Block.

    Adapted from impl at https://github.com/DingXiaoH/RepVGG
    r   r!   r   r3   NrW   r   Fr   r   r   r   r   r   rw   rH   r   r   r   inference_modec              
      s  t t|   t|| | _}|	pt }	|r$tj||||||dd| _nJd | _||ko6|dko6|d |d k}|r@|	j	|ddnd | _
|	j|||||d ||
dd| _|	j||d||dd| _|d	kri|rit|nt | _|	jd u rwt n|	|| _|	jdd
| _d S )NTin_channelsout_channelsr   r   r   rj   biasr!   r   Fr   r   )r   rj   r   r   r   )r   r   r   r   rj   r   r?   Conv2dreparam_convr   identityr   conv_kxkconv_1x1r   r   r   r   r   )r   r   r   r   r   r   r   rw   rH   r   r   r   r   rj   	use_identr   rD   rE   r   L  s0   


 zRepVggBlock.__init__rQ   c                 C   s\   |   D ]}t|tjrtj|jdd tj|jdd qt| j	dr,| j	
  d S d S )Ng?r   r   )modulesr   r?   BatchNorm2dr   normal_r   r   r   r   r   )r   rQ   mrD   rD   rE   r   w  s   zRepVggBlock.init_weightsc                 C   s   | j d ur| | |  |S | jd u r | || | }n| |}| || | }| |}||7 }| |}| |S r]   )r  r   r   r  r  r  r   )r   r   r  rD   rD   rE   r     s   





zRepVggBlock.forwardc              
   C   s   | j durdS |  \}}tj| jjj| jjj| jjj| jjj	| jjj
| jjj| jjjdd| _ || j j_|| j j_|  D ]\}}d|v rGq>|  q>| d | d | d | d dS )	   Following works like `RepVGG: Making VGG-style ConvNets Great Again` -
        https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched
        architecture used at training time to obtain a plain CNN-like structure
        for inference.
        NTr   r   r   r   paddingr   rj   r   r  r  r  r  r   r  _get_kernel_biasr?   r   r  r   r   r   r   r   r  r   rj   r   datar   named_parametersdetach___delattr__r   kernelr   namepararD   rD   rE   reparameterize  s.   







zRepVggBlock.reparameterizerk   c           
      C   s   d}d}| j dur&| | j \}}| jjjd d }tjj|||||g}d}d}| j	dur7| | j	\}}| | j\}}|| | }|| | }	||	fS z Method to obtain re-parameterized kernel and bias.
        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L83
        r   Nr0   )
r  _fuse_bn_tensorr  r   r   r   r?   
functionalpadr  )
r   
kernel_1x1bias_1x1r  kernel_identitybias_identitykernel_conv	bias_convkernel_final
bias_finalrD   rD   rE   r    s   

zRepVggBlock._get_kernel_biasc                 C   s  t |tr|jj}|jj}|jj}|jj}|jj}|jj}nRt |t	j
s&J t| ds^| jjj}|| j }	| jjj}
t| jjj}t|D ]}d||||	 |
d d |
d d f< qF|| _| j}|j}|j}|j}|j}|j}||  }|| dddd}|| ||| |  fS ) Method to fuse batchnorm layer with preceding conv layer.
        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L95
        	id_tensorr!   r   r0   r   r   r   r   r   running_meanrunning_varr   epsr?   r  r   r  r   rj   r   r   
zeros_liker   r%  sqrtreshaper   branchr  r(  r)  gammabetar*  r   	input_dimr   kernel_valuer   stdtrD   rD   rE   r    s2   





&zRepVggBlock._fuse_bn_tensor)
r   r!   r   r3   NrW   NNr   Fr   )r:   r;   r<   r=   rB   r   rC   r	   r>   r   r   rc   r   r   r   r  r   r   r  r  r   rD   rD   r   rE   r   F  sT    	
	
+	!"r   c                       s   e Zd ZdZ												d#d
ededededeeef dedee dede	dede
dededdf fddZdejdejfddZdd Zdeejejf fdd Zdeejejf fd!d"Z  ZS )$MobileOneBlocka8   MobileOne building block.

        This block has a multi-branched architecture at train-time
        and plain-CNN style architecture at inference time
        For more details, please refer to our paper:
        `An Improved One millisecond Mobile Backbone` -
        https://arxiv.org/pdf/2206.04040.pdf
    r   r!   r   r3   NrW   Fr   r   r   r   r   r   r   rw   rH   r   r   r   r   r   rk   c                    s<  t t|   |
| _t|| | _}|pt }|	r'tj||||||dd| _	n_d| _	||ko9|dko9|d |d k}|rC|j
|ddnd| _g }t| jD ]}||j|||||dd qMt|| _d| _|dkrw|j||d||dd| _|d	kr|rt|nt | _|jdu rt n||| _|jdd
| _dS )z, Construct a MobileOneBlock module.
        Tr   Nr!   r   Fr   )r   r   rj   r   r   r   )r   r6  r   r   r   rj   r   r?   r   r  r   r  r   appendr   
ModuleListr  
conv_scaler   r   r   r   r   )r   r   r   r   r   r   r   rw   rH   r   r   r   r   r   rj   r  convsr   r   rD   rE   r     sB   

	 
zMobileOneBlock.__init__r   c                 C   s   | j dur| | |  |S d}| jdur| |}d}| jdur(| |}|}| jD ]}|||7 }q-| |}||7 }| | |S )z Apply forward pass. Nr   )r  r   r   r  r9  r  r   )r   r   identity_out	scale_outoutckrD   rD   rE   r   -  s   






zMobileOneBlock.forwardc              
   C   s   | j durdS |  \}}tj| jd jj| jd jj| jd jj| jd jj	| jd jj
| jd jj| jd jjdd| _ || j j_|| j j_|  D ]\}}d|v rUqL|  qL| d | d | d | d	 dS )
r
  Nr   Tr  r  r  r9  r  r   r  r  rD   rD   rE   r  G  s.   

	




zMobileOneBlock.reparameterizec                 C   s   d}d}| j dur(| | j \}}| jd jjd d }tjj|||||g}d}d}| j	dur9| | j	\}}d}d}t
| jD ]}| | j| \}	}
||	7 }||
7 }qB|| | }|| | }||fS r  )r9  r  r  r   r   r   r?   r  r  r  r   r   )r   kernel_scale
bias_scaler  r  r  r   r!  ix_kernel_biasr"  r#  rD   rD   rE   r  g  s&   


zMobileOneBlock._get_kernel_biasc                 C   s$  t |tr|jj}|jj}|jj}|jj}|jj}|jj}nXt |t	j
s&J t| dsd| jd jj}|| j }	| jd jj}
t| jd jj}t|D ]}d||||	 |
d d |
d d f< qL|| _| j}|j}|j}|j}|j}|j}||  }|| dddd}|| ||| |  fS )r$  r%  r   r!   r0   r&  r'  r.  rD   rD   rE   r    s2   



&zMobileOneBlock._fuse_bn_tensor)r   r!   r   r3   NrW   Fr!   NNr   )r:   r;   r<   r=   rB   r   rC   r	   r>   rc   r   r   r   r   r   r   r  r  r  r   rD   rD   r   rE   r6    s\    
	
: "r6  c                !       s   e Zd ZdZ															
d"dededededeeef dedee dede	de	de	de	deeeef  de
dedef  fddZd#de	fddZd d! Z  ZS )$SelfAttnBlockzI ResNet-like Bottleneck Block - 1x1 - optional kxk - self attn - 1x1
    r   r!   r   r3   Nr   FTr   r   r   r   r   r   r   rw   rH   r   r   r   post_attn_na	feat_sizer   r   r   c              	      s*  t t|   |d usJ t|r|n|| }t||}t|||||d|d| _|||d| _|	rD|j|||||d ||d| _	d}nt
 | _	|d u rOi nt|d}|j|fd|i|| _|rg||nt
 | _|j||ddd| _|d	kr~t|nt
 | _|
rt
 | _d S |jd
d| _d S )NFr   r!   r   r   )rF  r   r   r   Tr   )r   rD  r   r   r   r   r   r   r   r   r?   r   r_   r   r   	post_attnr   r   r   r   )r   r   r   r   r   r   r   rw   rH   r   r   r   rE  rF  r   r   r   r   rj   
opt_kwargsr   rD   rE   r     s,   

$zSelfAttnBlock.__init__rQ   c                 C   sR   |r| j d urt| jjdd d urtj| jjj t| j	dr'| j	
  d S d S r   )r   r   r   r   r?   r   r   r   r   r   r   )r   rQ   rD   rD   rE   r     s
   "zSelfAttnBlock.init_weightsc                 C   sb   |}|  |}| |}| |}| |}| |}| |}| jd ur,|| | }| |S r]   )r   r   r   rG  r   r   r   r   r   rD   rD   rE   r     s   







zSelfAttnBlock.forward)r   r!   r   r3   Nr   FFFTNNNr   r   r   rD   rD   r   rE   rD    sf    
	
.rD  )basicbottledarkedgert   r   r   r   block_fnc                 C   s   |t | < d S r]   )_block_registry)r   rM  rD   rD   rE   register_block  s   rO  blockc                 K   sF   t | tjtfr| di |S | tv sJ d|  t|  di |S )NzUnknown block type (rD   )r   r?   r@   r   rN  )rP  r   rD   rD   rE   r,     s   r,   c                       s   e Zd Z							ddedeeee eedf f d	ed
edededee de	de
f fddZdeejeej f fddZ  ZS )Stemr   re   rK   N      ?r   r   .r   r   r   num_repnum_act	chs_decayr   c
              
      s\  t    |dv sJ |	pt }	tttfrt}}
n fddt|D d d d }
|| _g | _	d}dgdg|d   }|dkrK|sKd|d< |d u rQ|n|}d	g||  d
g|  }|}d}d}t
t|
||D ]A\}\}}}|ry|	jnt}d|d  }|dkr|dkr|d }| j	t|||dd | ||||||d |}||9 }|}qm|r| }|dv sJ |}| j	t|||dd |dkr| dtd n1|dkr| dtd n#d|v r| dtjdddd nd|v r	| dtjdddd	d |d9 }d}|dkr|nd | _| j	t|||dd ||ks,J d S )N)r0   re   c                    s   g | ]
}t  |  qS rD   )roundru   r   rU  r   rD   rE   rx     rs   z!Stem.__init__.<locals>.<listcomp>r&  rW   r0   r!   re   FTr   r   num_chs	reductionmodulestage)r   r   )maxrK   r   avgpoolmax2avg2r`  r   ra  r^  r   )r   r   r  r   )r   r   r  r   )r   r   r   r   r   rz   r   r   r   feature_infor   r{   r   r   r7  r_   
add_modulelowerr?   	MaxPool2dr   last_feat_idx)r   r   r   r   r   r   rS  rT  rU  r   rN   	prev_featstem_stridesstem_norm_actsprev_chscurr_striderf  r   chr1   nalayer_fn	conv_namer   rX  rE   r     s\   

"
zStem.__init__rk   c                 C   s>   d }t | D ]\}}||}| jd ur|| jkr|}q||fS r]   )r   rf  )r   r   intermediater   r	  rD   rD   rE   forward_intermediatesF  s   zStem.forward_intermediates)r   re   rK   r   NrR  N)r:   r;   r<   rB   r
   r   r   r>   r	   rC   r   r   r   r   rq  r   rD   rD   r   rE   rQ    s8    	
&CrQ  rW   stemrJ   	pool_typefeat_prefixc           	         s  |pt  }|dv sJ d|v r"d|v rdnd }t| |d|||d}nd|v r8t| d| d	 |d |f||d
}nld|v rGt| |dd||d}n]d|v rTt| |d|d}nPd|v rbt| |dd|d}nBd|v r}|rst| |dd||d}n1|j| |ddd}n't|ttfrt| |d||d
}n|rt| |dd||d}n	|j| |ddd}t|tr fdd|jD }||fS t	|d ddg}||fS )N)	rW   quadquad2tiereddeeprt   r   7x7rI   ru  rv  r0   re   )rS  rT  r   r   rw  r   r   )r   r   rx  r3   )rS  rU  r   r   rt   )r   r   r   )r   r   r   ry     r!   )rS  r   r   r   c              	      s&   g | ]}t |d  |d gdqS ).r\  )r\  )r_   joinru   frt  rD   rE   rx   x  ry   z$create_byob_stem.<locals>.<listcomp>r   rY  )
r   rQ  r   r6  r   r   rz   r   rb  r_   )	r   r   rJ   rs  rt  r   rT  rr  rb  rD   r  rE   r+   O  s6   
$
r+   r0   c                    s"   | d u rd S t  fdd| D S )Nc                    s   g | ]}|  qS rD   rD   ru   r1   r{  rD   rE   rx         z$reduce_feat_size.<locals>.<listcomp>)rz   )rF  r   rD   r{  rE   reduce_feat_size~  s   "r  c                 C   s   | dur| n|}|pi S )a2   Override model level attn/self-attn/block kwargs w/ block level

    NOTE: kwargs are NOT merged across levels, block_kwargs will fully replace model_kwargs
    for the block if set to anything that isn't None.

    i.e. an empty block_kwargs dict will remove kwargs set at model level for that block
    NrD   )r9   model_kwargs
out_kwargsrD   rD   rE   override_kwargs  s   r  r9   	block_cfg	model_cfgc           
      C   s  | d }|j d u}|s|jd ur;|r|j sd }nt|j|j}|j p$|j }|d ur3tt|fi |nd }t||d}|jd u}|sG|jd urr|rO|jsOd }nt|j|j}	|jp[|j}|d urjtt|fi |	nd }t||d}|| d< | t|j	|j	 d S )Nr   )r   r   )
r5   r6   r  r   r   r   r7   r8   updater9   )
r9   r  r  	layer_fnsattn_setr5   r6   self_attn_setr7   r8   rD   rD   rE   update_block_kwargs  s,   

 

r  r   r   output_stride	stem_featrF  block_kwargs_fnc                 C   s  |pt  }g }dd | jD }dd |D }	dd td|t|	|	D }
d}|d }|d }|}g }t|D ]\}}|d j}|dkrM|rM|| ||kr[|dkr[||9 }d}||9 }|d	v rednd
}g }t|D ]a\}}t	|j
| j }|j}t|tr|||}t|||dkr|nd||f||j| j|
| | |d	}|jdv r||d< |||| d |t|jfi |g7 }|}|}|dkr|dkrt||}qm|tj| g7 }t||d| |d d}q9|| tj| ||fS )Nc                 S   s   g | ]}t |qS rD   )r   r  rD   rD   rE   rx     r  z&create_byob_stages.<locals>.<listcomp>c                 S   s   g | ]}t d d |D qS )c                 S   s   g | ]}|j qS rD   r   )ru   bcrD   rD   rE   rx     s    z1create_byob_stages.<locals>.<listcomp>.<listcomp>)sum)ru   	stage_bcsrD   rD   rE   rx         c                 S   s   g | ]}|  qS rD   )tolist)ru   r   rD   rD   rE   rx     r  r   r!   r[  rZ  )r!   r0   r0   )	r   r   r   r   rw   r   rH   r   r   r  rF  )r  r  stages.rY  )r   rF   r   linspacer  splitr   r1   r7  r   r/   rO   r2   r   r   r_   r4   rH   r-   r,   r  r?   
Sequential)r   r   r  r  rF  r   r  rb  r   depthsdprr   
net_striderj  rg  stages	stage_idxstage_block_cfgsr   first_dilationrF   	block_idxr  r   rw   r9   rD   rD   rE   create_byob_stages  sb   
	"







r  Tallow_aac                 C   s   t | j}t| j|d}| jr|rtt| j|| jd}ntt| j|d}| jr3tt| jfi | j	nd }| j
rDtt| j
fi | jnd }t|||||d}|S )N)rV   rT   )rV   rT   rX   )r   r   r   r   r   )r   rT   r   rV   rX   r   r   r5   r   r6   r7   r8   r   )r   r  r   r   r   r   r   rn  rD   rD   rE   get_layer_fns  s   

""r  c                       s  e Zd ZdZ								d5ded	ed
edee dedeeee	eef f  de
de
def fddZejjd6dedeeef fddZejjd7deddfddZejjdejfddZd8d	edee ddfddZ						d9d ejd!eeeee f  d"ed#ed$ed%ed&edeeej e	ejeej f f fd'd(Z	)		d:d!eeee f d*ed+edee fd,d-Zd ejdejfd.d/Zd6d ejd0edejfd1d2Zd ejdejfd3d4Z  ZS );r(   a%  Bring-your-own-blocks Network.

    A flexible network backbone that allows building model stem + blocks via
    dataclass cfg definition w/ factory functions for module instantiation.

    Current assumption is that both stem and blocks are in conv-bn-act order (w/ block ending in act).
      r   NrM   r   Tr   num_classesin_chansglobal_poolr  img_size	drop_rater   rQ   c
              	      s  t    || _|| _d| _t fi |
 t dd}t } jr+|dus+J d|dur3t|nd}g | _	t
 jttfrK fdd jD }ntt jpU jd j j }t|| j j|d\| _}| j	|dd	  t||d	 d
 d}t |||d	 ||d\| _}}| j	|dd	  |d	 d
 }|d	 d } jrtt j j | _||| jd| _n|| _t | _|  j	t | j|dt!| jdg7  _	dd | j	D | _"| j| _# j$dv sJ  j$dkr|du rd}t%| j| j#| j& j'| jd| _(| j(j)| _#nl j$dkr5|du rd}|dv s J t*| j j#|||| jdd| _(| j(j+| _#nC j$dkr^|du rBd}|dv sIJ t,| j j#|||| jdd| _(| j(j+| _#n|du red} j#du smJ t-| j||| jd| _(|| _.t/t0t1|	d|  dS )a_  
        Args:
            cfg: Model architecture configuration.
            num_classes: Number of classifier classes.
            in_chans: Number of input channels.
            global_pool: Global pooling type.
            output_stride: Output stride of network, one of (8, 16, 32).
            img_size: Image size for fixed image size models (i.e. self-attn).
            drop_rate: Classifier dropout rate.
            drop_path_rate: Stochastic depth drop-path rate.
            zero_init_last: Zero-init last weight of residual path.
            **kwargs: Extra kwargs overlayed onto cfg.
        F)r  Nz8img_size argument is required for fixed input size modelc                    s   g | ]}t t| j qS rD   )rB   rV  rO   )ru   r/   r   rD   rE   rx   /  r  z$ByobNet.__init__.<locals>.<listcomp>r   )r   r   rJ   rs  r   r&  r[  r{  )r   rF  rZ  r!   
final_convrY  c                 S   s   g | ]}|d  qS )r]  rD   r~  rD   rD   rE   rx   P  r  )rW   rZ   mlpattn_absattn_rotr  r   )hidden_sizers  rV   rT   r  r  token)rW   r  T)	embed_dimout_featuresrF  rs  r  qkv_separater  )r  r  ref_feat_sizers  r  r  )rs  r  rQ   )2r   r   r  r  grad_checkpointingr   r  rR   r    rb  r   rN   r   rz   rB   rV  rF   r/   rO   r+   rJ   rL   rr  extendr  r  r  rP   r   r  r?   r   r_   r   
stage_endsrY   r[   r   rV   rT   headr  r   r  r   r   r  r$   r   _init_weights)r   r   r  r  r  r  r  r  r   rQ   r   stem_layersstage_layersrF  rN   r  
stage_featr[  rj  r   r   rE   r     s   
 
	
	
	
zByobNet.__init__Fcoarserk   c                 C   s    t d|rdnddfdgd}|S )zGroup matcher for parameter groups.

        Args:
            coarse: Whether to use coarse grouping.

        Returns:
            Dictionary mapping group names to patterns.
        z^stemz^stages\.(\d+)z^stages\.(\d+)\.(\d+)N)z^final_conv)i )rr  rF   r^   )r   r  matcherrD   rD   rE   group_matcher  s   
zByobNet.group_matcherenablec                 C   s
   || _ dS )zEnable or disable gradient checkpointing.

        Args:
            enable: Whether to enable gradient checkpointing.
        N)r  )r   r  rD   rD   rE   set_grad_checkpointing  s   
zByobNet.set_grad_checkpointingc                 C   s   | j jS )zPGet classifier module.

        Returns:
            Classifier module.
        )r  fcr   rD   rD   rE   get_classifier  s   zByobNet.get_classifierc                 C   s   || _ | j|| dS )zReset classifier.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        N)r  r  reset)r   r  r  rD   rD   rE   reset_classifier  s   zByobNet.reset_classifierNCHWr   indicesnorm
stop_early
output_fmtintermediates_onlyexclude_final_convc                    sF  |dv sJ dg }t t j|\}	}
 fdd|	D }	 j|
 }
d}t jdr3 j|\}}n |d}}||	v rJ||du rG|n|  jd }tj	 sV|sZ j
}n j
d|
 }|D ],}|d	7 } jrwtj	 swt||}n||}|s||kr |}||	v r|| qc|r|S |r||kr |}||fS )
a9   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
            exclude_final_conv: Exclude final_conv from last intermediate
        Returns:

        )r  zOutput shape must be NCHW.c                    s   g | ]} j | qS rD   )r  rW  r  rD   rE   rx     s    z1ByobNet.forward_intermediates.<locals>.<listcomp>r   rq  Nr&  r!   )r#   r   r  r   rr  rq  r7  r   jitis_scriptingr  r  r%   r  )r   r   r  r  r  r  r  r  intermediatestake_indices	max_indexfeat_idxx_interlast_idxr  r]  rD   r  rE   rq    s<   




zByobNet.forward_intermediatesr!   
prune_norm
prune_headc                 C   sZ   t t| j|\}}| j| }| jd| | _|| jd k r#t | _|r+| dd |S )aE  Prune layers not required for specified intermediates.

        Args:
            indices: Indices of intermediate layers to keep.
            prune_norm: Whether to prune normalization layer.
            prune_head: Whether to prune the classifier head.

        Returns:
            List of indices that were kept.
        Nr&  r   rW   )r#   r   r  r  r?   r   r  r  )r   r  r  r  r  r  rD   rD   rE   prune_intermediate_layers  s   

z!ByobNet.prune_intermediate_layersc                 C   s@   |  |}| jrtj st| j|}n| |}| |}|S )zForward pass through feature extraction.

        Args:
            x: Input tensor.

        Returns:
            Feature tensor.
        )rr  r  r   r  r  r%   r  r  r   rD   rD   rE   forward_features  s   
	

zByobNet.forward_features
pre_logitsc                 C   s   |r	| j ||dS |  |S )zForward pass through head.

        Args:
            x: Input features.
            pre_logits: Return features before final linear layer.

        Returns:
            Classification logits or features.
        )r  )r  )r   r   r  rD   rD   rE   forward_head#  s   
zByobNet.forward_headc                 C   s   |  |}| |}|S )zoForward pass.

        Args:
            x: Input tensor.

        Returns:
            Output logits.
        )r  r  r   rD   rD   rE   r   /  s   
	
zByobNet.forward)r  r   NrM   Nr   r   Tr   Tr]   )NFFr  FF)r!   FT) r:   r;   r<   r=   r)   rB   r	   r>   r
   r   rC   rc   r   r   r  ignorer   r   r  r  r?   r@   r  r  r   r   rq  r  r  r  r   r   rD   rD   r   rE   r(     s    	
  	
@
r(   r\  r  rQ   c                 C   s   t | tjr4| jd | jd  | j }|| j }| jjdt	
d|  | jdur2| jj  dS dS t | tjrTtjj| jddd | jdurRtj| j dS dS t | tjrjtj| j tj| j dS t| drw| j|d	 dS dS )
zInitialize weights.

    Args:
        module: Module to initialize.
        name: Module name.
        zero_init_last: Zero-initialize last layer.
    r   r!          @Nr   g{Gz?)meanr4  r   r  )r   r?   r   r   r   rj   r   r  r  mathr,  r   zero_Linearr   r   r  ones_r   r   )r\  r  rQ   fan_outrD   rD   rE   r  =  s$   



r  gernet_lrI  rn   r3   )r-   r.   r/   r1   r2   r4      rJ  rf   i  g      ?         @re   rM   i 
  )rF   rN   rL   rP   gernet_mgernet_s0   r   rz    i0  ro      i  	repvgg_a0)r0   re      r!   )      ?r  r        @)r.   ri   rt   )rF   rJ   rN   	repvgg_a1)r!   r!   r!   r  rm   	repvgg_a2)      ?r  r  g      @	repvgg_b0)r3   r3   r3   r  )ri   	repvgg_b1)r  r  r        @repvgg_b1g4)ri   rj   	repvgg_b2)r  r  r        @repvgg_b2g4	repvgg_b3)r  r  r  r  repvgg_b3g4repvgg_d2se)r   r     r!   r   g      ?)rd_ratio
rd_divisor)rF   rJ   rN   r5   r6   	resnet51qrp   i   rv  i   silu)rF   rN   rJ   rL   rP   rT   	resnet61qrL  )r-   r.   r/   r1   r2   r4   r9   ru  )r   )rF   rN   rJ   rL   rP   rT   r9   resnext26tsi   rw  rK   )rF   rN   rJ   rL   rT   gcresnext26tsgca)rF   rN   rJ   rL   rT   r5   seresnext26tseca_resnext26tsecabat_resnext26tsbatr   )
block_size)rF   rN   rJ   rL   rT   r5   r6   
resnet32ts
resnet33tsi   gcresnet33ts)rF   rN   rJ   rL   rP   rT   r5   seresnet33tseca_resnet33tsgcresnet50t)r-   r.   r/   r1   r4   )rF   rN   rJ   rL   r5   gcresnext50tsregnetz_b16rg   `         )r  )r   r   )	rF   rN   rL   rH   rP   rT   r5   r6   r9   regnetz_c16regnetz_d32i   )
rF   rN   rJ   rL   rH   rP   rT   r5   r6   r9   
regnetz_d8
regnetz_e8regnetz_b16_evosrv   )
rF   rN   rL   rH   rP   rT   rV   r5   r6   r9   regnetz_c16_evosregnetz_d8_evosrx  )rF   rN   rJ   rL   rH   rP   rT   rV   r5   r6   r9   mobileone_s0)r  r3   r3   r  )ri   r   r   mobileone_s1)r  r  r  r  mobileone_s2)r  r  r  r  mobileone_s3)r  r  r  r  mobileone_s4)r        @r'  r  )r   r   r  r!   )ri   r   resnet50_clip)rM   rM   rm   ra  r   r  )rF   rN   rJ   rL   rH   rX   r[   resnet101_clip   resnet50x4_clipr   g      ?)rF   rO   rN   rJ   rL   rH   rX   r[   resnet50x16_clip   r  resnet50x64_clip   $   r  resnet50_mlpr  )rF   rN   rJ   rL   rH   rX   rY   r[   test_byobnetrR  rK  r  rS   )rF   rN   rH   rL   rT   r5   r6   )r(  r)  r+  r,  r.  rZ   )r[   _gapvisual.
state_dictmodelprefixc           
      C   s  t |jttf}dd l}dd }dd }i }|  D ]g\}}	||s$q|| dd|}|| dd	|}|| d
||}|| d||}|| dr}|sWq||d d}|dd}|dd}|dd}|dd}|dd}|	||< q|S )Nr   c                 S   sn   t | dd }t | d| dt | d}}}d| d| d}ddd	d
}|| | }|| S )Nr!   r0   r   re   r  r|  z
conv1_1x1.z
conv2_kxk.z
conv3_1x1.)r!   r0   r   rB   group)r	  r  	layer_idx
layer_typelayer_id
prefix_strid_map
suffix_strrD   rD   rE   
_stage_sub  s   *z(_convert_openai_clip.<locals>._stage_subc                 S   sT   t | dd }t | dt | d}}d| d| d|dkr'd S d	 S )
Nr!   r0   r   r  r|  z
.shortcut.r   z	conv.convzconv.bnr8  )r	  r  r:  r<  rD   rD   rE   	_down_sub  s   $z'_convert_openai_clip.<locals>._down_subzconv([0-9])zstem.conv\1.convz	bn([0-9])zstem.conv\1.bnz'layer([0-9])\.([0-9]+)\.([a-z]+)([0-9])z+layer([0-9])\.([0-9]+)\.downsample\.([0-9])attnpoolr  positional_embedding	pos_embedq_projqk_projkv_projvc_projproj)	r   r  r   r   reitems
startswithsubr   )
r5  r6  r7  model_has_attn_poolrM  r@  rA  out_dictrH  rJ  rD   rD   rE   _convert_openai_clip  s.   

rS  c                 C   s   d| v r	t | |} | S )Nzvisual.conv1.weight)rS  )r5  r6  rD   rD   rE   checkpoint_filter_fn  s   
rT  variant
pretrainedc                 K   s&   t t| |ft|  ttddd|S )zCreate a ByobNet model.

    Args:
        variant: Model variant name.
        pretrained: Load pretrained weights.
        **kwargs: Additional model arguments.

    Returns:
        ByobNet model instance.
    T)flatten_sequential)r  pretrained_filter_fnfeature_cfg)r"   r(   
model_cfgsrT  r_   )rU  rV  r   rD   rD   rE   _create_byobnet  s   r[  urlc                 K      | dddddt tddd
|S )	zCreate default configuration dictionary.

    Args:
        url: Model weight URL.
        **kwargs: Additional configuration options.

    Returns:
        Configuration dictionary.
    r  r      r_  rz  rz        ?bilinear	stem.convhead.fc
r\  r  
input_size	pool_sizecrop_pctinterpolationr  r4  
first_convrZ   r   r   r\  r   rD   rD   rE   _cfg     rm  c                 K   r]  )	zCreate RepVGG configuration dictionary.

    Args:
        url: Model weight URL.
        **kwargs: Additional configuration options.

    Returns:
        Configuration dictionary.
    r  r   ro   ro   r   r   ?bicubiczstem.conv1.convrd  re  rk  rl  rD   rD   rE   _cfgr  rn  rs  zgernet_s.idstcv_in1kztimm/)	hf_hub_idzgernet_m.idstcv_in1kzgernet_l.idstcv_in1kro  rp  )rt  rf  rg  zrepvgg_a0.rvgg_in1k)zstem.conv_kxk.convzstem.conv_1x1.convmit)rt  rj  licensezrepvgg_a1.rvgg_in1kzrepvgg_a2.rvgg_in1kzrepvgg_b0.rvgg_in1kzrepvgg_b1.rvgg_in1kzrepvgg_b1g4.rvgg_in1kzrepvgg_b2.rvgg_in1kzrepvgg_b2g4.rvgg_in1kzrepvgg_b3.rvgg_in1kzrepvgg_b3g4.rvgg_in1kzrepvgg_d2se.rvgg_in1k)r   @  rw  )r   r   )rt  rj  rv  rf  rg  rh  zresnet51q.ra2_in1kzkhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet51q_ra2-d47dcc76.pthz
stem.conv1)r   r  r  )rt  r\  rj  rf  rg  test_input_sizetest_crop_pctzresnet61q.ra2_in1kzkhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet61q_ra2-6afc536c.pth)rt  r\  rx  ry  zresnext26ts.ra2_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnext26ts_256_ra2-8bbd9106.pthzseresnext26ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/seresnext26ts_256-6f0d74a3.pthzgcresnext26ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnext26ts_256-e414378b.pthzeca_resnext26ts.ch_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/eca_resnext26ts_256-5a1d030f.pthzbat_resnext26ts.ch_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/bat_resnext26ts_256-fa6fd595.pth)rt  r\  min_input_sizezresnet32ts.ra2_in1kzqhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnet32ts_256-aacf5250.pthzresnet33ts.ra2_in1kzqhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnet33ts_256-e91b09a4.pthzgcresnet33ts.ra2_in1kzshttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnet33ts_256-0e0cd345.pthzseresnet33ts.ra2_in1kzshttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/seresnet33ts_256-f8ad44d9.pthzeca_resnet33ts.ra2_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/eca_resnet33ts_256-8f98face.pthzgcresnet50t.ra2_in1kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnet50t_256-96374d1c.pthzgcresnext50ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnext50ts_256-3e0f515e.pthzregnetz_b16.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_b_raa-677d9606.pthrc  )rR  rR  rR  r^  r`  gGz?)
rt  r\  rj  r  r4  rf  rg  rh  rx  ry  zregnetz_c16.ra3_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_c_rab2_256-a54bf36a.pth)rt  r\  rj  r  r4  rh  rx  ry  zregnetz_d32.ra3_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_d_rab_256-b8073a89.pthgffffff?)rt  r\  r  r4  rh  rx  zregnetz_d8.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_d8_bh-afc03c55.pth)rt  r\  r  r4  rh  rx  ry  zregnetz_e8.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_e8_bh-aace8e6e.pthzregnetz_b16_evos.untrained)rj  r  r4  rf  rg  rh  rx  zregnetz_c16_evos.ch_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/regnetz_c16_evos_ch-d8311942.pth)rt  r\  rj  r  r4  rh  rx  zregnetz_d8_evos.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/regnetz_d8_evos_ch-2bc12646.pthzmobileone_s0.apple_in1kra  )zstem.conv_kxk.0.convzstem.conv_scale.conv)rt  rh  rj  zmobileone_s1.apple_in1krq  zmobileone_s2.apple_in1kzmobileone_s3.apple_in1kzmobileone_s4.apple_in1kzresnet50_clip.openaiz	head.proj)rt  r  r  r4  rR   rf  rg  rZ   zresnet101_clip.openaizresnet50x4_clip.openai)	   r{  zresnet50x16_clip.openaii   )r   r  r  )r  r  zresnet50x64_clip.openai)r     r|  )r  r  zresnet50_clip.cc12mzresnet50_clip.yfcc15mzresnet101_clip.yfcc15mzresnet50_clip_gap.openai)rt  r  r  r4  rf  rg  zresnet101_clip_gap.openai)rf  rg  )r      r}  )r  r  )rt  rj  rf  rh  rg  )zresnet50x4_clip_gap.openaizresnet50x16_clip_gap.openaizresnet50x64_clip_gap.openaizresnet50_clip_gap.cc12mzresnet50_clip_gap.yfcc15mzresnet101_clip_gap.yfcc15mzresnet50_mlp.untrainedztest_byobnet.r160_in1kc                 K      t dd| i|S )z GEResNet-Large (GENet-Large from official impl)
    `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
    r  rV  N)r  r[  rV  r   rD   rD   rE   r  
     c                 K   r~  )z GEResNet-Medium (GENet-Normal from official impl)
    `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
    r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )z EResNet-Small (GENet-Small from official impl)
    `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
    r  rV  N)r  r  r  rD   rD   rE   r  $
  r  c                 K   r~  )z^ RepVGG-A0
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  ,
  r  c                 K   r~  )z^ RepVGG-A1
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  4
  r  c                 K   r~  )z^ RepVGG-A2
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  <
  r  c                 K   r~  )z^ RepVGG-B0
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  D
  r  c                 K   r~  )z^ RepVGG-B1
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  L
  r  c                 K   r~  )z` RepVGG-B1g4
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  T
  r  c                 K   r~  )z^ RepVGG-B2
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  \
  r  c                 K   r~  )z` RepVGG-B2g4
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  d
  r  c                 K   r~  )z^ RepVGG-B3
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  l
  r  c                 K   r~  )z` RepVGG-B3g4
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r  rV  N)r  r  r  rD   rD   rE   r  t
  r  c                 K   r~  )z` RepVGG-D2se
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r   rV  N)r   r  r  rD   rD   rE   r   |
  r  c                 K   r~  )
    r  rV  N)r  r  r  rD   rD   rE   r  
     c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r
  rV  N)r
  r  r  rD   rD   rE   r
  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  
  r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r    r  c                 K   r~  )r  r  rV  N)r  r  r  rD   rD   rE   r  	  r  c                 K   r~  )r  r   rV  N)r   r  r  rD   rD   rE   r     r  c                 K   r~  )r  r!  rV  N)r!  r  r  rD   rD   rE   r!    r  c                 K   r~  )r  r"  rV  N)r"  r  r  rD   rD   rE   r"    r  c                 K   r~  )r  r#  rV  N)r#  r  r  rD   rD   rE   r#  %  r  c                 K   r~  )r  r$  rV  N)r$  r  r  rD   rD   rE   r$  ,  r  c                 K   r~  )r  r%  rV  N)r%  r  r  rD   rD   rE   r%  3  r  c                 K   r~  )r  r&  rV  N)r&  r  r  rD   rD   rE   r&  :  r  c                 K   r~  )z0 OpenAI Modified ResNet-50 CLIP image tower
    r(  rV  N)r(  r  r  rD   rD   rE   r(  A  r  c                 K   r~  )z1 OpenAI Modified ResNet-101 CLIP image tower
    r)  rV  N)r)  r  r  rD   rD   rE   r)  H  r  c                 K   r~  )z2 OpenAI Modified ResNet-50x4 CLIP image tower
    r+  rV  N)r+  r  r  rD   rD   rE   r+  O  r  c                 K   r~  )z3 OpenAI Modified ResNet-50x16 CLIP image tower
    r,  rV  N)r,  r  r  rD   rD   rE   r,  V  r  c                 K   r~  )z3 OpenAI Modified ResNet-50x64 CLIP image tower
    r.  rV  N)r.  r  r  rD   rD   rE   r.  ]  r  c                 K   r~  )zP OpenAI Modified ResNet-50 CLIP image tower w/ avg pool (no attention pool)
    resnet50_clip_gaprV  N)r  r  r  rD   rD   rE   r  d  r  r  c                 K   r~  )zQ OpenAI Modified ResNet-101 CLIP image tower w/ avg pool (no attention pool)
    resnet101_clip_gaprV  N)r  r  r  rD   rD   rE   r  k  r  r  c                 K   r~  )zR OpenAI Modified ResNet-50x4 CLIP image tower w/ avg pool (no attention pool)
    resnet50x4_clip_gaprV  N)r  r  r  rD   rD   rE   r  r  r  r  c                 K   r~  )zS OpenAI Modified ResNet-50x16 CLIP image tower w/ avg pool (no attention pool)
    resnet50x16_clip_gaprV  N)r  r  r  rD   rD   rE   r  y  r  r  c                 K   r~  )zS OpenAI Modified ResNet-50x64 CLIP image tower w/ avg pool (no attention pool)
    resnet50x64_clip_gaprV  N)r  r  r  rD   rD   rE   r    r  r  c                 K   r~  )r  r1  rV  N)r1  r  r  rD   rD   rE   r1    r  c                 K   r~  )z- Minimal test ResNet (BYOB based) model.
    r2  rV  N)r2  r  r  rD   rD   rE   r2    r  )rd   rh   r   )r~   rh   rD   r!   )r!   F)rW   rW   rr  N)r0   r  )rW   F)r4  r   )rW   )r=   r  dataclassesr   r   r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   torch.nnr?   	timm.datar   r   r   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r    _builderr"   	_featuresr#   _manipulater$   r%   	_registryr&   r'   __all__r*   r)   rB   rC   r}   r   r>   rc   r   r   r   r   r@   r   r   r   r   r   r   r   r6  rD  r_   rN  rO  r,   r  rQ  r+   r  r  r  r  r  r(   r  rZ  rH  r   rS  rT  r[  rm  rs  default_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r
  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r!  r"  r#  r$  r%  r&  r(  r)  r+  r,  r.  r  r  r  r  r  r1  r2  rD   rD   rD   rE   <module>   s|
   (D"!






(


" 	/

#;EB@ $ >EQ
/)

?   A&
+
0
5:?
DI
NS
X

br       *  7  G  V  f  t               .
    >
    N
    _
    p
     

      

      &

      9
      >      C      H      M
      S      a      o      ~                       .        
="+"	$$
"%
,17;?CGMQUY]bgmr
w{             !  &  -
  3
  9
  ?
  E
  K
  Q
  W
  _  d  i    