o پi_f@spdZdgZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl m Z ddlm mZddlmZmZddlmZmZmZmZmZmZdd lmZdd lmZdd lm Z dd l!m"Z"m#Z#dd l$m%Z%m&Z&Gddde j j'Z(Gddde j)Z*Gddde j)Z+Gddde j)Z,Gddde j)Z-Gddde j)Z.Gddde j j)Z/Gddde j)Z0e e0Gddde j)Z1Gd dde j)Z2d!d"Z3d@d$d%Z4e&e4d&d'd(e4d&d)e4d&d)e4d&d'd(e4d&d)e4d&d)e4d&d'd(e4d&d)e4d&d)e4d&d*d+d,d-e4d&d.d/d,d0d1d2 Z5dAd4d5Z6e%dAd6d7Z7e%dAd8d9Z8e%dAd:d;Z9e%dAdd?Z;dS)Bz TinyViT Paper: `TinyViT: Fast Pretraining Distillation for Small Vision Transformers` - https://arxiv.org/abs/2207.10666 Adapted from official impl at https://github.com/microsoft/Cream/tree/main/TinyViT TinyVitN)partial)DictListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) LayerNorm2dNormMlpClassifierHeadDropPath trunc_normal_resize_rel_pos_bias_table_levituse_fused_attn)build_model_with_cfg)feature_take_indices)register_notrace_module) checkpointcheckpoint_seq)register_modelgenerate_default_cfgscs.eZdZdfdd ZeddZZS)ConvNormrrc s^ttj|||||||dd|_t||_tjj |jj |tjj |jj ddS)NF)biasr) super__init__nnConv2dconv BatchNorm2dbntorchinit constant_weightr) selfin_chsout_chsksstridepaddilationgroupsbn_weight_init __class__H/home/ubuntu/.local/lib/python3.10/site-packages/timm/models/tiny_vit.pyrs  zConvNorm.__init__c Cs|j|j}}|j|j|jd}|j|dddddf}|j|j|j|j|jd}tjj | d|jj | d|j dd|jj |jj|jj|jj d}|jj||jj||S)Ng?rr)r+paddingr-r.)r r"r& running_varepsr running_meanr#rrsizer.shaper+r5r-datacopy_)r'cr"wbmr2r2r3fuse%s$z ConvNorm.fuse)rrrrrr)__name__ __module__ __qualname__rr#no_gradrA __classcell__r2r2r0r3rsrc$eZdZfddZddZZS) PatchEmbedcsHtd|_t||dddd|_||_t|d|ddd|_dS)Nr4r)rrr+rconv1actconv2)r'r(r) act_layerr0r2r3r5s zPatchEmbed.__init__cCs"||}||}||}|SN)rKrLrMr'xr2r2r3forward<s   zPatchEmbed.forwardrBrCrDrrRrFr2r2r0r3rH4s rHcrG)MBConvcstt||}t||dd|_||_t||ddd|d|_||_t||ddd|_||_ |dkr>t ||_ dSt |_ dS)Nr)r*rJr*r+r,r.)r*r/)rrintrrKact1rMact2conv3act3rrIdentity drop_path)r'r(r) expand_ratiorNr]mid_chsr0r2r3rDs  $zMBConv.__init__cCsV|}||}||}||}||}||}||}||7}||}|SrO)rKrXrMrYrZr]r[)r'rQshortcutr2r2r3rROs       zMBConv.forwardrSr2r2r0r3rTCs  rTcrG) PatchMergingcsXtt||ddd|_||_t||ddd|d|_||_t||ddd|_dS)NrrrJr4)r.)rrrrKrXrMrYrZ)r'dimout_dimrNr0r2r3r]s zPatchMerging.__init__cCs6||}||}||}||}||}|SrO)rKrXrMrYrZrPr2r2r3rRes     zPatchMerging.forwardrSr2r2r0r3ra\s racs*eZdZ  dfdd ZddZZS) ConvLayerrV@cs>t|_||_tjfddt|D|_dS)Nc s.g|]}tttr|nqSr2)rT isinstancelist.0irNconv_expand_ratiorbr]r2r3 zsz&ConvLayer.__init__..)rrrbdepthr Sequentialrangeblocks)r'rbrnrNr]rlr0rkr3ros zConvLayer.__init__cCs||}|SrO)rqrPr2r2r3rRs zConvLayer.forward)rVrerSr2r2r0r3rdns rdcs4eZdZddejejdffdd ZddZZS)NormMlpNrVcsdt|p|}|p |}|||_t|||_||_t||_t|||_ t||_ dSrO) rrnormrLinearfc1rLDropoutdrop1fc2drop2)r' in_featureshidden_features out_features norm_layerrNdropr0r2r3rs   zNormMlp.__init__cCs@||}||}||}||}||}||}|SrO)rsrurLrwrxryrPr2r2r3rRs      zNormMlp.forward) rBrCrDr LayerNormGELUrrRrFr2r2r0r3rrsrrcs|eZdZUejjeed<ee ej fed<   dfdd Z e dfd d Z d ejd ej fd dZddZZS) Attention fused_attnattention_bias_cacherIrc sptt|trt|dksJ||_|d|_||_t|||_ |j ||_ ||_ ||_ t |_t||_t|||j d||_t|j ||_ttt|dt|d}t|}i}g} |D],} |D]'} t| d| dt| d| df} | |vrt||| <| || qnqjtjt|t||_|jdt | !||ddi|_"dS)Nr4grrattention_bias_idxsF) persistent)#rrrftuplelen num_headsscalekey_dimrWval_dimrc attn_ratio resolutionrrrrrsrtqkvprojrg itertoolsproductrpabsappendr# Parameterzerosattention_biasesregister_buffer LongTensorviewr) r'rbrrrrpointsNattention_offsetsidxsp1p2offsetr0r2r3rs6     (  zAttention.__init__Tcs(t||r|jri|_dSdSdSrO)rtrainr)r'moder0r2r3rs   zAttention.traindevicereturncCsZtjs|jr|jdd|jfSt|}||jvr(|jdd|jf|j|<|j|SrO)r#jit is_tracingtrainingrrstrr)r'r device_keyr2r2r3get_attention_biasess   zAttention.get_attention_biasesc Cs||j}|j\}}}||}||}||||jdj|j|j|j gdd\}}} | dddd}| dddd}| dddd} |j rRt j ||| |d}n||j}||dd} | |} | jdd} | | }|dd|||j}||}|S) NrJ)rbrr4r) attn_mask)rrr:rsrrrsplitrrpermuterFscaled_dot_product_attentionr transposesoftmaxreshapercr) r'rQ attn_biasBr_rqkvattnr2r2r3rRs$    .   zAttention.forward)rrIrT)rBrCrDr#rFinalbool__annotations__rrTensorrrErrrrRrFr2r2r0r3rs % rcsFeZdZdZdddddejffdd Zdd Zd efd d Z Z S) TinyVitBlocka5 TinyViT Block. Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): Window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. drop (float, optional): Dropout rate. Default: 0.0 drop_path (float, optional): Stochastic depth rate. Default: 0.0 local_conv_size (int): the kernel size of the convolution between Attention and MLP. Default: 3 act_layer: the activation function. Default: nn.GELU rerVrJc st||_||_|dksJd||_||_||dks#Jd||} ||f} t|| |d| d|_|dkr=t|nt |_ t |t ||||d|_|dkrWt|nt |_|d} t|||d| |d |_dS) Nrz"window_size must be greater than 0z"dim must be divisible by num_headsr)rrrV)rzr{rNr~r4rU)rrrbr window_size mlp_ratiorrrrr\ drop_path1rrrWmlp drop_path2r local_conv) r'rbrrrr~r]local_conv_sizerNhead_dimwindow_resolutionr,r0r2r3rs(  zTinyVitBlock.__init__c Cs|j\}}}}||}|}||jkr,||jkr,||||}||}|||||}n|j||j|j}|j||j|j} |dkpI| dk} | rXt|ddd| d|f}|||| } } | |j} | |j}||| |j||j|dd|| ||j|j|}||}||| ||j|j|dd|| | |}| r|ddd|d|f}|| |}| dddd}| |}||||dd}|| | |}|||||SNrr4rJr)r:rrrrrr,r contiguousrrrrr)r'rQrHWCLr`pad_bpad_rr5pHpWnHnWr2r2r3rR+s8     , zTinyVitBlock.forwardrcCs$d|jd|jd|jd|jS)Ndim=z , num_heads=z, window_size=z , mlp_ratio=)rbrrrr'r2r2r3 extra_reprSs zTinyVitBlock.extra_repr rBrCrD__doc__rrrrRrrrFr2r2r0r3rs%(rcsFeZdZdZdddddejffdd Zdd Zd efd d Z Z S) TinyVitStagea A basic TinyViT layer for one stage. Args: dim (int): Number of input channels. out_dim: the output dimension of the layer depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. drop (float, optional): Dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None local_conv_size: the kernel size of the depthwise convolution between attention and MLP. Default: 3 act_layer: the activation function. Default: nn.GELU rerVNrJc svt||_|_| dur| |d|_n t|_|ks#Jtjfddt|D|_ dS)N)rbrcrNc s6g|]}tttr|ndqS))rbrrrr~r]rrN)rrfrgrhrNr~r]rrrrcrr2r3rms z)TinyVitStage.__init__..) rrrnrc downsamplerr\rorprq) r'rbrcrnrrrr~r]rrrNr0rr3rls    zTinyVitStage.__init__cCs8||}|dddd}||}|dddd}|Sr)rrrqrPr2r2r3rRs  zTinyVitStage.forwardrcCsd|jd|jS)Nrz, depth=)rcrnrr2r2r3rszTinyVitStage.extra_reprrr2r2r0r3r[s,rcsdeZdZddddddddd d d ddejffd d ZddZejj ddZ ejj ddZ ejj d6ddZ ejj d7ddZ ejj dejfddZd8dedeefdd Z  ! d9d"ejd#eeeeefd$ed%ed&ed'edeeejeejeejfffd(d)Z * d:d#eeeefd+ed,efd-d.Zd/d0Zd6d1efd2d3Zd4d5ZZS);rrJavg)`ir4r4r4)rJr rrrrrerV皙?Fcst||_||_t||_||_| |_t||d|d|_ ddt d| t |D}t |_|j j}|d}g|_t|jD]f}|dkr[t|||||d||| d}n8||}|t |d|t |d|d} t||d||||||||j| | | t|d }|}|d9}|j||jt||d |d g7_qC|d |_|_ttd d }t|j|||d|_||jdS)Nr)r(r)rNcSsg|]}|qSr2)itemrirQr2r2r3rmsz$TinyVit.__init__..)rbrnrNr]rlr) rbrcrnrrrr~rr]rrNr4zstages.)num_chs reductionmodulergh㈵>)r7) pool_typer}) rr num_classesdepthsr num_stagesrgrad_checkpointingrH patch_embedr#linspacesumrrostagesr+ feature_inforprdrrardict num_featureshead_hidden_sizerr r headapply _init_weights)r'in_chansr global_pool embed_dimsrr window_sizesr drop_ratedrop_path_rateuse_checkpointmbconv_expand_ratiorrNdprr+prev_dim stage_idxstagerc norm_layer_cfr0r2r3rsj   (   " zTinyVit.__init__cCsPt|tjr"t|jddt|tjr$|jdur&tj|jddSdSdSdS)Ng{Gz?)stdr)rfrrtrr&rr$r%)r'r@r2r2r3rs zTinyVit._init_weightscCsdhS)Nrr2rr2r2r3no_weight_decay_keywordssz TinyVit.no_weight_decay_keywordscCsdd|DS)NcSsh|]}d|vr|qS)rr2rr2r2r3 sz*TinyVit.no_weight_decay..) state_dictkeysrr2r2r3no_weight_decayszTinyVit.no_weight_decaycCstd|rdnddgd}|S)Nz ^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r)z^stages\.(\d+)\.\w+\.(\d+)N)stemrq)r)r'coarsematcherr2r2r3 group_matcherszTinyVit.group_matcherTcCs ||_dSrO)r)r'enabler2r2r3set_grad_checkpointings zTinyVit.set_grad_checkpointingrcCs|jjSrO)rfcrr2r2r3get_classifierszTinyVit.get_classifierNrrcCs||_|jj||ddS)N)r)rrreset)r'rrr2r2r3reset_classifierszTinyVit.reset_classifierNCHWrQindicesrs stop_early output_fmtintermediates_onlyc Cs|dvsJdg}tt|j|\}} ||}tjs |s$|j} n |jd| d} t| D]\} } |jrCtjsCt | |}n| |}| |vrP| |q1|rU|S||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )r zOutput shape must be NCHW.Nr) rrrrr#r is_scripting enumeraterrr) r'rQr!rsr"r#r$ intermediates take_indices max_indexrfeat_idxrr2r2r3forward_intermediatess"   zTinyVit.forward_intermediatesr prune_norm prune_headcCs<tt|j|\}}|jd|d|_|r|dd|S)z@ Prune layers not required for specified intermediates. Nrr)rrrr)r'r!r,r-r(r)r2r2r3prune_intermediate_layersIs  z!TinyVit.prune_intermediate_layerscCs8||}|jrtjst|j|}|S||}|SrO)rrr#rr%rrrPr2r2r3forward_featuresWs   zTinyVit.forward_features pre_logitscCs$|r |j||d}|S||}|S)N)r1)r)r'rQr1r2r2r3 forward_head_s zTinyVit.forward_headcCs||}||}|SrO)r0r2rPr2r2r3rRcs  zTinyVit.forwardFrrO)NFFr F)rFT)rBrCrDrrrrr#rignorerrrrModulerrWrrrrrrrrr+r/r0r2rRrFr2r2r0r3rszS      / cCspd|vr |d}|}i}|D]!\}}|drqd|vr1t|j||jdddj}|||<q|S)Nmodelrrr)rritemsendswithrTr:)rr6 target_sdout_dictrrr2r2r3checkpoint_filter_fnis   r<r.c Ks|dttdddddd |S)Nrzpatch_embed.conv1.convzhead.fc)rr)rJr=gffffff?) urlrmeanr first_conv classifier pool_size input_sizecrop_pctr )r>kwargsr2r2r3_cfgxs rFztimm/iQU) hf_hub_idr)rG)rJrr)rrg?)rGrCrBrD)rJrH)rIsquash)rGrCrBrD crop_mode) ztiny_vit_5m_224.dist_in22kz"tiny_vit_5m_224.dist_in22k_ft_in1kztiny_vit_5m_224.in1kztiny_vit_11m_224.dist_in22kz#tiny_vit_11m_224.dist_in22k_ft_in1kztiny_vit_11m_224.in1kztiny_vit_21m_224.dist_in22kz#tiny_vit_21m_224.dist_in22k_ft_in1kztiny_vit_21m_224.in1kz#tiny_vit_21m_384.dist_in22k_ft_in1kz#tiny_vit_21m_512.dist_in22k_ft_in1kFcKs2|dd}tt||ftd|dtd|}|S)N out_indices)rrr4rJT)flatten_sequentialrL) feature_cfgpretrained_filter_fn)poprrrr<)variant pretrainedrErLr6r2r2r3_create_tiny_vits  rScK>tgdgdgdgddd}||td|fi|S)N)@i@r)r4rI rrVrrrrrtiny_vit_5m_224rupdaterSrRrE model_kwargsr2r2r3r[ r[cKrT)N)rUrVir)r4rIrrrrrZtiny_vit_11m_224r\r^r2r2r3rbr`rbcKrT)Nrrri@rrJrrrg?rZtiny_vit_21m_224r\r^r2r2r3rfr`rfcKrT)Nrcrrd)rrrrrrZtiny_vit_21m_384r\r^r2r2r3rgr`rgcKrT)Nrcrrd)rIrI rIrrZtiny_vit_21m_512r\r^r2r2r3rir`ri)r.r3)s      RaHG  5