o پiv @sdZddlmZddlmZmZmZmZddlZddl m Z ddl m m Z ddlmZddlmZmZddlmZmZmZmZmZmZmZddlmZmZd d lmZd d lmZd d l m!Z!d d l"m#Z#m$Z$d dl%m&Z&m'Z'dgZ(Gddde j)Z*Gddde j)Z+Gddde j)Z,Gddde j)Z-Gddde j)Z.Gddde j)Z/dedee0e0ffddZ1e!d edee0e0fd!e0d"e0fd#d$Z2Gd%d&d&e j)Z3Gd'd(d(e j)Z4Gd)d*d*e j)Z5Gd+dde j)Z6dOd-d.Z7d/d0Z8dPd2d3Z9dQd5d6Z:e&e:d7d8e:d7d8e:d7d8e:e:e:e:d9dd:d;e:de6fd?d@Ze6fdAdBZ=e'dPd>e6fdCdDZ>e'dPd>e6fdEdFZ?e'dPd>e6fdGdHZ@e'dPd>e6fdIdJZAe'dPd>e6fdKdLZBe'dPd>e6fdMdNZCdS)Raf DaViT: Dual Attention Vision Transformers As described in https://arxiv.org/abs/2204.03645 Input size invariant transformer architecture that combines channel and spacial attention in each block. The attention mechanisms used are linear in complexity. DaViT model defs and weights adapted from https://github.com/dingmyu/davit, original copyright below )partial)ListOptionalTupleUnionN)TensorIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath to_2tuple trunc_normal_Mlp LayerNorm2dget_norm_layeruse_fused_attn)NormMlpClassifierHeadClassifierHead)build_model_with_cfg)feature_take_indices)register_notrace_function) checkpointcheckpoint_seq)generate_default_cfgsregister_modelDaVitcs:eZdZd dededeffdd Zdefd d ZZS) ConvPosEncFdimkactcsJtt|tj|||d|d|d|_|rt|_dSt|_dS)Nr) kernel_sizestridepaddinggroups) superr__init__nnConv2dprojGELUIdentityr!)selfrr r! __class__E/home/ubuntu/.local/lib/python3.10/site-packages/timm/models/davit.pyr(#s zConvPosEnc.__init__xcCs||}|||}|SN)r+r!)r.r3featr1r1r2forward0s zConvPosEnc.forward)rF) __name__ __module__ __qualname__intboolr(rr6 __classcell__r1r1r/r2r"s rcs8eZdZdZdddeffdd Zdefdd ZZS) Stemz Size-agnostic implementation of 2D image to patch embedding, allowing input size to be adjusted during model forward operation r`csXtt|}||_||_||_|ddksJtj||d|dd|_|||_ dS)Nrr?rr#r$r%) r'r(r r$in_chsout_chsr)r*convnorm)r.rBrCr$ norm_layerr/r1r2r(;s z Stem.__init__r3cCs~|j\}}}}|jd||jd|jd}|jd||jd|jd}t|d|d|f}||}||}|S)Nrr)shaper$FpadrDrE)r.r3BCHWpad_rpad_br1r1r2r6Qs""  z Stem.forward) r7r8r9__doc__rr(rr6r<r1r1r/r2r=6sr=cs0eZdZdeffdd ZdefddZZS) DownsamplercsVt||_||_|||_|ddk|_tj|||d|jr"dn|dd|_dS)Nr"rrA) r'r(rBrCrEeven_kr)r*rD)r.rBrCr#rFr/r1r2r(\s   zDownsample.__init__r3c Csl|j\}}}}||}|jr/|jj\}}||||}||||} t|d|d| f}||}|S)Nr)rGrErRrDr#rHrI) r.r3rJrKrLrMk_hk_wrNrOr1r1r2r6qs   zDownsample.forward)r7r8r9rr(rr6r<r1r1r/r2rQ[s rQcs&eZdZdfdd ZddZZS)ChannelAttentionV2TcsHt||_|||_||_tj||d|d|_t|||_dS)Nrbias) r'r(r&head_dim dynamic_scaler)Linearqkvr+)r.r num_headsqkv_biasrZr/r1r2r(s  zChannelAttentionV2.__init__c Cs|j\}}}||||d|j||jddddd}|d\}}}|jr/||d}n||jd}|dd|} | j dd } | |dddd}|dd|||}| |}|S) Nrr"rrr?r) rGr\reshaper&permuteunbindrZrY transposesoftmaxr+ r.r3rJNrKr\qr vattnr1r1r2r6s .  zChannelAttentionV2.forward)rVTT)r7r8r9r(r6r<r1r1r/r2rU}s rUcs,eZdZdfdd ZdefddZZS) ChannelAttentionrVFcsJt||_||}|d|_tj||d|d|_t|||_dS)Nr_rrW)r'r(r]scaler)r[r\r+)r.rr]r^rYr/r1r2r(s  zChannelAttention.__init__r3c Cs|j\}}}||||d|j||jddddd}|d\}}}||j}|dd|} | jdd} | |dddd}|dd|||}| |}|S) Nrr"rrr?r`rarb) rGr\rcr]rdrernrfrgr+rhr1r1r2r6s .   zChannelAttention.forward)rVF)r7r8r9r(rr6r<r1r1r/r2rms rmcs@eZdZdddejejdddffdd Zdefdd ZZ S) ChannelBlock@FTc stt|d| d|_||_|||_| rtnt} | |||d|_|dkr+t |nt |_ t|d| d|_ |jr^|||_t|t|||d|_|dkrWt ||_dSt |_dSd|_d|_d|_dSNr)rr r!)r]r^rq) in_featureshidden_features act_layer)r'r(rcpe1ffnnorm1rUrmrlr r)r- drop_path1cpe2norm2rr:mlp drop_path2) r.rr] mlp_ratior^ drop_pathrurFrwcpe_actv2 attn_layerr/r1r2r(s.     $ zChannelBlock.__init__r3cCs|j\}}}}||ddd}||}||}|||}||dd||||}|j durZ|ddd}|| | | |}|dd||||}|S)Nr"r) rGrvflattenrfrxrlryrzviewr|r}r{)r.r3rJrKrLrMcurr1r1r2r6s   zChannelBlock.forward) r7r8r9r)r, LayerNormr(rr6r<r1r1r/r2ros(ror3 window_sizecCsj|j\}}}}||||d|d||d|d|}|ddddddd|d|d|}|S)z Args: x: (B, H, W, C) window_size (int): window size Returns: windows: (num_windows*B, window_size, window_size, C) rrrr"r?r`rGrrd contiguous)r3rrJrLrMrKwindowsr1r1r2window_partitions,,rrrLrMcCs^|jd}|d||d||d|d|d|}|ddddddd|||}|S)z Args: windows: (num_windows*B, window_size, window_size, C) window_size (int): Window size H (int): Height of image W (int): Width of image Returns: x: (B, H, W, C) r`rrrr"r?rr)rrrLrMrKr3r1r1r2window_reverses ,$rcsBeZdZUdZejjeed<d fdd Z de fddZ Z S) WindowAttentiona Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window. Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True fused_attnTcslt||_||_||_||}|d|_t|_tj ||d|d|_ t |||_ tj dd|_ dS)Nr_rrWr`rb)r'r(rrr]rnrrr)r[r\r+Softmaxrg)r.rrr]r^rYr/r1r2r(s  zWindowAttention.__init__r3c Cs|j\}}}||||d|j||jddddd}|d\}}}|jr0t|||}n||j }|| dd} | | } | |}| dd|||}| |}|S)Nrr"rrr?rar`) rGr\rcr]rdrerrHscaled_dot_product_attentionrnrfrgr+) r.r3B_rirKr\rjr rkrlr1r1r2r6+s .   zWindowAttention.forwardT) r7r8r9rPtorchjitFinalr;__annotations__r(rr6r<r1r1r/r2rs rcsDeZdZdZddddejejddffdd Zd efd d Z Z S) SpatialBlocka< Windows Block. Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): Window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True drop_path (float, optional): Stochastic depth rate. Default: 0.0 act_layer (nn.Module, optional): Activation layer. Default: nn.GELU norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm r@rpTrqFc st||_| |_||_t||_||_t|d| d|_ |||_ t ||j||d|_ |dkr5t |nt|_t|d| d|_|jrj|||_t||} t|| |d|_|dkrct ||_dSt|_dSd|_d|_d|_dSrr)r'r(rrwr]r rr~rrvrxrrlr r)r-ryrzr{r:rr|r}) r.rr]rr~r^rrurFrwrmlp_hidden_dimr/r1r2r(Ks8     $ zSpatialBlock.__init__r3c Cs|j\}}}}||ddd}||}|||||}d}}|jd||jd|jd} |jd||jd|jd} t|dd|| || f}|j\} } } } t ||j}|d|jd|jd|}| |}|d|jd|jd|}t ||j| | }|ddd|d|ddf }|||||}|| |}||dd||||}|jdur|ddd}|||||}|dd||||}|S)Nr"rrr`)rGrvrrfrxrrrHrIrrlrrryrzr|r}r{)r.r3rJrKrLrMshortcutpad_lpad_trNrO_HpWp x_windows attn_windowsr1r1r2r6xs. ""  $ zSpatialBlock.forward) r7r8r9rPr)r,rr(rr6r<r1r1r/r2r>s-rcs^eZdZddddddddeejddd ddffd d Zejj dd d Z de fddZ Z S) DaVitStagerTspatialchannelrr@rp)rrFr"cstd|_|rt|||| d|_nt|_ g}t|D]^}ddlm }g}t |D]5\}}|dkrK| dt |||| | || | ||d fq.|dkrc| d t |||| | || | ||d fq.|rq| t||q | tjd d |Dq tj||_dS) NF)r#rFr) OrderedDictr spatial_block) rr]r~r^rrFrwrrr channel_block) rr]r~r^rrFrwrrcSsg|]}|dqS)rr1).0br1r1r2 z'DaVitStage.__init__..)r'r(grad_checkpointingrQ downsampler)r-range collectionsr enumerateappendrro Sequentialblocks)r.rBrCdepthr attn_typesr]rr~r^drop_path_ratesrF norm_layer_clrwrdown_kernel_size named_blockschannel_attn_v2 stage_blocks block_idxrdual_attention_blockattn_idx attn_typer/r1r2r(sP        zDaVitStage.__init__cCs ||_dSr4)r)r.enabler1r1r2set_grad_checkpointings z!DaVitStage.set_grad_checkpointingr3cCs8||}|jrtjst|j|}|S||}|Sr4)rrrr is_scriptingrrr.r3r1r1r2r6s   zDaVitStage.forwardr)r7r8r9rr)rr(rrignorerrr6r<r1r1r/r2rs(H rcsTeZdZdZ              d7fdd ZddZejjd8ddZ ejjd9ddZ ejjde j fddZ d:dedeefd d!Z  " d;d#ejd$eeeeefd%ed&ed'ed(edeeejeejeejfffd)d*Z + d