o پiz"@sdZddlmZddlmZddlmZddlmZGdddej Z Gd d d ej Z ee ej d d Z Gd ddej ZGdddej ZGdddej ZGdddej ZdS)zl MLP module w/ dropout and configurable activation layer Hacked together by / Copyright 2020 Ross Wightman )partial)nn)GlobalResponseNorm) to_2tuplecs:eZdZdZddejddddffdd Zdd ZZS) Mlpz MLP as used in Vision Transformer, MLP-Mixer and related networks NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected. NTFc st|p|}|p |}t|}t|} |rttjddntj} | |||dd|_||_t | d|_ |dur?||nt |_ | |||dd|_ t | d|_dS)Nr kernel_sizerbias)super__init__rrrConv2dLinearfc1actDropoutdrop1Identitynormfc2drop2) self in_featureshidden_features out_features act_layer norm_layerr dropuse_conv drop_probs linear_layer __class__C/home/ubuntu/.local/lib/python3.10/site-packages/timm/layers/mlp.pyrs z Mlp.__init__cC@||}||}||}||}||}||}|SN)rrrrrrrxr%r%r&forward+      z Mlp.forward __name__ __module__ __qualname____doc__rGELUrr+ __classcell__r%r%r#r&r srcsDeZdZdZddejdddddffdd Zdd Zd d ZZ S) GluMlpz MLP w/ GLU style gating See: https://arxiv.org/abs/1612.08083, https://arxiv.org/abs/2002.05202 NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected. NTrFc st|p|}|p |}|ddksJt|}t|} |r&ttjddntj} |r-dnd|_| |_| |||dd|_ ||_ t | d|_ |durS||dnt |_| |d||dd|_t | d|_dS)Nrrr r )r rrrrrr chunk_dim gate_lastrrrrrrrr) rrrrrrr rr r8r!r"r#r%r&r;s zGluMlp.__init__cCs`|jjdurtj|jj|jjjdddtjj|jj|jjjddddddS)Nrr5ư>std)rr rinitones_shapenormal_weightrr%r%r& init_weightsXs &.zGluMlp.init_weightscCsl||}|jd|jd\}}|jr|||n|||}||}||}||}||}|S)Nr5)dim) rchunkr7r8rrrrr)rr*x1x2r%r%r&r+^s "    zGluMlp.forward) r.r/r0r1rSigmoidrrBr+r3r%r%r#r&r45sr4F)rr8cs@eZdZdZddejdddffdd ZddZd d ZZ S) SwiGLUz SwiGLU NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and better matches some other common impl which makes mapping checkpoints simpler. NTrc st|p|}|p |}t|}t|}tj|||dd|_tj|||dd|_||_t|d|_ |dur?||nt |_ tj|||dd|_ t|d|_ dS)Nrr r)r rrrrfc1_gfc1_xrrrrrrr) rrrrrrr rr!r#r%r&rqs zSwiGLU.__init__cCs4|jjdurtj|jjtjj|jjdddS)Nr9r:)rIr rr<r=r?r@rAr%r%r&rBs zSwiGLU.init_weightscCsN||}||}|||}||}||}||}||}|Sr()rIrJrrrrr)rr*x_gater%r%r&r+s      zSwiGLU.forward) r.r/r0r1rSiLUrrBr+r3r%r%r#r&rHlsrHcs:eZdZdZddejddddffdd ZddZZS) GatedMlpz MLP as used in gMLP NTrc st|p|}|p |}t|}t|} tj|||dd|_||_t| d|_|durB|ddks8J|||_ |d}nt |_ |durO||nt |_ tj|||dd|_ t| d|_ dS)Nrr r5r)r rrrrrrrrgaterrrr) rrrrrr gate_layerr rr!r#r%r&rs    zGatedMlp.__init__cCsJ||}||}||}||}||}||}||}|Sr()rrrrNrrrr)r%r%r&r+s       zGatedMlp.forwardr-r%r%r#r&rMsrMcs8eZdZdZddejdddffdd ZddZZS) ConvMlpzG MLP using 1x1 convs that keeps spatial dims (for 2D NCHW tensors) NTrcst|p|}|p |}t|}tj||d|dd|_|r#||nt|_||_t ||_ tj||d|dd|_ dS)Nrr)r r ) r rrrrrrrrrrr)rrrrrrr rr#r%r&rs  zConvMlp.__init__cCs6||}||}||}||}||}|Sr()rrrrrr)r%r%r&r+s     zConvMlp.forward) r.r/r0r1rReLUrr+r3r%r%r#r&rPsrPcs8eZdZdZddejdddffdd Zdd ZZS) GlobalResponseNormMlpz MLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d NOTE: Intended for '2D' NCHW (use_conv=True) or NHWC (use_conv=False, channels-last) tensor layouts NTrFc st|p|}|p |}t|}t|}|rttjddntj} | |||dd|_||_t |d|_ t || d|_ | |||dd|_ t |d|_dS)Nrr rr ) channels_last)r rrrrrrrrrrrgrnrr) rrrrrr rr r!r"r#r%r&rs zGlobalResponseNormMlp.__init__cCr'r()rrrrTrrr)r%r%r&r+r,zGlobalResponseNormMlp.forwardr-r%r%r#r&rRsrRN)r1 functoolsrtorchrrTrhelpersrModulerr4rL SwiGLUPackedrHrMrPrRr%r%r%r&s    (4.,!