o پiM@sdZddlmZmZmZmZddlZddlmZddlm Z ddl m Z ddl mZdd lmZmZmZmZmZmZdd lmZmZdd lmZzdd lmZWney_dd lmZYnwd+ddZGdddejZ Gdddej!Z"d,ddZ#Gdddejj$Z%ddZ&ddZ'ddZ(Gdd d ej)Z*Gd!d"d"ej)Z+Gd#d$d$ej,Z-Gd%d&d&ej,Z.Gd'd(d(eZ/Gd)d*d*eZ0dS)-aw Normalization + Activation Layers Provides Norm+Act fns for standard PyTorch norm layers such as * BatchNorm * GroupNorm * LayerNorm This allows swapping with alternative layers that are natively both norm + act such as * EvoNorm (evo_norm.py) * FilterResponseNorm (filter_response_norm.py) * InplaceABN (inplace_abn.py) Hacked together by / Copyright 2022 Ross Wightman )UnionListOptionalAnyN)nn) functional)FrozenBatchNorm2d)create_act_layer) is_fast_normfast_group_normfast_layer_norm fast_rms_norm rms_norm2dfast_rms_norm2d)RmsNorm RmsNorm2d)_assert)rms_normFTcCs@|pi}|d|d}|rt|fi|}|durtS|S)Ninplace) setdefaultr rIdentity) act_layer act_kwargsr apply_actactrH/home/ubuntu/.local/lib/python3.10/site-packages/timm/layers/norm_act.py _create_act!s  rc sBeZdZdZdddddejdddddf fdd Zdd ZZS) BatchNormAct2dzBatchNorm + Activation This module performs BatchNorm + Activation in a manner that will remain backwards compatible with weights trained with separate bn, act. This is why we inherit from BN instead of composing it as a .bn member. h㈵>g?TNc  sz| | d} tt|j|f||||d| Wnty.tt|j|||||dYnw| dur6| nt|_t||| |d|_dS)N)devicedtype)epsmomentumaffinetrack_running_statsrrr) superr__init__ TypeErrorrrdroprr)self num_featuresr#r$r%r&rrrr drop_layerr!r"factory_kwargs __class__rrr)1s.      zBatchNormAct2d.__init__c Cst|jdkd|jd|jdurd}n|j}|jr9|jr9|jdur9|jd|jdur6dt|j}n|j} |jr@d}n |jduoI|j du} t ||jrT|jrW|jnd|jr^|jra|j nd|j |j |||j}||}||}|S)Nzexpected 4D input (got zD input)gr g?T)rndimr$trainingr&num_batches_trackedadd_float running_mean running_varF batch_normweightbiasr#r+r)r,xexponential_average_factor bn_trainingrrrforwardVs8      zBatchNormAct2d.forward) __name__ __module__ __qualname____doc__rReLUr)rA __classcell__rrr0rr*s %rcs*eZdZdejdejffdd ZZS)SyncBatchNormActr>returncs8t|}t|dr||}t|dr||}|S)Nr+r)r(rAhasattrr+rr,r>r0rrrAs     zSyncBatchNormAct.forward)rBrCrDtorchTensorrArGrrr0rrHs"rHcCs |}t|tjjjjrpt|tr(t|j|j |j |j |j |d}|j |_ |j|_ntj|j|j |j |j |j |}|j rWt|j|_|j|_Wdn1sRwY|j|_|j|_|j|_|j|_t|drp|j|_|D] \}}||t||qt~|S)N) process_groupqconfig) isinstancerLrmodules batchnorm _BatchNormrrHr-r#r$r%r&rr+ SyncBatchNormno_gradr<r=r8r9r5r4rJrOnamed_children add_moduleconvert_sync_batchnorm)modulerN module_outputnamechildrrrrXsH     rXcseZdZdZddejdddfdedeffdd Zd e d e d e d e d e e de e de e ffdd Z dejdejfddZde fddZZS)FrozenBatchNormAct2da$ BatchNormAct2d where the batch statistics and the affine parameters are fixed Args: num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)`` eps (float): a value added to the denominator for numerical stability. Default: 1e-5 r TNr-r#cst||_|dt||dt||dt||dt||dur3|nt|_ t ||||d|_ dS)Nr<r=r8r9r') r(r)r#register_bufferrLoneszerosrrr+rr)r,r-r#rrrrr.r0rrr)s zFrozenBatchNormAct2d.__init__ state_dictprefixlocal_metadatastrict missing_keysunexpected_keys error_msgsc s2|d}||vr ||=t|||||||dS)Nr5)r(_load_from_state_dict) r,rarbrcrdrerfrgnum_batches_tracked_keyr0rrrhs  z*FrozenBatchNormAct2d._load_from_state_dictr>rIcCs|jdddd}|jdddd}|jdddd}|jdddd}|||j}|||}|||}|||}|S)Nr ) r<reshaper=r9r8r#rsqrtrr+)r,r>wbrvrmscaler=rrrrAs  zFrozenBatchNormAct2d.forwardcCs,|jjd|jjdd|jd|jdS)N(rz, eps=z, act=))r1rBr<shaper#r)r,rrr__repr__s,zFrozenBatchNormAct2d.__repr__)rBrCrDrErrFintr7r)dictstrboolrrhrLrMrArurGrrr0rr]s<  r]cCsJ|}t|ttfrGt|j}|j|_|j|_|jr-|jj |j_|j j |j _|j j|j _|j j|j _|j |_ |j|_|j|_|St|tjjjjtjjjjfrt|j}|j|_|j|_|jrz|jj |j_|j j |j _|j j|j _|j j|j _|j |_ |S|D]\}}t|}||ur|||q|S)a Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively. Args: module (torch.nn.Module): Any PyTorch module. Returns: torch.nn.Module: Resulting module Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762 )rPrrHr]r-r%r<dataclonedetachr=r8r9r#r+rrLrrQrR BatchNorm2drTrrVfreeze_batch_norm_2drWrYresr[r\ new_childrrrr~s>        r~cCs|}t|tr=t|j}|jr#|jj|j_|j j|j _|j j|j _|j j|j _|j |_ |j |_ |j|_|St|trrtj|j}|jr`|jj|j_|j j|j _|j j|j _|j j|j _|j |_ |S|D]\}}t|}||ur|||qv|S)a Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked recursively and submodules are converted in place. Args: module (torch.nn.Module): Any PyTorch module. Returns: torch.nn.Module: Resulting module Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762 )rPr]rr-r%r<rzr{r|r=r8r9r#r+rrrLrr}rVunfreeze_batch_norm_2drWrrrrr.s6        rcCs |r||dks J||S|S)Nrr) num_channels num_groups group_sizerrr _num_groupsWsrc s:eZdZdddddejdddf fdd ZddZZS) GroupNormAct r TNc sVtt|jt||||||d| dur| nt|_t||| |d|_t |_ dS)Nr#r%r') r(rr)rrrr+rrr _fast_norm) r,rrr#r%rrrrrr.r0rrr)`s  zGroupNormAct.__init__cCR|jrt||j|j|j|j}n t||j|j|j|j}||}| |}|SN rr rr<r=r#r: group_normr+rrKrrrrAx   zGroupNormAct.forwardrBrCrDrrFr)rArGrrr0rr^src6eZdZdddejdddffdd ZddZZS) GroupNorm1Actr TNc sNtt|jd|||d|dur|nt|_t||||d|_t|_ dS)Nr rr') r(rr)rrr+rrr r r,rr#r%rrrrr.r0rrr)s  zGroupNorm1Act.__init__cCrrrrKrrrrArzGroupNorm1Act.forwardrrrr0rrsrcsLeZdZdddejdddfdeeeeej fffdd Z ddZ Z S) LayerNormActr TNnormalization_shapec Ltt|j|||d|dur|nt|_t||||d|_t|_ dSN)r#elementwise_affiner') r(rr)rrr+rrr r) r,rr#r%rrrrr.r0rrr)s  zLayerNormAct.__init__cCrr) rr normalized_shaper<r=r#r: layer_normr+rrKrrrrArzLayerNormAct.forward) rBrCrDrrFrrvrrLSizer)rArGrrr0rrsrcr)LayerNormAct2dr TNc rr) r(rr)rrr+rrr rrr0rrr)s  zLayerNormAct2d.__init__cCsr|dddd}|jrt||j|j|j|j}n t||j|j|j|j}|dddd}| |}| |}|S)Nrr ) permuterr rr<r=r#r:rr+rrKrrrrAs  zLayerNormAct2d.forwardrrrr0rrsrcHeZdZdZdddejdddffdd Zdejdejfd d Z Z S) RmsNormAct* RMSNorm + Activation for '2D' NCHW tensors NOTE: It's currently (2025-05-10) faster to use an eager 2d kernel that does reduction on dim=1 than to permute and use internal PyTorch F.rms_norm, this may change if something like https://github.com/pytorch/pytorch/pull/150576 lands. ư>TNc Htj|||d|dur|nt|_t||||d|_t|_dSN)channelsr#r%r' r(r)rrr+rrr rrr0rrr)  zRmsNormAct.__init__r>rIcCH|jrt||j|j|j}n t||j|j|j}||}||}|Sr)rrrr<r#rr+rrKrrrrA   zRmsNormAct.forward rBrCrDrErrFr)rLrMrArGrrr0rr rcr) RmsNormAct2drrTNc rrrrr0rrr)rzRmsNormAct2d.__init__r>rIcCrr)rrrr<r#rr+rrKrrrrArzRmsNormAct2d.forwardrrrr0rrrr)NFTr)1rEtypingrrrrrLrtorch.nnrr:torchvision.ops.miscr create_actr fast_normr r r rrrnormrr trace_utilsrtorch.nn.functionalr ImportErrorrr}rrTrHrXModuler]r~rr GroupNormrr LayerNormrrrrrrrrs:         _ +@,)$!