o }oiC @sdZddlZddlZddlmZddlmmZz ddl m Z dZ Wne y2e ddZ YnwddZd d Zd d Zd"ddZddZddZddZd#ddZdejfddZGdddejZGdddejZGd d!d!ejZdS)$z Brought from: https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py Various utilities for neural networks. N GroupNormTz2Fused optimized group norm has not been installed.FcOV|dkr tj|i|S|dkrtj|i|S|dkr$tj|i|Std|)z4 Create a 1D, 2D, or 3D convolution module. unsupported dimensions: )nnConv1dConv2dConv3d ValueErrordimsargskwargsrv/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/multimodal/modules/imagen/diffusionmodules/layers.pyconv_nd8rcOstj|i|S)z! Create a linear module. )r Linear)rrrrrlinearEsrcOr)z8 Create a 1D, 2D, or 3D average pooling module. rrrr)r AvgPool1d AvgPool2d AvgPool3dr rrrr avg_pool_ndLrrGz?cCs4t||D]\}}||j|d|dqdS)a# Update target parameters to be closer to those of source parameters using an exponential moving average. :param target_params: the target parameter sequence. :param source_params: the source parameter sequence. :param rate: the EMA rate (closer to 1 means slower). r)alphaN)zipdetachmul_add_) target_params source_paramsratetargsrcrrr update_emaYs r'cCs|D]}|q|S)z< Zero out the parameters of a module and return it. ) parametersrzero_)moduleprrr zero_modulefs r,cCs |D] }||q|S)z9 Scale the parameters of a module and return it. )r(rr )r*scaler+rrr scale_moduleos r.cCs|jttdt|jdS)z6 Take the mean over all non-batch dimensions. rdim)meanlistrangelenshape)tensorrrr mean_flatxsr7cCstd||dS)z Make a standard normalization layer. :param channels: number of input channels. :return: an nn.Module for normalization. )actr)channelsr:rrr normalizationsr<i'cCs|d}tt| tjd|tjd|j|jd}|dddf|d}|j|d}tj t |t |gdd}|drXtj |t |dddd fgdd}|S) aY Create sinusoidal timestep embeddings. :param timesteps: a 1-D Tensor of N indices, one per batch element. These may be fractional. :param dim: the dimension of the output. :param max_period: controls the minimum frequency of the embeddings. :return: an [N x dim] Tensor of positional embeddings. rr)startenddtype)deviceN)r?r/r) thexpmathlogarangefloat32tor@floatcatcossin zeros_like) timestepsr0 max_periodr?halffreqsr embeddingrrrtimestep_embeddings ( (rSc*eZdZdZdfdd ZddZZS) UpsampleaB An upsampling layer with an optional convolution. :param channels: channels in the inputs and outputs. :param use_conv: a bool determining if a convolution is applied. :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then upsampling occurs in the inner-two dimensions. rNcsJt||_|p ||_||_||_|r#t||j|jddd|_dSdS)Nrr)padding)super__init__r; out_channelsuse_convrrconvselfr;rZrrY __class__rrrXs  zUpsample.__init__cCst|jd|jks J|jdkr(tj||jd|jdd|jddfdd}ntj|ddd}|jr8||}|S)Nrrrnearest)mode) scale_factorrb)r5r;rF interpolaterZr[r]xrrrforwards 2 zUpsample.forwardrN__name__ __module__ __qualname____doc__rXrh __classcell__rrr^rrUs  rUcrT) UpsampleLearnableaX Upsampling based on ConvTranspose2d. This is needed for bfloat support. :param channels: channels in the inputs and outputs. :param use_conv: a bool determining if a convolution is applied. :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then upsampling occurs in the inner-two dimensions. rNcs|t||_|p ||_||_||_|jdkr&t|j|jddd|_dS|jdkr:tj |j|jdddd|_dSt d ) Nrr`rr)rr`r`rrr)rrr) kernel_sizestriderVz%Upsampling support only for 2D and 3D) rWrXr;rYrZrr ConvTranspose2dr[ConvTranspose3dr r\r^rrrXs     zUpsampleLearnable.__init__cCs"|jd|jks J||}|SNr)r5r;r[rfrrrrhs zUpsampleLearnable.forwardrirjrrr^rrps rpcrT) DownsampleaE A downsampling layer with an optional convolution. :param channels: channels in the inputs and outputs. :param use_conv: a bool determining if a convolution is applied. :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then downsampling occurs in the inner-two dimensions. rNcs|t||_|p ||_||_||_|dkrdnd}|r,t||j|jd|dd|_dS|j|jks4Jt|||d|_dS)Nrrrqr)rsrV)rrrs) rWrXr;rYrZrropr)r]r;rZrrYrsr^rrrXs  zDownsample.__init__cCs|jd|jks J||Srv)r5r;rxrfrrrrhs zDownsample.forwardrirjrrr^rrws  rw)r)r8)rnrDtorchrBtorch.nnr torch.nn.functional functionalrdapex.contrib.group_normrOPT_GROUP_NORM Exceptionprintrrrr'r,r.r7r<rGrSModulerUrprwrrrrs0!