o Gi@sHddlmZddlZddlmZddlmmZddlm Z ddl m Z ddl m Z ddlmZmZmZmZmZdd lmZdd lmZmZmZmZmZmZGd d d ejZGd ddejZdej dej fddZ!GdddejZ"GdddejZ#GdddejZ$GdddejZ%GdddejZ&GdddejZ'dS))partialN) deprecate)get_activation) SpatialNorm) Downsample1D Downsample2DFirDownsample2D KDownsample2D downsample_2d) AdaGroupNorm) FirUpsample2D KUpsample2D Upsample1D Upsample2Dupfirdn2d_native upsample_2dc"seZdZdZddddddddd d dddd dd d ededBdedededededBdedededededBdededededBf fddZde j d e j d!e j fd"d#Z Z S)$ResnetBlockCondNorm2Da) A Resnet block that use normalization layer that incorporate conditioning information. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. groups (`int`, *optional*, default to `32`): The number of groups to use for the first normalization layer. groups_out (`int`, *optional*, default to None): The number of groups to use for the second normalization layer. if set to None, same as `groups`. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. non_linearity (`str`, *optional*, default to `"swish"`): the activation function to use. time_embedding_norm (`str`, *optional*, default to `"ada_group"` ): The normalization layer for time embedding `temb`. Currently only support "ada_group" or "spatial". kernel (`torch.Tensor`, optional, default to None): FIR filter, see [`~models.resnet.FirUpsample2D`] and [`~models.resnet.FirDownsample2D`]. output_scale_factor (`float`, *optional*, default to be `1.0`): the scale factor to use for the output. use_in_shortcut (`bool`, *optional*, default to `True`): If `True`, add a 1x1 nn.conv2d layer for skip-connection. up (`bool`, *optional*, default to `False`): If `True`, add an upsample layer. down (`bool`, *optional*, default to `False`): If `True`, add a downsample layer. conv_shortcut_bias (`bool`, *optional*, default to `True`): If `True`, adds a learnable bias to the `conv_shortcut` output. conv_2d_out_channels (`int`, *optional*, default to `None`): the number of channels in the output. If None, same as `out_channels`. NF ư>swish ada_group?T) out_channels conv_shortcutdropout temb_channelsgroups groups_outeps non_linearitytime_embedding_normoutput_scale_factoruse_in_shortcutupdownconv_shortcut_biasconv_2d_out_channels in_channelsrrrrr r!r"r#r$r%r&r'r(r)r*cst||_|dur|n|}||_||_| |_||_| |_| |_|dur(|}|jdkr7t ||||d|_ n|jdkrCt |||_ nt d|jt j||dddd|_|jdkret ||||d|_n|jdkrqt |||_nt d|jtj ||_|p|}t j||dddd|_t| |_d|_|_|jrt|dd |_n |jrt|ddd d |_| dur|j|kn| |_d|_|jrt j||ddd |d |_dSdS)Nr)r"spatialz" unsupported time_embedding_norm: r kernel_sizestridepaddingFuse_convopr3r1namerr/r0r1bias)super__init__r+ruse_conv_shortcutr'r(r%r$r norm1r ValueErrornnConv2dconv1norm2torchDropoutrconv2r nonlinearityupsample downsamplerr r&r)selfr+rrrrr r!r"r#r$r%r&r'r(r)r* __class__K/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/models/resnet.pyr:IsV        zResnetBlockCondNorm2D.__init__ input_tensortembreturncOst|dks|dddurd}tdd||}|||}||}|jdurB|jddkr7|}|}||}||}n|jdurQ||}||}| |}| ||}||}| |}| |}|j duru| |}|||j}|S)NrscaleThe `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.1.0.0@)lengetrr<rErFshape contiguousrGr@rArrDrr%)rHrMrNargskwargsdeprecation_message hidden_states output_tensorrKrKrLforwards0                zResnetBlockCondNorm2D.forward) __name__ __module__ __qualname____doc__intboolfloatstrr:rBTensorr] __classcell__rKrKrIrLr+sf!     $Krc(seZdZdZddddddddd dd dd dddddd d ededBdedededededBdedededededej dBdededBdededededBf&fd d!Z d"ej d#ej d$ej fd%d&Z Z S)' ResnetBlock2Da9 A Resnet block. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. groups (`int`, *optional*, default to `32`): The number of groups to use for the first normalization layer. groups_out (`int`, *optional*, default to None): The number of groups to use for the second normalization layer. if set to None, same as `groups`. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. non_linearity (`str`, *optional*, default to `"swish"`): the activation function to use. time_embedding_norm (`str`, *optional*, default to `"default"` ): Time scale shift config. By default, apply timestep embedding conditioning with a simple shift mechanism. Choose "scale_shift" for a stronger conditioning with scale and shift. kernel (`torch.Tensor`, optional, default to None): FIR filter, see [`~models.resnet.FirUpsample2D`] and [`~models.resnet.FirDownsample2D`]. output_scale_factor (`float`, *optional*, default to be `1.0`): the scale factor to use for the output. use_in_shortcut (`bool`, *optional*, default to `True`): If `True`, add a 1x1 nn.conv2d layer for skip-connection. up (`bool`, *optional*, default to `False`): If `True`, add an upsample layer. down (`bool`, *optional*, default to `False`): If `True`, add a downsample layer. conv_shortcut_bias (`bool`, *optional*, default to `True`): If `True`, adds a learnable bias to the `conv_shortcut` output. conv_2d_out_channels (`int`, *optional*, default to `None`): the number of channels in the output. If None, same as `out_channels`. NFrrrTrrdefaultr)rrrrr r!pre_normr"r# skip_time_actr$kernelr%r&r'r(r)r*r+rrrrr r!rjr"r#rkr$rlr%r&r'r(r)r*cs>t| dkr td| dkrtdd|_||_|dur!|n|}||_||_||_||_||_ | |_ | |_ |dur>|}t j j||| dd|_t j||dddd |_|dur}|j d kret |||_n|j d krtt |d ||_n td |j dd|_t j j||| dd|_t j ||_|p|}t j||dddd |_t| |_d|_|_|jr| dkrdfdd|_n>| dkrttjddd|_n0t|dd|_n(|jr| dkrdfdd|_n| dkrttj d d d|_n t!|dddd|_|dur|j|kn||_"d|_#|j"rt j||ddd|d|_#dSdS)NrzkThis class cannot be used with `time_embedding_norm==ada_group`, please use `ResnetBlockCondNorm2D` insteadr,ziThis class cannot be used with `time_embedding_norm==spatial`, please use `ResnetBlockCondNorm2D` insteadT num_groups num_channelsr"affiner-rr.ri scale_shiftrzunknown time_embedding_norm :  fir)rr-r-rc t|dSN)rl)rx fir_kernelrKrL$ z(ResnetBlock2D.__init__..sde_vpg@nearest) scale_factormodeFr2crtru)r rvrxrKrLrz,r{)r/r0r4r5rr7)$r9r:r=rjr+rr;r'r(r%r$rkrBr> GroupNormr<r?r@Linear time_emb_projrArCrrDrrErFrGrF interpolater avg_pool2dr r&r)rHr+rrrrr r!rjr"r#rkr$rlr%r&r'r(r)r*rIrxrLr:s|      zResnetBlock2D.__init__rMrNrOc Ost|dks|dddurd}tdd||}||}||}|jdurA|jddkr6|}|}||}||}n|jdurP||}||}| |}|j durq|j sb||}| |ddddddf}|j dkr|dur~||}| |}n.|j dkr|durtd|j tj|d d d \}}| |}|d ||}n| |}||}||}||}|jdur|jr|}||}|||j} | S) NrrPrQrRrSrirqz9 `temb` should not be None when `time_embedding_norm` is rr)dim)rTrUrr<rErFrVrWrGr@rrkr$rAr=rBchunkrrDrtrainingr%) rHrMrNrXrYrZr[ time_scale time_shiftr\rKrKrLr]?sT                       zResnetBlock2D.forward) r^r_r`rarbrcrdrerBrfr:r]rgrKrKrIrLrhsx"     $drhtensorrOcCst|jdkr|dddddfSt|jdkr'|dddddddfSt|jdkr<|dddddddfStdt|d)Nrr-rz`len(tensor)`: z has to be 2, 3 or 4.)rTrVr=)rrKrKrLrearrange_dims}src s^eZdZdZ  ddededeeeefBdedef fd d Zd ej d ej fd dZ Z S) Conv1dBlocka Conv1d --> GroupNorm --> Mish Parameters: inp_channels (`int`): Number of input channels. out_channels (`int`): Number of output channels. kernel_size (`int` or `tuple`): Size of the convolving kernel. n_groups (`int`, default `8`): Number of groups to separate the channels into. activation (`str`, defaults to `mish`): Name of the activation function. mish inp_channelsrr/n_groups activationcs>ttj||||dd|_t|||_t||_dS)Nrr1) r9r:r>Conv1dconv1dr group_normrr)rHrrr/rrrIrKrLr:s zConv1dBlock.__init__inputsrOcCs2||}t|}||}t|}||}|SN)rrrr)rHrintermediate_reproutputrKrKrLr]s   zConv1dBlock.forward)rr r^r_r`rarbtuplerer:rBrfr]rgrKrKrIrLrs rc sdeZdZdZ  ddedededeeeefBdef fd d Zd ej d ej d ej fddZ Z S)ResidualTemporalBlock1Da Residual 1D block with temporal convolutions. Parameters: inp_channels (`int`): Number of input channels. out_channels (`int`): Number of output channels. embed_dim (`int`): Embedding dimension. kernel_size (`int` or `tuple`): Size of the convolving kernel. activation (`str`, defaults `mish`): It is possible to choose the right activation function. rrr embed_dimr/rcshtt||||_t||||_t||_t|||_ ||kr-t ||d|_ dSt |_ dS)Nr) r9r:rconv_inconv_outr time_emb_actr>rtime_embrIdentity residual_conv)rHrrrr/rrIrKrLr:s  z ResidualTemporalBlock1D.__init__rtrOcCs>||}||}||t|}||}|||S)z Args: inputs : [ batch_size x inp_channels x horizon ] t : [ batch_size x embed_dim ] returns: out : [ batch_size x out_channels x horizon ] )rrrrrr)rHrroutrKrKrLr]s  zResidualTemporalBlock1D.forward)rrrrKrKrIrLrs $rc sZeZdZdZ   ddededBdedeffd d Zdd ejd edejfddZ Z S)TemporalConvLayera Temporal convolutional layer that can be used for video (sequence of images) input Code mostly copied from: https://github.com/modelscope/modelscope/blob/1509fdb973e5871f37148a4b5e5964cafd43e64d/modelscope/models/multi_modal/video_synthesis/unet_sd.py#L1016 Parameters: in_dim (`int`): Number of input channels. out_dim (`int`): Number of output channels. dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. Nrrin_dimout_dimrnorm_num_groupsc s t|p|}||_||_tt||ttj||ddd|_ tt||tt |tj||ddd|_ tt||tt |tj||ddd|_ tt||tt |tj||ddd|_ tj|j djtj|j djdS)Nr-rr)rrrr)r9r:rrr> SequentialrSiLUConv3dr@rCrDconv3conv4initzeros_weightr8)rHrrrrrIrKrLr:s:     zTemporalConvLayer.__init__rr[ num_framesrOcCs|dddfd|f|jddddddd}|}||}||}||}||}||}|ddddd|jd|jddf|jdd}|S)Nrrrrr-r)reshaperVpermuter@rDrr)rHr[ridentityrKrKrLr]s4    $zTemporalConvLayer.forward)Nrr)r r^r_r`rarbrdr:rBrfr]rgrKrKrIrLrs $)rc sZeZdZdZ   ddededBdedeffd d Zd ejd ejd ejfddZ Z S)TemporalResnetBlocka A Resnet block. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. Nrrr+rrr"cst||_|dur|n|}||_d}dd|D}tjjd||dd|_tj|||d|d|_ |dur>t |||_ nd|_ tjjd||dd|_ tj d |_tj|||d|d|_td |_|j|k|_d|_|jr|tj||ddd d|_dSdS) NrcSsg|]}|dqS)rrK).0krKrKrL ;sz0TemporalResnetBlock.__init__..rTrmrr.rsilur)r9r:r+rrBr>rr<rr@rrrArCrrDrrEr&r)rHr+rrr"r/r1rIrKrLr:.sL    zTemporalResnetBlock.__init__rMrNrOcCs|}||}||}||}|jdur:||}||ddddddddf}|ddddd}||}||}||}||}||}|jdurX||}||}|S)Nrrrr-r) r<rEr@rrrArrDr)rHrMrNr[r\rKrKrLr]ds"     $      zTemporalResnetBlock.forward)NrrrrKrKrIrLr"s$6rcseZdZdZ       dded edBd ed ed edBd edeffdd Z  ddej dej dBdej dBfddZ Z S)SpatioTemporalResBlocka A SpatioTemporal Resnet block. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the spatial resenet. temporal_eps (`float`, *optional*, defaults to `eps`): The epsilon to use for the temporal resnet. merge_factor (`float`, *optional*, defaults to `0.5`): The merge factor to use for the temporal mixing. merge_strategy (`str`, *optional*, defaults to `learned_with_images`): The merge strategy to use for the temporal mixing. switch_spatial_to_temporal_mix (`bool`, *optional*, defaults to `False`): If `True`, switch the spatial and temporal mixing. Nrr?learned_with_imagesFr+rrr" temporal_eps merge_factorswitch_spatial_to_temporal_mixc sftt||||d|_t|dur|n||dur|n|||dur$|n|d|_t|||d|_dS)N)r+rrr")alphamerge_strategyr)r9r:rhspatial_res_blockrtemporal_res_block AlphaBlender time_mixer) rHr+rrr"rrrrrIrKrLr:s$  zSpatioTemporalResBlock.__init__r[rNimage_only_indicatorc Cs|jd}|||}|j\}}}}||} |dddf| ||||ddddd} |dddf| ||||ddddd}|durM|| |d}|||}|j| ||d}|ddddd||||}|S)Nrrrrr-r) x_spatial x_temporalr)rVrrrrr) rHr[rNrr batch_frameschannelsheightwidth batch_sizehidden_states_mixrKrKrLr]s$  ** zSpatioTemporalResBlock.forward)NrrNrrF)NN) r^r_r`rarbrdrcr:rBrfr]rgrKrKrIrLrsB $rc seZdZdZgdZ  ddededeffdd Zd e j d e d e j fd dZ dde j de j d e j dBd e j fddZ ZS)ra A module to blend spatial and temporal features. Parameters: alpha (`float`): The initial value of the blending factor. merge_strategy (`str`, *optional*, defaults to `learned_with_images`): The merge strategy to use for the temporal mixing. switch_spatial_to_temporal_mix (`bool`, *optional*, defaults to `False`): If `True`, switch the spatial and temporal mixing. )learnedfixedrrFrrrcst||_||_||jvrtd|j|jdkr)|dt|gdS|jdks3|jdkrC| dtj t|gdStd|j)Nzmerge_strategy needs to be in r mix_factorrrzUnknown merge strategy ) r9r:rr strategiesr=register_bufferrBrfregister_parameterr> Parameter)rHrrrrIrKrLr:s    zAlphaBlender.__init__rndimsrOcCs|jdkr |j}|S|jdkrt|j}|S|jdkrg|dur$tdt|tjdd|jdt|jd}|dkrM|dddddddf}|S|d kr_| d ddddf}|Std |d t ) NrrrzMPlease provide image_only_indicator to use learned_with_images merge strategyr)device).Nrr-rzUnexpected ndims z. Dimensions should be 3 or 5) rrrBsigmoidr=whererconesrrNotImplementedError)rHrrrrKrKrL get_alphas,    zAlphaBlender.get_alphaNrrcCs@|||j}||j}|jrd|}||d||}|S)Nr)rndimtodtyper)rHrrrrrwrKrKrLr]s  zAlphaBlender.forward)rFr)r^r_r`rarrdrercr:rBrfrbrr]rgrKrKrIrLrs. #r)( functoolsrrBtorch.nnr>torch.nn.functional functionalrutilsr activationsrattention_processorr downsamplingrr r r r normalizationr upsamplingrrrrrrModulerrhrfrrrrrrrrKrKrKrLs*        B $/G]T