o pi}@s\ddlmZddlmZmZmZddlZddlmZddl mm Z ddl m Z ddlmZddlmZdd lmZmZmZmZmZdd lmZdd lmZmZmZmZmZm Z Gd d d ej!Z"Gdddej!Z#dej$dej$fddZ%Gdddej!Z&Gdddej!Z'Gdddej!Z(Gdddej!Z)Gdddej!Z*Gdddej!Z+dS) )partial)OptionalTupleUnionN) deprecate)get_activation) SpatialNorm) Downsample1D Downsample2DFirDownsample2D KDownsample2D downsample_2d) AdaGroupNorm) FirUpsample2D KUpsample2D Upsample1D Upsample2Dupfirdn2d_native upsample_2dc"seZdZdZddddddddd d dddd dd d edeedededededeedededededeededededeef fddZ de j d e j d!e j fd"d#Z Z S)$ResnetBlockCondNorm2Da) A Resnet block that use normalization layer that incorporate conditioning information. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. groups (`int`, *optional*, default to `32`): The number of groups to use for the first normalization layer. groups_out (`int`, *optional*, default to None): The number of groups to use for the second normalization layer. if set to None, same as `groups`. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. non_linearity (`str`, *optional*, default to `"swish"`): the activation function to use. time_embedding_norm (`str`, *optional*, default to `"ada_group"` ): The normalization layer for time embedding `temb`. Currently only support "ada_group" or "spatial". kernel (`torch.Tensor`, optional, default to None): FIR filter, see [`~models.resnet.FirUpsample2D`] and [`~models.resnet.FirDownsample2D`]. output_scale_factor (`float`, *optional*, default to be `1.0`): the scale factor to use for the output. use_in_shortcut (`bool`, *optional*, default to `True`): If `True`, add a 1x1 nn.conv2d layer for skip-connection. up (`bool`, *optional*, default to `False`): If `True`, add an upsample layer. down (`bool`, *optional*, default to `False`): If `True`, add a downsample layer. conv_shortcut_bias (`bool`, *optional*, default to `True`): If `True`, adds a learnable bias to the `conv_shortcut` output. conv_2d_out_channels (`int`, *optional*, default to `None`): the number of channels in the output. If None, same as `out_channels`. NF ư>swish ada_group?T) out_channels conv_shortcutdropout temb_channelsgroups groups_outeps non_linearitytime_embedding_normoutput_scale_factoruse_in_shortcutupdownconv_shortcut_biasconv_2d_out_channels in_channelsrr r!r"r#r$r%r&r'r(r)r*r+r,r-cst||_|dur|n|}||_||_| |_||_| |_| |_|dur(|}|jdkr7t ||||d|_ n|jdkrCt |||_ nt d|jt j||dddd|_|jdkret ||||d|_n|jdkrqt |||_nt d|jtj ||_|p|}t j||dddd|_t| |_d|_|_|jrt|dd |_n |jrt|ddd d |_| dur|j|kn| |_d|_|jrt j||ddd |d |_dSdS)Nr)r%spatialz" unsupported time_embedding_norm: r kernel_sizestridepaddingFuse_convopr6r4namerr2r3r4bias)super__init__r.ruse_conv_shortcutr*r+r(r'rnorm1r ValueErrornnConv2dconv1norm2torchDropoutr!conv2r nonlinearityupsample downsamplerr r)r )selfr.rr r!r"r#r$r%r&r'r(r)r*r+r,r- __class__U/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/diffusers/models/resnet.pyr=JsV        zResnetBlockCondNorm2D.__init__ input_tensortembreturncOst|dks|dddurd}tdd||}|||}||}|jdurB|jddkr7|}|}||}||}n|jdurQ||}||}| |}| ||}||}| |}| |}|j duru| |}|||j}|S)NrscaleThe `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.1.0.0@)lengetrr?rHrIshape contiguousrJrCrDr!rGr r()rKrPrQargskwargsdeprecation_message hidden_states output_tensorrNrNrOforwards0                zResnetBlockCondNorm2D.forward)__name__ __module__ __qualname____doc__intrboolfloatstrr=rETensorr` __classcell__rNrNrLrOr,sf!     $Krc(seZdZdZddddddddd dd dd dddddd d edeedededededeededededededee j dedeededededeef&fd d!Z d"e j d#e j d$e j fd%d&Z Z S)' ResnetBlock2Da9 A Resnet block. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. groups (`int`, *optional*, default to `32`): The number of groups to use for the first normalization layer. groups_out (`int`, *optional*, default to None): The number of groups to use for the second normalization layer. if set to None, same as `groups`. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. non_linearity (`str`, *optional*, default to `"swish"`): the activation function to use. time_embedding_norm (`str`, *optional*, default to `"default"` ): Time scale shift config. By default, apply timestep embedding conditioning with a simple shift mechanism. Choose "scale_shift" for a stronger conditioning with scale and shift. kernel (`torch.Tensor`, optional, default to None): FIR filter, see [`~models.resnet.FirUpsample2D`] and [`~models.resnet.FirDownsample2D`]. output_scale_factor (`float`, *optional*, default to be `1.0`): the scale factor to use for the output. use_in_shortcut (`bool`, *optional*, default to `True`): If `True`, add a 1x1 nn.conv2d layer for skip-connection. up (`bool`, *optional*, default to `False`): If `True`, add an upsample layer. down (`bool`, *optional*, default to `False`): If `True`, add a downsample layer. conv_shortcut_bias (`bool`, *optional*, default to `True`): If `True`, adds a learnable bias to the `conv_shortcut` output. conv_2d_out_channels (`int`, *optional*, default to `None`): the number of channels in the output. If None, same as `out_channels`. NFrrrTrrdefaultr)rr r!r"r#r$pre_normr%r& skip_time_actr'kernelr(r)r*r+r,r-r.rr r!r"r#r$rmr%r&rnr'ror(r)r*r+r,r-cs>t| dkr td| dkrtdd|_||_|dur!|n|}||_||_||_||_||_ | |_ | |_ |dur>|}t j j||| dd|_t j||dddd |_|dur}|j d kret |||_n|j d krtt |d ||_n td |j dd|_t j j||| dd|_t j ||_|p|}t j||dddd |_t| |_d|_|_|jr| dkrdfdd|_n>| dkrttjddd|_n0t|dd|_n(|jr| dkrdfdd|_n| dkrttj d d d|_n t!|dddd|_|dur|j|kn||_"d|_#|j"rt j||ddd|d|_#dSdS)NrzkThis class cannot be used with `time_embedding_norm==ada_group`, please use `ResnetBlockCondNorm2D` insteadr/ziThis class cannot be used with `time_embedding_norm==spatial`, please use `ResnetBlockCondNorm2D` insteadT num_groups num_channelsr%affiner0rr1rl scale_shiftrzunknown time_embedding_norm :  fir)rr0r0rc t|dSN)ro)rx fir_kernelrNrO% z(ResnetBlock2D.__init__..sde_vpg@nearest) scale_factormodeFr5crwrx)rryr{rNrOr}-r~)r2r3r7r8rr:)$r<r=r@rmr.rr>r*r+r(r'rnrErA GroupNormr?rBrCLinear time_emb_projrDrFr!rGr rHrIrJrF interpolater avg_pool2dr r)r )rKr.rr r!r"r#r$rmr%r&rnr'ror(r)r*r+r,r-rLr{rOr=s|      zResnetBlock2D.__init__rPrQrRc Ost|dks|dddurd}tdd||}||}||}|jdurA|jddkr6|}|}||}||}n|jdurP||}||}| |}|j durq|j sb||}| |ddddddf}|j dkr|dur~||}| |}n.|j dkr|durtd|j tj|d d d \}}| |}|d ||}n| |}||}||}||}|jdur||}|||j} | S) NrrSrTrUrVrlrtz9 `temb` should not be None when `time_embedding_norm` is rr)dim)rWrXrr?rHrIrYrZrJrCrrnr'rDr@rEchunkr!rGr r() rKrPrQr[r\r]r^ time_scale time_shiftr_rNrNrOr`@sP                       zResnetBlock2D.forward)rarbrcrdrerrfrgrhrErir=r`rjrNrNrLrOrksx"     $drktensorrRcCst|jdkr|dddddfSt|jdkr'|dddddddfSt|jdkr<|dddddddfStdt|d)Nrr0rz`len(tensor)`: z has to be 2, 3 or 4.)rWrYr@)rrNrNrOrearrange_dimsysrc sbeZdZdZ  ddededeeeeeffdedef fd d Zd e j d e j fd dZ Z S) Conv1dBlocka Conv1d --> GroupNorm --> Mish Parameters: inp_channels (`int`): Number of input channels. out_channels (`int`): Number of output channels. kernel_size (`int` or `tuple`): Size of the convolving kernel. n_groups (`int`, default `8`): Number of groups to separate the channels into. activation (`str`, defaults to `mish`): Name of the activation function. mish inp_channelsrr2n_groups activationcs>ttj||||dd|_t|||_t||_dS)Nrr4) r<r=rAConv1dconv1dr group_normr r)rKrrr2rrrLrNrOr=s zConv1dBlock.__init__inputsrRcCs2||}t|}||}t|}||}|SN)rrrr)rKrintermediate_reproutputrNrNrOr`s   zConv1dBlock.forward)rr rarbrcrdrerrrhr=rErir`rjrNrNrLrOrs rc sheZdZdZ  ddedededeeeeeffdef fd d Zd e j d e j d e j fddZ Z S)ResidualTemporalBlock1Da Residual 1D block with temporal convolutions. Parameters: inp_channels (`int`): Number of input channels. out_channels (`int`): Number of output channels. embed_dim (`int`): Embedding dimension. kernel_size (`int` or `tuple`): Size of the convolving kernel. activation (`str`, defaults `mish`): It is possible to choose the right activation function. rrr embed_dimr2rcshtt||||_t||||_t||_t|||_ ||kr-t ||d|_ dSt |_ dS)Nr) r<r=rconv_inconv_outr time_emb_actrArtime_embrIdentity residual_conv)rKrrrr2rrLrNrOr=s  z ResidualTemporalBlock1D.__init__rtrRcCs>||}||}||t|}||}|||S)z Args: inputs : [ batch_size x inp_channels x horizon ] t : [ batch_size x embed_dim ] returns: out : [ batch_size x out_channels x horizon ] )rrrrrr)rKrroutrNrNrOr`s  zResidualTemporalBlock1D.forward)rrrrNrNrLrOrs $rc sZeZdZdZ   ddedeededeffd d Zdd ej d edej fddZ Z S)TemporalConvLayera Temporal convolutional layer that can be used for video (sequence of images) input Code mostly copied from: https://github.com/modelscope/modelscope/blob/1509fdb973e5871f37148a4b5e5964cafd43e64d/modelscope/models/multi_modal/video_synthesis/unet_sd.py#L1016 Parameters: in_dim (`int`): Number of input channels. out_dim (`int`): Number of output channels. dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. Nrrin_dimout_dimr!norm_num_groupsc s t|p|}||_||_tt||ttj||ddd|_ tt||tt |tj||ddd|_ tt||tt |tj||ddd|_ tt||tt |tj||ddd|_ tj|j djtj|j djdS)Nr0rr)rrrr)r<r=rrrA SequentialrSiLUConv3drCrFrGconv3conv4initzeros_weightr;)rKrrr!rrLrNrOr=s:     zTemporalConvLayer.__init__rr^ num_framesrRcCs|dddfd|f|jddddddd}|}||}||}||}||}||}|ddddd|jd|jddf|jdd}|S)Nrrrrr0r)reshaperYpermuterCrGrr)rKr^ridentityrNrNrOr` s4    $zTemporalConvLayer.forward)Nrr)r rarbrcrdrerrgr=rErir`rjrNrNrLrOrs $)rc sZeZdZdZ   ddedeededeffd d Zd ej d ej d ej fddZ Z S)TemporalResnetBlocka A Resnet block. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. Nrrr.rr"r%cst||_|dur|n|}||_d}dd|D}tjjd||dd|_tj|||d|d|_ |dur>t |||_ nd|_ tjjd||dd|_ tj d |_tj|||d|d|_td |_|j|k|_d|_|jr|tj||ddd d|_dSdS) NrcSsg|]}|dqS)rrN).0krNrNrO 7sz0TemporalResnetBlock.__init__..rTrprr1rsilur)r<r=r.rrErArr?rrCrrrDrFr!rGr rHr)r )rKr.rr"r%r2r4rLrNrOr=*sL    zTemporalResnetBlock.__init__rPrQrRcCs|}||}||}||}|jdur:||}||ddddddddf}|ddddd}||}||}||}||}||}|jdurX||}||}|S)Nrrrr0r) r?rHrCrrrDr!rGr )rKrPrQr^r_rNrNrOr``s"     $      zTemporalResnetBlock.forward)NrrrrNrNrLrOrs$6rcseZdZdZ       dded eed ed ed eed edeffdd Z  dde j dee j dee j fddZ Z S)SpatioTemporalResBlocka A SpatioTemporal Resnet block. Parameters: in_channels (`int`): The number of channels in the input. out_channels (`int`, *optional*, default to be `None`): The number of output channels for the first conv2d layer. If None, same as `in_channels`. temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the spatial resenet. temporal_eps (`float`, *optional*, defaults to `eps`): The epsilon to use for the temporal resnet. merge_factor (`float`, *optional*, defaults to `0.5`): The merge factor to use for the temporal mixing. merge_strategy (`str`, *optional*, defaults to `learned_with_images`): The merge strategy to use for the temporal mixing. switch_spatial_to_temporal_mix (`bool`, *optional*, defaults to `False`): If `True`, switch the spatial and temporal mixing. Nrr?learned_with_imagesFr.rr"r% temporal_eps merge_factorswitch_spatial_to_temporal_mixc sftt||||d|_t|dur|n||dur|n|||dur$|n|d|_t|||d|_dS)N)r.rr"r%)alphamerge_strategyr)r<r=rkspatial_res_blockrtemporal_res_block AlphaBlender time_mixer) rKr.rr"r%rrrrrLrNrOr=s$  zSpatioTemporalResBlock.__init__r^rQimage_only_indicatorc Cs|jd}|||}|j\}}}}||} |dddf| ||||ddddd} |dddf| ||||ddddd}|durM|| |d}|||}|j| ||d}|ddddd||||}|S)Nrrrrr0r) x_spatial x_temporalr)rYrrrrr) rKr^rQrr batch_frameschannelsheightwidth batch_sizehidden_states_mixrNrNrOr`s$  ** zSpatioTemporalResBlock.forward)NrrNrrF)NN) rarbrcrdrerrgrfr=rErir`rjrNrNrLrOr{sB $rc seZdZdZgdZ  ddededeffdd Zd e j d e d e j fd dZ dde j de j d e e j d e j fddZZS)ra A module to blend spatial and temporal features. Parameters: alpha (`float`): The initial value of the blending factor. merge_strategy (`str`, *optional*, defaults to `learned_with_images`): The merge strategy to use for the temporal mixing. switch_spatial_to_temporal_mix (`bool`, *optional*, defaults to `False`): If `True`, switch the spatial and temporal mixing. )learnedfixedrrFrrrcst||_||_||jvrtd|j|jdkr)|dt|gdS|jdks3|jdkrC| dtj t|gdStd|j)Nzmerge_strategy needs to be in r mix_factorrrzUnknown merge strategy ) r<r=rr strategiesr@register_bufferrEriregister_parameterrA Parameter)rKrrrrLrNrOr=s    zAlphaBlender.__init__rndimsrRcCs|jdkr |j}|S|jdkrt|j}|S|jdkrg|dur$tdt|tjdd|jdt|jd}|dkrM|dddddddf}|S|d kr_| d ddddf}|Std |d t ) NrrrzMPlease provide image_only_indicator to use learned_with_images merge strategyr)device).Nrr0rzUnexpected ndims z. Dimensions should be 3 or 5) rrrEsigmoidr@whererfonesrrNotImplementedError)rKrrrrNrNrO get_alphas,    zAlphaBlender.get_alphaNrrcCs@|||j}||j}|jrd|}||d||}|S)Nr)rndimtodtyper)rKrrrrrzrNrNrOr`s  zAlphaBlender.forward)rFr)rarbrcrdrrgrhrfr=rErirerrr`rjrNrNrLrOrs. #r), functoolsrtypingrrrrEtorch.nnrAtorch.nn.functional functionalrutilsr activationsr attention_processorr downsamplingr r r rr normalizationr upsamplingrrrrrrModulerrkrirrrrrrrrNrNrNrOs,        = $/G]T