o ۷i@@sDddlZddlmZddlZddlZddlmZddlm Z ddl m Z m Z ddl mZddlmZddlmZd d lmZd d lmZGd ddejZGdddejZGdddejZGdddejZGdddeZeGdddeZeGdddeZGdddejZ GdddejZ!Gdd d eee Z"dS)!N) dataclass) weight_norm) ConfigMixinregister_to_config) BaseOutput)apply_forward_hook) randn_tensor) ModelMixin)AutoencoderMixincs*eZdZdZdfdd ZddZZS)Snake1dz; A 1-dimensional Snake activation function module. TcsPtttd|d|_ttd|d|_d|j_d|j_||_ dS)Nr T) super__init__nn Parametertorchzerosalphabeta requires_gradlogscale)self hidden_dimr __class__g/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/models/autoencoders/autoencoder_oobleck.pyr#s  zSnake1d.__init__cCs~|j}|js |jnt|j}|js|jnt|j}||d|dd}||dt|| d}||}|S)Nrr g& .>r ) shaperrrexprreshape reciprocalsinpow)r hidden_statesr rrrrrforward,s$ zSnake1d.forwardT__name__ __module__ __qualname____doc__rr' __classcell__rrrrrs rcs4eZdZdZd dedeffdd Zdd ZZS) OobleckResidualUnitza A residual unit composed of Snake1d and weight-normalized Conv1d layers with dilations. r dimensiondilationcs^td|d}t||_ttj||d||d|_t||_ttj||dd|_ dS)Nr ) kernel_sizer2paddingr )r5) rrrsnake1rrConv1dconv1snake2conv2)rr1r2padrrrr=s    zOobleckResidualUnit.__init__cCsb|}|||}|||}|jd|jdd}|dkr+|d|| f}||}|S)aq Forward pass through the residual unit. Args: hidden_state (`torch.Tensor` of shape `(batch_size, channels, time_steps)`): Input tensor . Returns: output_tensor (`torch.Tensor` of shape `(batch_size, channels, time_steps)`) Input tensor after passing through the residual unit. rr r.)r9r7r;r:r )r hidden_state output_tensorr6rrrr'Fs zOobleckResidualUnit.forward)r0r r*r+r,r-intrr'r.rrrrr/8s r/c0eZdZdZddeffdd ZddZZS) OobleckEncoderBlockz&Encoder block used in Oobleck encoder.r stridec sjtt|dd|_t|dd|_t|dd|_t||_tt j ||d||t |dd|_ dS)Nr r2r r r5rCr6)rrr/ res_unit1 res_unit2 res_unit3rr7rrr8mathceilr9r input_dim output_dimrCrrrr`s    zOobleckEncoderBlock.__init__cCs2||}||}|||}||}|SN)rGrHr7rIr9rr=rrrr'ks   zOobleckEncoderBlock.forwardr r?rrrrrB]s rBcrA) OobleckDecoderBlockz&Decoder block used in Oobleck decoder.r rCc sjtt||_ttj||d||t|dd|_ t |dd|_ t |dd|_ t |dd|_ dS)Nr rFr rDrrE)rrrr7rrConvTranspose1drJrKconv_t1r/rGrHrIrLrrrrws    zOobleckDecoderBlock.__init__cCs6||}||}||}||}||}|SrO)r7rTrGrHrIrPrrrr's     zOobleckDecoderBlock.forwardrQr?rrrrrRtsrRc@sdeZdZddejdefddZddejdBdejfd d Zdd ddejfd d Z dejfddZ dS)#OobleckDiagonalGaussianDistributionF parameters deterministiccCsV||_|jddd\|_|_tj|jd|_|j|j|_t |j|_ ||_ dS)Nr r )dimg-C6?) rVchunkmeanscaler functionalsoftplusstdvarrloglogvarrW)rrVrWrrrrs  z,OobleckDiagonalGaussianDistribution.__init__N generatorreturncCs0t|jj||jj|jjd}|j|j|}|S)N)rbdevicedtype)r rZr rVrdrer^)rrbsamplexrrrrfsz*OobleckDiagonalGaussianDistribution.sampleothercCs|jr tdgS|dur |j|j|j|jddSt|j|jd|j}|j|j}|j|j}|||d}|d}|S)Ngg?r r )rWrTensorrZr_rasumr%)rrhnormalized_diff var_ratio logvar_diffklrrrrns &  z&OobleckDiagonalGaussianDistribution.klcCs|jSrO)rZrrrrmodesz(OobleckDiagonalGaussianDistribution.mode)FrO) r*r+r,rriboolr GeneratorrfrnrprrrrrUs  rUc@seZdZUdZded<dS)AutoencoderOobleckOutputar Output of AutoencoderOobleck encoding method. Args: latent_dist (`OobleckDiagonalGaussianDistribution`): Encoded outputs of `Encoder` represented as the mean and standard deviation of `OobleckDiagonalGaussianDistribution`. `OobleckDiagonalGaussianDistribution` allows for sampling latents from the distribution. rU latent_distN)r*r+r,r-__annotations__rrrrrss  rsc@seZdZUdZejed<dS)OobleckDecoderOutputz Output of decoding method. Args: sample (`torch.Tensor` of shape `(batch_size, audio_channels, sequence_length)`): The decoded output sample from the last layer of the model. rfN)r*r+r,r-rrirurrrrrvs rvc(eZdZdZfddZddZZS)OobleckEncoderzOobleck Encoderc st|}dg|}ttj||ddd|_g|_t|D]\}}|jt||||||d|dg7_qt |j|_||d}t ||_ ttj||ddd|_ dS)Nr r4rr5r6rMrNrCr) rrrrr8r9block enumeraterB ModuleListrr7r;) rencoder_hidden_sizeaudio_channelsdownsampling_ratioschannel_multiplesstrides stride_indexrCd_modelrrrrs"      zOobleckEncoder.__init__cC6||}|jD]}||}q||}||}|SrOr9r{r7r;)rr=modulerrrr'     zOobleckEncoder.forwardr)rrrrrx rxcrw)OobleckDecoderzOobleck Decoderc st|}dg|}ttj|||dddd|_g}t|D] \}} |t||t||||t||d| dg7}q"t ||_ |} t | |_ ttj||dddd|_ dS) Nr rr4rryrzF)r5r6bias)rrrrr8r9r|rRlenr}r{rr7r;) rchannelsinput_channelsrupsampling_ratiosrrr{rrCrNrrrrs"     zOobleckDecoder.__init__cCrrOr)rr=layerrrrr'rzOobleckDecoder.forwardr)rrrrrrrc seZdZdZdZdZedgdgdddddffd d Ze dd e j d e de e eBfddZdde j d e dee j BfddZe dde jd e dee jBfddZ  dde j de d e de jdBdee j Bf ddZZS)AutoencoderOoblecka An autoencoder for encoding waveforms into latents and decoding latent representations into waveforms. First introduced in Stable Audio. This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented for all models (such as downloading or saving). Parameters: encoder_hidden_size (`int`, *optional*, defaults to 128): Intermediate representation dimension for the encoder. downsampling_ratios (`list[int]`, *optional*, defaults to `[2, 4, 4, 8, 8]`): Ratios for downsampling in the encoder. These are used in reverse order for upsampling in the decoder. channel_multiples (`list[int]`, *optional*, defaults to `[1, 2, 4, 8, 16]`): Multiples used to determine the hidden sizes of the hidden layers. decoder_channels (`int`, *optional*, defaults to 128): Intermediate representation dimension for the decoder. decoder_input_channels (`int`, *optional*, defaults to 64): Input dimension for the decoder. Corresponds to the latent dimension. audio_channels (`int`, *optional*, defaults to 2): Number of channels in the audio data. Either 1 for mono or 2 for stereo. sampling_rate (`int`, *optional*, defaults to 44100): The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz). F)r rr)r r rrr0@r iDcstt||_||_||_|ddd|_tt||_ ||_ t ||||d|_ t ||||j|d|_d|_dS)Nr)r~rrr)rrrrrF)rrr~rdecoder_channelsrr@npprod hop_length sampling_raterxencoderrdecoder use_slicing)rr~rrrdecoder_input_channelsrrrrrrBs*  zAutoencoderOobleck.__init__Trg return_dictrccs^jr|jddkrfdd|dD}t|}n|}t|}|s*|fSt|dS)a Encode a batch of images into latents. Args: x (`torch.Tensor`): Input batch of images. return_dict (`bool`, *optional*, defaults to `True`): Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. Returns: The latent representations of the encoded images. If `return_dict` is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. rr csg|]}|qSr)r).0x_slicerorr xsz-AutoencoderOobleck.encode..)rt)rr splitrcatrrUrs)rrgrencoded_slicesh posteriorrrorencodegs   zAutoencoderOobleck.encodezcCs||}|s |fSt|dS)Nrf)rrv)rrrdecrrr_decodes  zAutoencoderOobleck._decodeNcsXjr|jddkrfdd|dD}t|}n|j}|s'|fSt|dS)a Decode a batch of images. Args: z (`torch.Tensor`): Input batch of latent vectors. return_dict (`bool`, *optional*, defaults to `True`): Whether to return a [`~models.vae.OobleckDecoderOutput`] instead of a plain tuple. Returns: [`~models.vae.OobleckDecoderOutput`] or `tuple`: If return_dict is True, a [`~models.vae.OobleckDecoderOutput`] is returned, otherwise a plain `tuple` is returned. rr csg|]}|jqSr)rrf)rz_slicerorrrsz-AutoencoderOobleck.decode..r)rr rrrrrfrv)rrrrbdecoded_slicesdecodedrrordecodes   zAutoencoderOobleck.decoderfsample_posteriorrbc CsJ|}||j}|r|j|d}n|}||j}|s |fSt|dS)ah Args: sample (`torch.Tensor`): Input sample. sample_posterior (`bool`, *optional*, defaults to `False`): Whether to sample from the posterior. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`OobleckDecoderOutput`] instead of a plain tuple. )rbr)rrtrfrprrv) rrfrrrbrgrrrrrrr's   zAutoencoderOobleck.forwardr()TN)FTN)r*r+r,r- _supports_gradient_checkpointing_supports_group_offloadingrrrrrirqrstuplerUrrvr FloatTensorrrrr'r.rrrrr&s\$   r)#rJ dataclassesrnumpyrrtorch.nnrtorch.nn.utilsrconfiguration_utilsrrutilsrutils.accelerate_utilsrutils.torch_utilsr modeling_utilsr vaer Modulerr/rBrRobjectrUrsrvrxrrrrrrs0        %( ((