o ۷i@s,ddlZddlZddlmZddlmZmZddlmZddl m Z m Z ddl m Z mZddlmZdd lmZdd lmZdd lmZdd lmZd dlmZGdddejZGdddejZGdddejZGdddejZGdddejZ GdddejZ!GdddeeeZ"ddZ#dS)N) ConfigMixinregister_to_config)apply_forward_hook) Attention SpatialNorm) DecoderOutputDiagonalGaussianDistribution) Downsample2D)AutoencoderKLOutput) ModelMixin) ResnetBlock2D) Upsample2D)AutoencoderMixincseZdZdZ      ddededBd ed ed ed ed eddffdd Zede j de j fddZ de j dede j fddZ Z S)AllegroTemporalConvLayera  Temporal convolutional layer that can be used for video (sequence of images) input. Code adapted from: https://github.com/modelscope/modelscope/blob/1509fdb973e5871f37148a4b5e5964cafd43e64d/modelscope/models/multi_modal/video_synthesis/unet_sd.py#L1016 N Frin_dimout_dimdropoutnorm_num_groups up_sample down_samplestridereturnc st|p|}t|dd}} d} ||_||_|r:tt||ttj ||d||fdd|| fd|_ n;|rZtt||ttj ||dd||fd|| fd|_ ntt||ttj ||d||f| || fd|_ tt||tt |tj ||d||f| || fd|_ tt||tt |tj ||d||f| ||fd|_ tt||ttj ||d||f| ||fd|_dS) Nrg?rr)rrr)rpaddingrr)super__init__intrrnn Sequential GroupNormSiLUConv3dconv1Dropoutconv2conv3conv4) selfrrrrrrrpad_hpad_wpad_t __class__j/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/models/autoencoders/autoencoder_kl_allegro.pyr 'sP         z!AllegroTemporalConvLayer.__init__ hidden_statescCsTtj|ddddddf|fdd}tj||ddddddffdd}|S)Nrrr)dim)torchcat)r4r2r2r3_pad_temporal_dim^s((z*AllegroTemporalConvLayer._pad_temporal_dim batch_sizecCs |d|dfddddd}|jr!|dddddddf}n|jr2|jdd|jddd}n|}|js:|jr@||}n ||}||}|jr_|dddddddd dd}||}| |}||}| |}||}| |}||}|ddddddd}|S) Nrr6rrr)r5 output_size)rr6) unflattenpermuterrrepeat_interleaveshaper'r9flattenr)r*r+)r,r4r:identityr2r2r3forwardds*    $      z AllegroTemporalConvLayer.forward)NrrFFr)__name__ __module__ __qualname____doc__r!floatboolr staticmethodr7Tensorr9rD __classcell__r2r2r0r3r!s8 7"rcseZdZ          dd ed ed ed edededededededededeffdd Zdej dej fddZ Z S)AllegroDownBlock3Drrư>defaultswishrT?F in_channels out_channelsr num_layers resnet_epsresnet_time_scale_shift resnet_act_fn resnet_groupsresnet_pre_normoutput_scale_factorspatial_downsampletemporal_downsampledownsample_paddingcstg}g}t|D]&}|dkr|n|}|t||d|||||| | d |t||d|dq t||_t||_ | rMt||d|ddd|_ | |_ | rbtt |d|| dd g|_ dSd|_ dS) Nr rSrT temb_channelsepsgroupsrtime_embedding_norm non_linearityr[pre_norm皙?rrTr)rrrrop)use_convrTrname)rr rangeappendrrr" ModuleListresnets temp_convstemp_convs_downadd_temp_downsampler downsamplers)r,rSrTrrUrVrWrXrYrZr[r\r]r^rnroir0r2r3r sT      zAllegroDownBlock3D.__init__r4rcC|jd}|ddddddd}t|j|jD]\}}||dd}|||d}q|jr4|j||d}|jdurC|jD]}||}q<| d|dfddddd}|S Nrrrrr;)temb)r:r6) rAr?rBziprnrorqrprrr>)r,r4r:resnet temp_conv downsamplerr2r2r3rD     zAllegroDownBlock3D.forward) rrrOrPrQrTrRTFr rErFrGr!rIstrrJr r7rLrDrMr2r2r0r3rNsP     BrNcseZdZ          dd ed ed ededededededededededed Bffdd Zdej dej fddZ Z S)AllegroUpBlock3DrrrOrPrQrTrRFNrSrTrrUrVrWrXrYrZr[spatial_upsampletemporal_upsampler`cstg}g}t|D]&}|dkr|n|}|t||| |||||| | d |t||d|dq t||_t||_ | |_ | rPt||d|ddd|_ | r`tt |d|dg|_ dSd|_ dS) Nrr_rfrgTr)rrrr)rirT)rr rkrlrrr"rmrnroadd_temp_upsample temp_conv_upr upsamplers)r,rSrTrrUrVrWrXrYrZr[rrr`rnrorsinput_channelsr0r2r3r sJ     zAllegroUpBlock3D.__init__r4rcCrtru) rAr?rBrwrnrorrrr>)r,r4r:rxry upsamplerr2r2r3rDr{zAllegroUpBlock3D.forward) rrrOrPrQrTrRTFNr|r2r2r0r3r~sP     =r~cszeZdZ          dd ed ed ed ed edededededededeffdd Zdej dej fddZ Z S)AllegroMidBlock3DConvrrrOrPrQrTrRrSr`rrUrVrWrXrYrZ add_attentionattention_head_dimr[c stt||||||||| | d g} t||d|dg}g}| dur%|} t|D]F}| rN|t||| | | ||dkr=|nd|dkrD|ndddddd n|d| t||||||||| | d |t||d|dq)t| |_ t||_ t||_ dS)Nr_rfrgrPspatialT) headsdim_headrescale_output_factorrarspatial_norm_dimresidual_connectionbiasupcast_softmax_from_deprecated_attn_block) rr rrrkrlrr"rmrnro attentions)r,rSr`rrUrVrWrXrYrZrrr[rnror_r0r2r3r 0s    zAllegroMidBlock3DConv.__init__r4rcCs|jd}|ddddddd}|jd|dd}|jd||d}t|j|jdd|jddD]\}}}||}||dd}|||d}q5|d|dfddddd}|Sru)rAr?rBrnrorwrr>)r,r4r:attnrxryr2r2r3rDs , zAllegroMidBlock3DConv.forward) rrrOrPrQrTTrrRr|r2r2r0r3r/sJ     ]rcseZdZddddgdddddf d ed ed eed fd eed fdeed fdedededeffdd Zdej dej fddZ Z S)AllegroEncoder3DrrNrNrNrNrTTFFrrsiluTrSrTdown_block_types.block_out_channelstemporal_downsample_blockslayers_per_blockract_fndouble_zc  sPttj||ddddd|_tj|d|dddd|_tg|_|d} t |D]1\} } | } || } | t |dk}| dkrUt || | | || d d||d }nt d |j |q.t|d d |dd |d |dd|_tj|d |d d|_t|_| rd|n|}tj|d |d ddd|_tj|d |ddd|_d|_dS)Nrrr kernel_sizerrrrrrrr)rSrTrrrNrO) rUrSrTr\r]rVr^rXrYzCInvalid `down_block_type` encountered. Must be `AllegroDownBlock3D`r6rPrSrVrXr[rWrrYr` num_channels num_groupsrarrF)rr r"Conv2dconv_inr& temp_conv_inrm down_blocks enumeratelenrN ValueErrorrlr mid_blockr$ conv_norm_outr%conv_act temp_conv_outconv_outgradient_checkpointing)r,rSrTrrrrrrroutput_channelrsdown_block_type input_channelis_final_block down_blockconv_out_channelsr0r2r3r sd     zAllegroEncoder3D.__init__samplercCs^|jd}|ddddddd}||}|d|dfddddd}|}||}||}trL|jrL|j D]}| ||}q;| |j |}n|j D]}||}qO| |}|ddddddd}| |}| |}|d|dfddddd}|}||}||}|ddddddd}||}|d|dfddddd}|SNrrrrr;r6)rAr?rBrr>rr7is_grad_enabledrr_gradient_checkpointing_funcrrrrr)r,rr:residualrr2r2r3rDs2           zAllegroEncoder3D.forward rErFrGr!tupler}rJr r7rLrDrMr2r2r0r3rs<     SrcseZdZdddgdddddd f d ed ed eed fdeed fdeed fdedededeffdd Zdej dej fddZ Z S)AllegroDecoder3Dr;rr~r~r~r~FTTFrrrrgrouprSrTup_block_types.temporal_upsample_blocksrrrr norm_typec  sttj||ddddd|_tj|d|dddd|_d|_tg|_ | dkr/|nd} t |dd |d| d kr>d n| |d|| d |_t t |} | d } t |D]6\} }| }| | } | t|dk}|dkrt|d|| | || d ||| | d }ntd|j || }qV| dkrt|d | |_n tj|d |d d|_t|_tj|d |d ddd|_tj|d |ddd|_d|_dS)Nr6rrrrrrrrOrrPrrr~) rUrSrTrrrVrXrYr`rWz?Invalid `UP_block_type` encountered. Must be `AllegroUpBlock3D`rF)rr r"rrr&rrrm up_blocksrlistreversedrrr~rrlrrr$r%rrrr)r,rSrTrrrrrrrr`reversed_block_out_channelsrrs up_block_typeprev_output_channelrup_blockr0r2r3r sf      zAllegroDecoder3D.__init__rrcCs||jd}|ddddddd}||}|d|dfddddd}|}||}||}tt|j j }t rV|j rV||j|}|jD]}|||}qLn||}||}|jD]}||}qc|ddddddd}||}||}|d|dfddddd}|}||}||}|ddddddd}||}|d|dfddddd}|Sr)rAr?rBrr>rnextiterr parametersdtyper7rrrrtorrrr)r,rr:r upscale_dtyperr2r2r3rDus8            zAllegroDecoder3D.forwardrr2r2r0r3rs<     Wrc"seZdZdZdZe         d8dededeedfdeedfdeedfdee dfdee dfdedededede dede de dd f fd!d" Z d#e j de j fd$d%Ze d9d#e j d&e deeeBfd'd(Zd)e j de j fd*d+Zed9d)e j d&e dee j Bfd,d-Zd#e j de j fd.d/Zd)e j de j fd0d1Z 2  d:d3e j d4e d&e d5e jd Bdee j Bf d6d7ZZS);AutoencoderKLAllegroa! A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos. Used in [Allegro](https://github.com/rhymes-ai/Allegro). This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented for all models (such as downloading or saving). Parameters: in_channels (int, defaults to `3`): Number of channels in the input image. out_channels (int, defaults to `3`): Number of channels in the output. down_block_types (`tuple[str, ...]`, defaults to `("AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D")`): tuple of strings denoting which types of down blocks to use. up_block_types (`tuple[str, ...]`, defaults to `("AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D")`): tuple of strings denoting which types of up blocks to use. block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): tuple of integers denoting number of output channels in each block. temporal_downsample_blocks (`tuple[bool, ...]`, defaults to `(True, True, False, False)`): tuple of booleans denoting which blocks to enable temporal downsampling in. latent_channels (`int`, defaults to `4`): Number of channels in latents. layers_per_block (`int`, defaults to `2`): Number of resnet or attention or temporal convolution layers per down/up block. act_fn (`str`, defaults to `"silu"`): The activation function to use. norm_num_groups (`int`, defaults to `32`): Number of groups to use in normalization layers. temporal_compression_ratio (`int`, defaults to `4`): Ratio by which temporal dimension of samples are compressed. sample_size (`int`, defaults to `320`): Default latent size. scaling_factor (`float`, defaults to `0.13235`): The component-wise standard deviation of the trained latent space computed using the first batch of the training set. This is used to scale the latent space to have unit variance when training the diffusion model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image Synthesis with Latent Diffusion Models](https://huggingface.co/papers/2112.10752) paper. force_upcast (`bool`, default to `True`): If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE can be fine-tuned / trained to a lower range without losing too much precision in which case `force_upcast` can be set to `False` - see: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix Trrrrrrr;rrr@p= ף?rSrTr.rrrrlatent_channelsrrrtemporal_compression_ratio sample_sizescaling_factor force_upcastrNc stt|||||| | | dd |_t|||||| | | d|_td|d|d|_t||d|_ d|_ d|_ dt |d|_ d|_d|_d |_d }|| | f|_||j| |j| |jf|_dS) NT) rSrTrrrrrrr)rSrTrrrrrrrrFxP)rr rencoderrdecoderr"r quant_convpost_quant_conv use_slicing use_tilingrspatial_compression_ratiotile_overlap_ttile_overlap_htile_overlap_wkernelr)r,rSrTrrrrrrrrrrrrr sample_framesr0r2r3r sH     zAutoencoderKLAllegro.__init__xcC|jr||Std)Nz5Encoding without tiling has not been implemented yet.)r tiled_encodeNotImplementedError)r,rr2r2r3_encode zAutoencoderKLAllegro._encode return_dictcs^jr|jddkrfdd|dD}t|}n|}t|}|s*|fSt|dS)a Encode a batch of videos into latents. Args: x (`torch.Tensor`): Input batch of videos. return_dict (`bool`, defaults to `True`): Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. Returns: The latent representations of the encoded videos. If `return_dict` is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. rrcg|]}|qSr2)r).0x_slicer,r2r3 8z/AutoencoderKLAllegro.encode..) latent_dist)rrAsplitr7r8rr r )r,rrencoded_slicesh posteriorr2rr3encode&s   zAutoencoderKLAllegro.encodezcCr)Nz5Decoding without tiling has not been implemented yet.)r tiled_decoder)r,rr2r2r3_decodeCrzAutoencoderKLAllegro._decodecsVjr|jddkrfdd|dD}t|}n|}|s&|fSt|dS)a Decode a batch of videos. Args: z (`torch.Tensor`): Input batch of latent vectors. return_dict (`bool`, defaults to `True`): Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. Returns: [`~models.vae.DecoderOutput`] or `tuple`: If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is returned. rrcrr2)r)rz_slicerr2r3r\rz/AutoencoderKLAllegro.decode..r)rrArr7r8rr )r,rrdecoded_slicesdecodedr2rr3decodeKs   zAutoencoderKLAllegro.decodec Csd}|j}|jj}|j\}}}}} t||jd|jdd} t||jd|jdd} t| |jd|jdd} d} || | | d|jj |jd||jd||jd|f}||||jd|jd|jdf}t | D]}t | D]}t | D]}||jd||jd|jd}}||jd||jd|jd}}||jd||jd|jd}}|dddd||||||f}||| |<| ||dks| | | | dkr?| |}| | | | dkr | ||dkr |d| |d|| | |d<n ||| |d| d<||||jd|jd|jdf}| d7} qqq~||d|jj ||||| |f}|jd||jd||jd|f}|jd||jd||jd|f}|d|d|d|d|d|df}t | D]}||d||d|d}}t | D]i}||d||d|d}}t | D]P}||d||d|d}}t || |df|| |df|| |df||| | || | d}|dddd||||||f|7<qːqq|ddddddd}||}|d|dfddddd}|SNrrrrr;r6)rconfigrrAmathfloorrr new_zerosrrkr_prepare_for_blend unsqueezer?rBrr>)r,rlocal_batch_sizersrtr:r num_framesheightwidthoutput_num_frames output_height output_widthcount output_latentvae_batch_inputrsjkn_startn_endh_starth_endw_startw_end video_cubelatent output_kernel output_strideoutput_overlap latent_meanr2r2r3res"""     &   (((&  &  (( " " "    2 z!AutoencoderKLAllegro.tiled_encodec Cs~d}|j}|jj}|jd||jd||jd|f}|jd||jd||jd|f}|j\}}} } } |ddddddd}||}| d|dfddddd}t | |d|dd} t | |d|dd} t | |d|dd}d}| | | ||jj |jd|jd|jdf}| |||d|d|df}t| D]}t| D]}t|D]}||d||d|d}}||d||d|d}}||d||d|d}}|dddd||||||f}||||<|||dks$|| | |dkrl||}|| | |dkrP|||dkrP|d||d||||d<n ||||d|d<| |||d|d|df}|d7}qqq| ||jj | || || |f}|jd|jd|jd|jd|jd|jdf}t| D]}||jd||jd|jd}}t| D]o}||jd||jd|jd}}t|D]S}||jd||jd|jd}}t|| |df|| |df|||df||| ||||d}|dddd||||||f|7<qڐqq|ddddd}|Sr )rr rrrrAr?rBrr>rrrrTrkrrr contiguous) r,rrrr latent_kernel latent_strider:rrrrrrrrdecoded_videosrrsrr r!r"r#r$r%r&current_latent current_videovideo video_overlapout_video_blendr2r2r3rs((      """&   $ ( ( (    2 z!AutoencoderKLAllegro.tiled_decodeFrsample_posterior generatorc CsJ|}||j}|r|j|d}n|}||j}|s |fSt|dS)a Args: sample (`torch.Tensor`): Input sample. sample_posterior (`bool`, *optional*, defaults to `False`): Whether to sample from the posterior. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`DecoderOutput`] instead of a plain tuple. generator (`torch.Generator`, *optional*): PyTorch random number generator. )r7r)rrrmoder r ) r,rr6rr7rrrdecr2r2r3rDs   zAutoencoderKLAllegro.forward)rrrrrrrr;rrrr;rrT)T)FTN)rErFrGrH _supports_gradient_checkpointingrr!rr}rJrIr r7rLrrr r rrr r rr GeneratorrDrMr2r2r0r3rs-     I  "S\rc Cs|\}}}|\}}} |\} } } |dkr|dkrN|ddddd|ddddftd||j||dd|ddddd|ddddf<||dkr|dddd| dddddfdtd||j||dd|dddd| dddddf<|dkr|ddddddd| ddftd| |j| | d|ddddddd| ddf<||dkr |dddddd| dddfdtd| |j| | d|dddddd| dddf<| dkrA|ddddddddd| ftd| |j| |ddddddddd| f<| | dkr~|dddddddd| dfdtd| |j| |dddddddd| df<|S)Nrr)r7arangerIrdevicereshape) n_paramh_paramw_paramrnn_max overlap_nrh_max overlap_hww_max overlap_wr2r2r3r0sB   $ & & ($&&( $&&(r)$rr7torch.nnr"configuration_utilsrrutils.accelerate_utilsrattention_processorrrautoencoders.vaer r downsamplingr modeling_outputsr modeling_utilsr rxr upsamplingrvaerModulerrNr~rrrrrr2r2r2r3s2        eWRo