o ۷i @sddlZddlmZmZddlZddlZddlZddlm m Z ddl m Z mZmZmZddlmZmZddlmZmZmZmZddlmZmZmZmZddlmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5m6Z6ddl7m8Z8e(rddl9m:m;Z<dZ=ndZ=e)>e?Z@dZA ddejBdejCdBdeDfddZEGddde5e6eeee3e ZFdS) N)AnyCallable)CLIPImageProcessor CLIPTextModel CLIPTokenizerCLIPVisionModelWithProjection)PipelineImageInputVaeImageProcessor)FromSingleFileMixinIPAdapterMixinStableDiffusionLoraLoaderMixinTextualInversionLoaderMixin) AutoencoderKLImageProjectionUNet2DConditionModelUNetMotionModel)SparseControlNetModel)adjust_lora_scale_text_encoder) MotionAdapter)KarrasDiffusionSchedulers)USE_PEFT_BACKENDis_torch_xla_availableloggingreplace_example_docstringscale_lora_layersunscale_lora_layers)is_compiled_module randn_tensor)VideoProcessor) FreeInitMixin)DiffusionPipelineStableDiffusionMixin)AnimateDiffPipelineOutputTFa Examples: ```python >>> import torch >>> from diffusers import AnimateDiffSparseControlNetPipeline >>> from diffusers.models import AutoencoderKL, MotionAdapter, SparseControlNetModel >>> from diffusers.schedulers import DPMSolverMultistepScheduler >>> from diffusers.utils import export_to_gif, load_image >>> model_id = "SG161222/Realistic_Vision_V5.1_noVAE" >>> motion_adapter_id = "guoyww/animatediff-motion-adapter-v1-5-3" >>> controlnet_id = "guoyww/animatediff-sparsectrl-scribble" >>> lora_adapter_id = "guoyww/animatediff-motion-lora-v1-5-3" >>> vae_id = "stabilityai/sd-vae-ft-mse" >>> device = "cuda" >>> motion_adapter = MotionAdapter.from_pretrained(motion_adapter_id, torch_dtype=torch.float16).to(device) >>> controlnet = SparseControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float16).to(device) >>> vae = AutoencoderKL.from_pretrained(vae_id, torch_dtype=torch.float16).to(device) >>> scheduler = DPMSolverMultistepScheduler.from_pretrained( ... model_id, ... subfolder="scheduler", ... beta_schedule="linear", ... algorithm_type="dpmsolver++", ... use_karras_sigmas=True, ... ) >>> pipe = AnimateDiffSparseControlNetPipeline.from_pretrained( ... model_id, ... motion_adapter=motion_adapter, ... controlnet=controlnet, ... vae=vae, ... scheduler=scheduler, ... torch_dtype=torch.float16, ... ).to(device) >>> pipe.load_lora_weights(lora_adapter_id, adapter_name="motion_lora") >>> pipe.fuse_lora(lora_scale=1.0) >>> prompt = "an aerial view of a cyberpunk city, night time, neon lights, masterpiece, high quality" >>> negative_prompt = "low quality, worst quality, letterboxed" >>> image_files = [ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-scribble-1.png", ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-scribble-2.png", ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-scribble-3.png", ... ] >>> condition_frame_indices = [0, 8, 15] >>> conditioning_frames = [load_image(img_file) for img_file in image_files] >>> video = pipe( ... prompt=prompt, ... negative_prompt=negative_prompt, ... num_inference_steps=25, ... conditioning_frames=conditioning_frames, ... controlnet_conditioning_scale=1.0, ... controlnet_frame_indices=condition_frame_indices, ... generator=torch.Generator().manual_seed(1337), ... ).frames[0] >>> export_to_gif(video, "output.gif") ``` sampleencoder_output generator sample_modecCsRt|dr|dkr|j|St|dr|dkr|jSt|dr%|jStd)N latent_distr&argmaxlatentsz3Could not access latents of provided encoder_output)hasattrr*r&moder,AttributeError)r'r(r)r0u/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.pyretrieve_latentsws   r2c6seZdZdZdZgdZgdZ  dXdedede d e e Bd e d e d ed edeffdd Z     dYdejdBdejdBdedBdedBfddZdZddZddZddZddZ        d[d efd!d"Zd#d$Z dZd%d&Zd'd(Zd)ejd*ed+ed,ejd-ej d.e!ejejff d/d0Z"e#d1d2Z$e#d3d4Z%e#d5d6Z&e#d7d8Z'e#d9d:Z(e)e*e+dddd;dd?dddddddd@dAdddBgdCdddDgfdEe,e-e,BdBdFedBdGedBd*edHedIedJe,e-e,BdBdKedLedMej.e-ej.BdBdDejdBdejdBdejdBdNe/dBdOe-ejdBd)e-e/dBdPe,dQe0dRe1e,e2fdBd ee-eBd+e-edSe0dedBdTe3eegdfdBdUe-e,f2dVdWZ4Z5S)\#AnimateDiffSparseControlNetPipelinea Pipeline for controlled text-to-video generation using the method described in [SparseCtrl: Adding Sparse Controls to Text-to-Video Diffusion Models](https://huggingface.co/papers/2311.16933). This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). The pipeline also inherits the following loading methods: - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. text_encoder ([`CLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). tokenizer (`CLIPTokenizer`): A [`~transformers.CLIPTokenizer`] to tokenize text. unet ([`UNet2DConditionModel`]): A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. motion_adapter ([`MotionAdapter`]): A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. z&text_encoder->image_encoder->unet->vae)feature_extractor image_encodermotion_adapter)r, prompt_embedsnegative_prompt_embedsNvae text_encoder tokenizerunetr6 controlnet schedulerr4r5c stt|trt||}|j||||||||| d t|ddr/dt|j j j dnd|_ t d|j d|_t|j ddd |_dS) N) r9r:r;r<r6r=r>r4r5r9r r$F) do_resizevae_scale_factorT)rAdo_convert_rgb do_normalize)super__init__ isinstancerr from_unet2dregister_modulesgetattrlenr9configblock_out_channelsrArvideo_processorr control_image_processor) selfr9r:r;r<r6r=r>r4r5 __class__r0r1rEs&  (  z,AnimateDiffSparseControlNetPipeline.__init__r7r8 lora_scale clip_skipc  Cs|durt|tr||_tst|j|nt|j||dur't|tr'd} n|dur5t|tr5t |} n|j d} |durt|t rJ| ||j }|j |d|j jddd} | j} |j |ddd j} | j d | j d krt| | s|j | dd|j jdd f}td |j jd |t|jjd r|jjjr| j|}nd}| dur|j| ||d}|d}n|j| ||dd}|d | d }|jj|}|jdur|jj}n |jdur|jj}n|j}|j||d}|j \}}}|d|d}||||d }|r|dur|durdg| }nC|dur8t |t |ur8t!dt |dt |dt|trB|g}n| t |kr\t"d|dt |d|d| d |}t|t rk| ||j }|j d}|j |d|ddd}t|jjd r|jjjr|j|}nd}|j|j||d}|d}|r|j d}|j||d}|d|d}|| ||d }|jdurt|trtrt#|j|||fS)a Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. lora_scale (`float`, *optional*): A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. Nr$r max_lengthTpt)paddingrT truncationreturn_tensorslongest)rVrXz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: use_attention_mask)attention_mask)r\output_hidden_statesdtypedevicez?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)$rFr _lora_scalerrr:rstrlistrJshapermaybe_convert_promptr;model_max_length input_idstorchequal batch_decodeloggerwarningr-rKr[r\to text_modelfinal_layer_normr_r<repeatviewtype TypeError ValueErrorr)rOpromptr`num_images_per_promptdo_classifier_free_guidancenegative_promptr7r8rRrS batch_size text_inputstext_input_idsuntruncated_ids removed_textr\prompt_embeds_dtypebs_embedseq_len_ uncond_tokensrT uncond_inputr0r0r1 encode_prompts+                     z1AnimateDiffSparseControlNetPipeline.encode_promptc Cst|jj}t|tjs|j|ddj}|j ||d}|rH|j|ddj d}|j |dd}|jt |ddj d}|j |dd}||fS||j }|j |dd}t |} || fS) NrU)rX)r`r_T)r]rdim)nextr5 parametersr_rFrjTensorr4 pixel_valuesro hidden_statesrepeat_interleave zeros_like image_embeds) rOimager`rxr]r_image_enc_hidden_statesuncond_image_enc_hidden_statesruncond_image_embedsr0r0r1 encode_images(    z0AnimateDiffSparseControlNetPipeline.encode_imagecCslg}|rg}|duret|ts|g}t|t|jjjkr/tdt|dt|jjjdt||jjjD],\}} t| t } | ||d| \} } | | dddf|rc| | dddfq7n|D]} |rw| d\} } | | | | qgg}t |D]0\} } t j| g|dd} |rt j|| g|dd} t j| | gdd} | j|d} | | q|S) NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got z images and z IP Adapters.r$r rrr`)rFrerJr<encoder_hid_projimage_projection_layersrvziprrappendchunk enumeraterjcatro)rOip_adapter_imageip_adapter_image_embedsr`rxryrnegative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsir0r0r1prepare_ip_adapter_image_embedssH        zCAnimateDiffSparseControlNetPipeline.prepare_ip_adapter_image_embedsc Csd|jjj|}|j\}}}}}|ddddd|||||}|j|j}|dddf||df|jddddddd}|}|S)Nr$rr rrZ) r9rKscaling_factorrfpermutereshapedecoder&float) rOr,r{channels num_framesheightwidthrvideor0r0r1decode_latentss"8z2AnimateDiffSparseControlNetPipeline.decode_latentscCsXdtt|jjjv}i}|r||d<dtt|jjjv}|r*||d<|S)Netar()setinspect signaturer>steprkeys)rOr(r accepts_etaextra_step_kwargsaccepts_generatorr0r0r1prepare_extra_step_kwargssz=AnimateDiffSparseControlNetPipeline.prepare_extra_step_kwargs?controlnet_conditioning_scalec sN|ddks |ddkrtd|d|d| dur8tfdd| Ds8tdjd fd d | D|durK|durKtd |d |d|durW|durWtd|durnt|tsnt|tsntdt||dur|durtd|d|d|dur|dur|j|jkrtd|jd|jd|dur|durtd|durt|tstdt||djdvrtd|djdt t dotj t j jj} tj ts| rtj jtrt| tr| D] } | ||qn | ||nJtj ts| r%tj jtr%t| ts#tddSJ)Nr?rz7`height` and `width` have to be divisible by 8 but are z and rbc3s|]}|jvVqdSN_callback_tensor_inputs.0krOr0r1 s  zCAnimateDiffSparseControlNetPipeline.check_inputs..z2`callback_on_step_end_tensor_inputs` has to be in z , but found csg|] }|jvr|qSr0rrrr0r1 szDAnimateDiffSparseControlNetPipeline.check_inputs..zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z2`prompt` has to be of type `str` or `list` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` zProvide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined.z:`ip_adapter_image_embeds` has to be of type `list` but is )rrzF`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is Dscaled_dot_product_attentionFzLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.)rvallrrFrdrertrfndimr-Fr=rj_dynamo eval_frameOptimizedModuler _orig_mod check_imagerru)rOrwrrrzr7r8rr"callback_on_step_end_tensor_inputsrr is_compiledimage_r0rr1 check_inputss           z0AnimateDiffSparseControlNetPipeline.check_inputsc Cs$t|tjj}t|tj}t|tj}t|to t|dtjj}t|to-t|dtj}t|to:t|dtj} |sP|sP|sP|sP|sP| sPtdt ||rUd} nt |} |duret|t red} n|durst|trst |} n |dur||j d} | dkr| | krt d| d| dSdS)Nrzimage must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is r$zdIf image batch size is not 1, image batch size must be same as prompt batch size. image batch size: z, prompt batch size: )rFPILImagerjrnpndarrayrerurtrJrdrfrv) rOrrwr7 image_is_pilimage_is_tensor image_is_npimage_is_pil_listimage_is_tensor_listimage_is_np_listimage_batch_sizeprompt_batch_sizer0r0r1rKsF     z/AnimateDiffSparseControlNetPipeline.check_imagec Cs~|||||j||jf} t|tr%t||kr%tdt|d|d| dur2t| |||d} n| |} | |jj} | S)Nz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)r(r`r_) rArFrerJrvrror>init_noise_sigma) rOr{num_channels_latentsrrrr_r`r(r,rfr0r0r1prepare_latentsqs"   z3AnimateDiffSparseControlNetPipeline.prepare_latentsc Cs|jj|||d}|d||}|j\}}} }}|dkr&|dks(J|jjrY| ||| ||}d|d}t |j ||j j j} | ||d||j||j} n|} | ddddd} | S)N)rrrr$r rr)rN preprocess unsqueezerorfminmaxr="use_simplified_condition_embeddingrr2r9encoderKrrAr) rOrrrr`r_controlnet_imagesr{rrconditioning_framesr0r0r1 prepare_images z1AnimateDiffSparseControlNetPipeline.prepare_imagerrcontrolnet_frame_indicesr`r_returnc Cs|jdt|ks J|j\}}}} } tj|||| | f||d} tj|d|| | f||d} |dddddt|f| dddd|f<d| dddd|f<| | fS)Nr r^r$)rfrJrjzeros) rOrrrr`r_r{rrrrcontrolnet_condcontrolnet_cond_maskr0r0r1#prepare_sparse_control_conditionings0zGAnimateDiffSparseControlNetPipeline.prepare_sparse_control_conditioningcC|jSr_guidance_scalerr0r0r1guidance_scalez2AnimateDiffSparseControlNetPipeline.guidance_scalecCrr) _clip_skiprr0r0r1rSrz-AnimateDiffSparseControlNetPipeline.clip_skipcCs |jdkS)Nr$rrr0r0r1rys z?AnimateDiffSparseControlNetPipeline.do_classifier_free_guidancecCrr)_cross_attention_kwargsrr0r0r1cross_attention_kwargsrz:AnimateDiffSparseControlNetPipeline.cross_attention_kwargscCrr)_num_timestepsrr0r0r1 num_timestepsrz1AnimateDiffSparseControlNetPipeline.num_timesteps2g@r$gpilTrFr,rwrrnum_inference_stepsrrznum_videos_per_promptrr(rr output_type return_dictr guess_modecallback_on_step_endrc9 Csrt|jr |jjn|j}|p|jjj|j}|p|jjj|j}d}|j||||| | |||||d ||_||_ ||_ |durGt |t rGd}n|durUt |t rUt|}n| jd}|j}t |trf|jjn|jdjj}|pp|}|jdur}|jddnd}|j||||j|| | ||jd \} } |jrt| | g} | j|dd} |dus|dur|||||||j}||||||j}||||||j\} }!|j j!||d|j j"}"|jjj#}#|$|||#|||| j|| | } |%| | }$|dus|durd |ind}%|j&r|j'nd}&t(|&D]}'|j&r"|)| |'||| j| \} }"t|"|_*t|"||j j+}(|j,|j*d })t-|"D]\}*}+|jrMt| gd n| },|j .|,|+},|rn|jrn| }-|j .|-|+}-| /d d}.n|,}-| }.|j|-|+|.| |!||d d \}/}0|j|,|+| ||%|/|0dj0}1|jr|1/d \}2}3|2||3|2}1|j j1|1|+| fi|$j2} |duri}4|D] }5t3|5|4|5<q|||*|+|4}6|64d| } |64d| } |64d| } |*t|"dks|*d|(kr|*d|j j+dkr|)5t6rt78q=Wdn 1swYq|dkr| }7n |9| }8|j:j;|8|d}7|<|s4|7fSt=|7dS)uI The call function to the pipeline for generation. Args: prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The width in pixels of the generated video. num_frames (`int`, *optional*, defaults to 16): The number of video frames that are generated. Defaults to 16 frames which at 8 frames per seconds amounts to 2 seconds of video. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor is generated by sampling using the supplied random `generator`. Latents should be of shape `(batch_size, num_channel, num_frames, height, width)`. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, text embeddings are generated from the `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. conditioning_frames (`list[PipelineImageInput]`, *optional*): The SparseControlNet input to provide guidance to the `unet` for generation. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated video. Choose between `torch.Tensor`, `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead of a plain tuple. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. controlnet_frame_indices (`list[int]`): The indices where the conditioning frames must be applied for generation. Multiple frames can be provided to guide the model to generate similar structure outputs, where the `unet` can "fill-in-the-gaps" for interpolation videos, or a single frame could be provided for general expected structure. Must have the same length as `conditioning_frames`. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. Examples: Returns: [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. r$) rwrrrzr7r8rrrrrNrscale)r7r8rRrS)repeatsrrr)totalr F)encoder_hidden_statesrconditioning_maskconditioning_scalerr)r radded_cond_kwargsdown_block_additional_residualsmid_block_additional_residualr,r7r8latent)rr)frames)>rr=rr<rK sample_sizerArrrrrFrdrerJrf_execution_devicerglobal_pool_conditionsnetsrgetrryrSrjrrrrr_rr> set_timesteps timesteps in_channelsrrfree_init_enabled_free_init_num_itersrange_apply_free_initrorder progress_barrscale_model_inputrr&r prev_samplelocalspopupdate XLA_AVAILABLExm mark_steprrMpostprocess_videomaybe_free_model_hooksr%)9rOrwrrrrrrzrrr(r,r7r8rrrrrrrrrrSrrr=r{r`rtext_encoder_lora_scalerrrrrrrnum_free_init_itersfree_init_iternum_warmup_stepsr!rtlatent_model_inputcontrol_model_inputcontrolnet_prompt_embedsdown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textcallback_kwargsrcallback_outputsr video_tensorr0r0r1__call__s"s                6 ?  z,AnimateDiffSparseControlNetPipeline.__call__)NN)NNNNNr)NNNNNNNr)6__name__ __module__ __qualname____doc__model_cpu_offload_seq_optional_componentsrrrrrrrrrrrrErjrrintrrrrrrrrrr`r_tuplerpropertyrrSryrrno_gradrEXAMPLE_DOC_STRINGrdre Generatorr booldictrrr< __classcell__r0r0rPr1r3sB    (   8.  _'               r3)Nr&)Grtypingrrnumpyrrrjtorch.nn.functionalnn functionalr transformersrrrrimage_processorr r loadersr r r rmodelsrrrr(models.controlnets.controlnet_sparsectrlr models.lorarmodels.unets.unet_motion_modelr schedulersrutilsrrrrrrutils.torch_utilsrrrMrfree_init_utilsr!pipeline_utilsr"r#pipeline_outputr%torch_xla.core.xla_modelcore xla_modelr(r' get_loggerr=rmrGrrHrdr2r3r0r0r0r1sV         @