o Gi @sddlZddlmZmZddlZddlmmZddl m Z m Z m Z m Z ddlmZddlmZmZmZmZddlmZmZmZmZmZmZddlmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/dd l0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7m8Z8ddl9m:Z:e)rddl;m<m=Z>dZ?ndZ?e*@eAZBdZC d%dejDdejEdBdeFfddZG    d&deHdBdeFejIBdBdeJeHdBd eJeKdBfd!d"ZLGd#d$d$e7e8eeee3e5e ZMdS)'N)AnyCallable)CLIPImageProcessor CLIPTextModel CLIPTokenizerCLIPVisionModelWithProjection)PipelineImageInput)FromSingleFileMixinIPAdapterMixinStableDiffusionLoraLoaderMixinTextualInversionLoaderMixin) AutoencoderKLControlNetModelImageProjectionMultiControlNetModelUNet2DConditionModelUNetMotionModel)adjust_lora_scale_text_encoder) MotionAdapter) DDIMSchedulerDPMSolverMultistepSchedulerEulerAncestralDiscreteSchedulerEulerDiscreteSchedulerLMSDiscreteScheduler PNDMScheduler)USE_PEFT_BACKENDis_torch_xla_availableloggingscale_lora_layersunscale_lora_layers)is_compiled_module randn_tensor)VideoProcessor) FreeInitMixin)AnimateDiffFreeNoiseMixin)DiffusionPipelineStableDiffusionMixin)AnimateDiffPipelineOutputTFak Examples: ```py >>> import torch >>> from PIL import Image >>> from tqdm.auto import tqdm >>> from diffusers import AnimateDiffVideoToVideoControlNetPipeline >>> from diffusers.utils import export_to_gif, load_video >>> from diffusers import AutoencoderKL, ControlNetModel, MotionAdapter, LCMScheduler >>> controlnet = ControlNetModel.from_pretrained( ... "lllyasviel/sd-controlnet-openpose", torch_dtype=torch.float16 ... ) >>> motion_adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM") >>> vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16) >>> pipe = AnimateDiffVideoToVideoControlNetPipeline.from_pretrained( ... "SG161222/Realistic_Vision_V5.1_noVAE", ... motion_adapter=motion_adapter, ... controlnet=controlnet, ... vae=vae, ... ).to(device="cuda", dtype=torch.float16) >>> pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear") >>> pipe.load_lora_weights( ... "wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora" ... ) >>> pipe.set_adapters(["lcm-lora"], [0.8]) >>> video = load_video( ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/dance.gif" ... ) >>> video = [frame.convert("RGB") for frame in video] >>> from controlnet_aux.processor import OpenposeDetector >>> open_pose = OpenposeDetector.from_pretrained("lllyasviel/Annotators").to("cuda") >>> for frame in tqdm(video): ... conditioning_frames.append(open_pose(frame)) >>> prompt = "astronaut in space, dancing" >>> negative_prompt = "bad quality, worst quality, jpeg artifacts, ugly" >>> strength = 0.8 >>> with torch.inference_mode(): ... video = pipe( ... video=video, ... prompt=prompt, ... negative_prompt=negative_prompt, ... num_inference_steps=10, ... guidance_scale=2.0, ... controlnet_conditioning_scale=0.75, ... conditioning_frames=conditioning_frames, ... strength=strength, ... generator=torch.Generator().manual_seed(42), ... ).frames[0] >>> video = [frame.resize(conditioning_frames[0].size) for frame in video] >>> export_to_gif(video, f"animatediff_vid2vid_controlnet.gif", fps=8) ``` sampleencoder_output generator sample_modecCsRt|dr|dkr|j|St|dr|dkr|jSt|dr%|jStd)N latent_distr+argmaxlatentsz3Could not access latents of provided encoder_output)hasattrr/r+moder1AttributeError)r,r-r.r5/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.pyretrieve_latents~s   r7num_inference_stepsdevice timestepssigmascKs|dur |dur td|dur>dtt|jjv}|s(td|jd|jd ||d||j}t |}||fS|durpdtt|jjv}|sZtd|jd|jd ||d ||j}t |}||fS|j|fd |i||j}||fS) a Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. Args: scheduler (`SchedulerMixin`): The scheduler to get timesteps from. num_inference_steps (`int`): The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. NzYOnly one of `timesteps` or `sigmas` can be passed. Please choose one to set custom valuesr:zThe current scheduler class zx's `set_timesteps` does not support custom timestep schedules. Please check whether you are using the correct scheduler.)r:r9r;zv's `set_timesteps` does not support custom sigmas schedules. Please check whether you are using the correct scheduler.)r;r9r9r5) ValueErrorsetinspect signature set_timesteps parameterskeys __class__r:len) schedulerr8r9r:r;kwargsaccepts_timesteps accept_sigmasr5r5r6retrieve_timestepss2  rIc@seZdZdZdZgdZgdZ  dfdedede d e e Bd e d e ee Bee BeBd eeBeBeBeBeBd edeffdd Z     dgdejdBdejdBdedBdedBfddZdhddZddZ didedejfddZ!didefd d!Z"d"d#Z#          $ % $djd&d'Z$d(d)Z%  * * + ,       -dkd.ejdBd/ed0ed1ed2ed3edBd4ej&dBd5ej'dBd6ej(eej(BdBd7ejdBded8e)dejfd9d:Z* - -dld;d<Z+e,d=d>Z-e,d?d@Z.e,dAdBZ/e,dCdDZ0e,dEdFZ1e,dGdHZ2e3dddddId-dddJdKdd,d%ddddddddLdMdd$d-d%d$ddd7gdfd.eee4dNe5ee5BdBd/edBd0edBdOedPe)dQeedBdReedBdSedTedUe5ee5BdBdVedBdWed6ej(eej(BdBd7ejdBdejdBdejdBdXe4dBdYeejdBdZee4dBd[e5dBd\e)d]e6e5e7fdBd^eeeBd_e)d`eeeBdaeeeBdedBdbe8eegdfdBdcee5def>dddeZ9Z:S)m)AnimateDiffVideoToVideoControlNetPipelinea  Pipeline for video-to-video generation with ControlNet guidance. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). The pipeline also inherits the following loading methods: - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. text_encoder ([`CLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). tokenizer (`CLIPTokenizer`): A [`~transformers.CLIPTokenizer`] to tokenize text. unet ([`UNet2DConditionModel`]): A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. motion_adapter ([`MotionAdapter`]): A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. controlnet ([`ControlNetModel`] or `list[ControlNetModel]` or `tuple[ControlNetModel]` or `MultiControlNetModel`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. z&text_encoder->image_encoder->unet->vae)feature_extractor image_encodermotion_adapter)r1 prompt_embedsnegative_prompt_embedsNvae text_encoder tokenizerunetrM controlnetrErKrLc stt|trt||}t|ttfrt|}|j ||||||||| d t |ddr:dt |j j jdnd|_t|jd|_t|jddd |_dS) N) rPrQrRrSrMrTrErKrLrPr$r))vae_scale_factorTF)rVdo_convert_rgb do_normalize)super__init__ isinstancerr from_unet2dlisttuplerregister_modulesgetattrrDrPconfigblock_out_channelsrVr#video_processorcontrol_video_processor) selfrPrQrRrSrMrTrErKrLrCr5r6rZs*   (  z2AnimateDiffVideoToVideoControlNetPipeline.__init__rNrO lora_scale clip_skipc  Cs|durt|tr||_tst|j|nt|j||dur)t|ttfr)d} n|dur7t|t r7t |} n|j d} |durt|t rL| ||j}|j|d|jjddd} | j} |j|ddd j} | j d | j d krt| | s|j| dd|jjdd f}td |jjd |t|jjd r|jjjr| j|}nd}| dur|j| ||d}|d}n|j| ||dd}|d | d }|jj|}|jdur|jj}n |jdur|jj}n|j}|j||d}|j \}}}|d|d}| |||d }|r|dur|durdg| }nC|dur:t!|t!|ur:t"dt!|dt!|dt|trD|g}n| t |kr^t#d|dt |d|d| d |}t|t rm| ||j}|j d}|j|d|ddd}t|jjd r|jjjr|j|}nd}|j|j||d}|d}|r|j d}|j||d}|d|d}| | ||d }|jdurt|trtrt$|j|||fS)a Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. lora_scale (`float`, *optional*): A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. Nr)r max_lengthTpt)paddingri truncationreturn_tensorslongest)rkrmz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: use_attention_mask)attention_mask)rqoutput_hidden_states)dtyper9z?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)%r[r _lora_scalerrrQrstrdictr]rDshaper maybe_convert_promptrRmodel_max_length input_idstorchequal batch_decodeloggerwarningr2rarprqto text_modelfinal_layer_normrsrSrepeatviewtype TypeErrorr<r )repromptr9num_images_per_promptdo_classifier_free_guidancenegative_promptrNrOrgrh batch_size text_inputstext_input_idsuntruncated_ids removed_textrqprompt_embeds_dtypebs_embedseq_len_ uncond_tokensri uncond_inputr5r5r6 encode_prompts+                     z7AnimateDiffVideoToVideoControlNetPipeline.encode_promptc Cst|jj}t|tjs|j|ddj}|j ||d}|rH|j|ddj d}|j |dd}|jt |ddj d}|j |dd}||fS||j }|j |dd}t |} || fS) Nrj)rmr9rsT)rrrdim)nextrLrArsr[r}TensorrK pixel_valuesr hidden_statesrepeat_interleave zeros_like image_embeds) reimager9rrrrsimage_enc_hidden_statesuncond_image_enc_hidden_statesruncond_image_embedsr5r5r6 encode_images(    z6AnimateDiffVideoToVideoControlNetPipeline.encode_imagecCslg}|rg}|duret|ts|g}t|t|jjjkr/tdt|dt|jjjdt||jjjD],\}} t| t } | ||d| \} } | | dddf|rc| | dddfq7n|D]} |rw| d\} } | | | | qgg}t |D]0\} } t j| g|dd} |rt j|| g|dd} t j| | gdd} | j|d} | | q|S) NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got z images and z IP Adapters.r)r$rr)r9)r[r]rDrSencoder_hid_projimage_projection_layersr<ziprrappendchunk enumerater}catr)reip_adapter_imageip_adapter_image_embedsr9rrrnegative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsir5r5r6prepare_ip_adapter_image_embedssH        zIAnimateDiffVideoToVideoControlNetPipeline.prepare_ip_adapter_image_embedsdecode_chunk_sizereturncCsRg}tdt||D]}||||}t|j||d}||q t|S)Nr)r-)rangerDr7rPencoderr}r)revideor-rr1r batch_videor5r5r6 encode_videos   z6AnimateDiffVideoToVideoControlNetPipeline.encode_videoc Csd|jjj|}|j\}}}}}|ddddd|||||}g}td|jd|D]} || | |} |j| j} | | q-t |}|dddf||df|jddddddd}| }|S)Nr)rr$rro) rPrascaling_factorrypermutereshaperdecoder+rr}rfloat) rer1rrchannels num_framesheightwidthrr batch_latentsr5r5r6decode_latents&s"  8z8AnimateDiffVideoToVideoControlNetPipeline.decode_latentscCsXdtt|jjjv}i}|r||d<dtt|jjjv}|r*||d<|S)Netar-)r=r>r?rEsteprArB)rer-r accepts_etaextra_step_kwargsaccepts_generatorr5r5r6prepare_extra_step_kwargs9szCAnimateDiffVideoToVideoControlNetPipeline.prepare_extra_step_kwargs?c s*|dks|dkrtd||ddks|ddkr&td|d|d| durGtfdd | DsGtd jd fd d | D|durZ| durZtd|d| d|durf| durftd|dur{t|tttfs{tdt||dur| durtd|d| d| dur| dur| j| jkrtd| jd| jd|dur|durtd| dur| durtd| durt| tstdt| | dj dvrtd| dj dtj t rt|trt dtj jdt|dttd otj tjjj}|durt|n|jd!}tj ts3|rVtj jtrVttsBtd"tt|krUtd#|d$tnQtj t sh|rtj jt rttrvtdtstd%ttd|krtd&|d$ttfd'd Drtd(nJtj ts|rtj jtrt|tstd*n?tj t s|rtj jt rt|trtd+d |Drtd,nt|trt|tj jkrtd-nJt|ttfs|g}t|ttfs|g}t|t|kr1td.t|d/t|d0tj t r^t|tj jkr^td1|d2t|d3tj jd4tj jd t||D]/\}}||krwtd5|d6|d|d7krtd5|d8|d9krtd:|d;qcdS)cs  zIAnimateDiffVideoToVideoControlNetPipeline.check_inputs..z2`callback_on_step_end_tensor_inputs` has to be in z , but found csg|] }|jvr|qSr5rrrr5r6 gszJAnimateDiffVideoToVideoControlNetPipeline.check_inputs..zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z:`prompt` has to be of type `str`, `list` or `dict` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` z3Only one of `video` or `latents` should be providedzProvide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined.z:`ip_adapter_image_embeds` has to be of type `list` but is )rrzF`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is Dz You have z! ControlNets and you have passed z= prompts. The conditionings will be fixed across the prompts.scaled_dot_product_attentionr$z>For single controlnet, `image` must be of type `list` but got zExcepted image to have length z" but got len(conditioning_frames)=z_For multiple controlnets: `image` must be type list of lists but got type(conditioning_frames)=z$Expected length of image sublist as c3s$|] }t|tdkVqdS)rN)rD)rimg)conditioning_framesr5r6rs"zDAll conditioning frame batches for multicontrolnet must be same sizeFzLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.css|]}t|tVqdSr)r[r]rrr5r5r6rszEA single batch of multiple conditionings are supported at the moment.zFor multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have the same length as the number of controlnetsz`control_guidance_start` has z* elements, but `control_guidance_end` has zI elements. Make sure to provide the same number of elements to each list.z`control_guidance_start`: z has z elements but there are z- controlnets available. Make sure to provide zcontrol guidance start: z4 cannot be larger or equal to control guidance end: rz can't be smaller than 0.rzcontrol guidance end: z can't be larger than 1.0.)r<allrr[rwr]rxrryndimrTrrrrDnetsr2Fr}_dynamo eval_frameOptimizedModuler _orig_modranyrr^r)rerstrengthrrrrr1rrNrOrr"callback_on_step_end_tensor_inputscontrolnet_conditioning_scalecontrol_guidance_startcontrol_guidance_end is_compiledrstartendr5)rrer6 check_inputsJs                  0   z6AnimateDiffVideoToVideoControlNetPipeline.check_inputscCs@tt|||}t||d}|||jjd}|||fS)Nr)minintmaxrEorder)rer8r:rr9 init_timestept_startr5r5r6 get_timestepss z7AnimateDiffVideoToVideoControlNetPipeline.get_timesteps@rr)Frrrnum_channels_latentsrtimesteprsr9r-r1 add_noisec s"| dur jdn| jd} ||| |j|jf}ttr3t|kr3tdtd|d| durjjjrH jj t j dttr\fddt |D}n fd dD}t j|d d }jjjryj || |}jjj|}||jd kr||jd d krd |d |jd d}t|||jd kr||jd d krtd|jd d|dt j|gd d }t|j||d}j|||d dddd} | S|| jkrtd|d| j| j ||d} | rt|||d}j| ||} | S)Nr)r$z/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.rscs(g|]}||dqSrr unsqueezerrr-rerr5r6r$szMAnimateDiffVideoToVideoControlNetPipeline.prepare_latents..cs g|] }|dqSrr)rvid)rr-rer5r6r)s rrzYou have passed z# text prompts (`prompt`), but only zp initial images (`image`). Please make sure to update your script to pass as many initial images as text promptsz'Cannot duplicate `image` of batch size z to z text prompts.)r-r9rsrrz!`latents` expected to have shape=z, but found latents.shape=)ryrVr[r]rDr<rPra force_upcastrrr}float32rrrr"rErr)rerrrrrrrsr9r-r1rrrry init_latents error_messagenoiser5rr6prepare_latentss\         z9AnimateDiffVideoToVideoControlNetPipeline.prepare_latentsc Cs|jj|||djtjd}|ddddddd}|jd} | dkr'|} n|} |j| dd}|j||d }|rC| sCt |gd}|S) Nrrrrr$r)rrrr) rdpreprocess_videorr}rrflattenryrr) rerrrrnum_videos_per_promptr9rsr guess_modevideo_batch_size repeat_byr5r5r6prepare_conditioning_framesRs  zEAnimateDiffVideoToVideoControlNetPipeline.prepare_conditioning_framescC|jSr_guidance_scalerr5r5r6guidance_scalerz8AnimateDiffVideoToVideoControlNetPipeline.guidance_scalecCrr) _clip_skiprr5r5r6rhvrz3AnimateDiffVideoToVideoControlNetPipeline.clip_skipcCs |jdkS)Nr)rrr5r5r6r}s zEAnimateDiffVideoToVideoControlNetPipeline.do_classifier_free_guidancecCrr)_cross_attention_kwargsrr5r5r6cross_attention_kwargsrz@AnimateDiffVideoToVideoControlNetPipeline.cross_attention_kwargscCrr)_num_timestepsrr5r5r6 num_timestepsrz7AnimateDiffVideoToVideoControlNetPipeline.num_timestepscCrr) _interruptrr5r5r6 interruptrz3AnimateDiffVideoToVideoControlNetPipeline.interrupt2g@g?pilTrr8enforce_inference_stepsr:r;rrrrrrrr output_type return_dictrrrrrcallback_on_step_endrc Gst|jr |jjn|j} t|tst|trt||g}n3t|ts0t|tr0t||g}n!t|tsQt|tsQt| trDt| jnd}!|!|g|!|g}}|pZ|jj j |j }|pd|jj j |j }d} |j d,id|d| d|d|d| d|d|d |d |d |d |d |d|d|d|d|| |_ ||_||_d|_|durt|ttfrd}"n|durt|trt|}"n|jd}"|j}#|j}$trd}%n|#}%|st|j||%|\}||| |#\}dd|"| }&n#t|| }'t|j|'|%|\}'| ddd|"| }&|dur;|jj|||d}|ddddd}|j|#|$d}|jj j }(|j!||||(|"| |&|$|#||||d }|j"dur`|j"#ddnd})|jd}*|j$r|j%||*|#| |j&| |||)|j'd \}}n$|j(||#| |j&| |||)|j'd \}}|j&rt)*||g}|j+|*dd}|dus|dur|,|||#|"| |j&}+t| trt|t-r|gt| j}t| t.r| j j/n| jdj j/},|p|,}g}-t0tD]fd d!t1||D}.|-2t| t.r|.dn|.qt| t.r&|j3||||"| |*| |#| j|j&|d" }n,t| trPg}/|D]}0|j3|0|||"| |*| |#| j|j&|d" }1|/2|1q0|/}nJ|4|| }2|dusb|durfd#|+ind}3|j5ro|j6nd}4t0|4D][}5|j5r|7||5||#|j|\}t}||| |#\}t|_8t||jj9}6|j:|j8d$}7t;D] \}8|j<rq|j&rt)*|gdn|}9|j=|9|8}9|r|j&r|}:|j=|:|8}:|>dd};n|9}:|};t|-trd%d!t1||-D}}?|j|9|8||j"|3|>|?d(jA}@|j&r^|@>d\}A}B|A| |B|A}@|jjB|@|8|fi|2jC}|duri}C|D] }DtD|D|C|D<qt|||8|C}E|EEd |}|EEd|}|EEd|}tdksd|6krd|jj9dkr|7FtrtGHqWdn 1swYqu|d)kr|}n|I||}F|jjJ|F|d*}|K|s|fStL|d+S)-u The call function to the pipeline for generation. Args: video (`list[PipelineImageInput]`): The input video to condition the generation on. Must be a list of images/frames of the video. prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The width in pixels of the generated video. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. strength (`float`, *optional*, defaults to 0.8): Higher strength leads to more differences between original video and generated video. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor is generated by sampling using the supplied random `generator`. Latents should be of shape `(batch_size, num_channel, num_frames, height, width)`. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, text embeddings are generated from the `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. conditioning_frames (`list[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If multiple ControlNets are specified, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated video. Choose between `torch.Tensor`, `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`AnimateDiffPipelineOutput`] instead of a plain tuple. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. decode_chunk_size (`int`, defaults to `16`): The number of frames to decode at a time when calling `decode_latents` method. Examples: Returns: [`pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. r)rrrrrrNrOrrr1rrrrrrFNrcpur r$rrr) rrrrrrrsr9r-r1rrscale) rrr9rrrrNrOrgrh)rNrOrgrh)repeatsrcs<g|]\}}dtt|kpdt|kqS)rr))rrD)rserr:r5r6rs*zFAnimateDiffVideoToVideoControlNetPipeline.__call__..) rrrrrr9rsrrr)totalcSsg|]\}}||qSr5r5)rcr*r5r5r6rsro)encoder_hidden_statescontrolnet_condconditioning_scalerr%)r/radded_cond_kwargsdown_block_additional_residualsmid_block_additional_residuallatent)rr$)framesr5)Mr!rTrr[r]rDrrrSra sample_sizerVrrrrrrwrxry_execution_devicers XLA_AVAILABLErIrErrrrcrrr in_channelsr rgetfree_noise_enabled_encode_prompt_free_noiserrhrr}rrrrrglobal_pool_conditionsrrrrrfree_init_enabled_free_init_num_iters_apply_free_initrr progress_barrr scale_model_inputr transposerr+r prev_samplelocalspopupdatexm mark_steprpostprocess_videomaybe_free_model_hooksr*)Grerrrrr8r#r:r;rrrrrr-r1rNrOrrrr$r%rrrrrrhr&rrrTmultrr9rstimestep_devicelatent_timestepdenoising_inference_stepsrtext_encoder_lora_scalerrr>controlnet_keepkeepscond_prepared_videosframe_prepared_videorr2num_free_init_itersfree_init_iternum_warmup_stepsrBtlatent_model_inputcontrol_model_inputcontrolnet_prompt_embeds cond_scalecontrolnet_cond_scaledown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textcallback_kwargsrcallback_outputs video_tensorr5r,r6__call__s                     "                6 N  z2AnimateDiffVideoToVideoControlNetPipeline.__call__)NN)NNNNNr)r) NNNNNNNNNrrr) Nrrrr)NNNNNrF)FF);__name__ __module__ __qualname____doc__model_cpu_offload_seq_optional_componentsrrrrrrrrr]r^rrrrrrrrrrZr}rrrrrrrrrrrrsr9 Generatorboolr rpropertyrrhrrrr no_gradr rwrxrrrh __classcell__r5r5rfr6rJs  0   8.  ,       \                    rJ)Nr+)NNNN)Nr>typingrrr}torch.nn.functionalnn functionalr transformersrrrrimage_processorr loadersr r r r modelsrrrrrr models.lorarmodels.unets.unet_motion_modelr schedulersrrrrrrutilsrrrrr utils.torch_utilsr!r"rcr#free_init_utilsr%free_noise_utilsr&pipeline_utilsr'r(pipeline_outputr*torch_xla.core.xla_modelcore xla_modelrIr9 get_loggerrirEXAMPLE_DOC_STRINGrrorwr7rr9r]rrIrJr5r5r5r6sn          B     ;