o ۷i @spddlZddlmZmZddlZddlmmZddl m Z m Z m Z m Z ddlmZddlmZmZmZmZddlmZmZmZmZmZmZddlmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5e$rddl6m7m8Z9dZ:ndZ:e%;e<Z=dZ>Gddde2e3eeee.e0e Z?dS)N)AnyCallable)CLIPImageProcessor CLIPTextModel CLIPTokenizerCLIPVisionModelWithProjection)PipelineImageInput)FromSingleFileMixinIPAdapterMixinStableDiffusionLoraLoaderMixinTextualInversionLoaderMixin) AutoencoderKLControlNetModelImageProjectionMultiControlNetModelUNet2DConditionModelUNetMotionModel)adjust_lora_scale_text_encoder) MotionAdapter)KarrasDiffusionSchedulers)USE_PEFT_BACKENDis_torch_xla_availableloggingscale_lora_layersunscale_lora_layers)is_compiled_module randn_tensor)VideoProcessor) FreeInitMixin)AnimateDiffFreeNoiseMixin)DiffusionPipelineStableDiffusionMixin)AnimateDiffPipelineOutputTFaT Examples: ```py >>> import torch >>> from diffusers import ( ... AnimateDiffControlNetPipeline, ... AutoencoderKL, ... ControlNetModel, ... MotionAdapter, ... LCMScheduler, ... ) >>> from diffusers.utils import export_to_gif, load_video >>> # Additionally, you will need a preprocess videos before they can be used with the ControlNet >>> # HF maintains just the right package for it: `pip install controlnet_aux` >>> from controlnet_aux.processor import ZoeDetector >>> # Download controlnets from https://huggingface.co/lllyasviel/ControlNet-v1-1 to use .from_single_file >>> # Download Diffusers-format controlnets, such as https://huggingface.co/lllyasviel/sd-controlnet-depth, to use .from_pretrained() >>> controlnet = ControlNetModel.from_single_file("control_v11f1p_sd15_depth.pth", torch_dtype=torch.float16) >>> # We use AnimateLCM for this example but one can use the original motion adapters as well (for example, https://huggingface.co/guoyww/animatediff-motion-adapter-v1-5-3) >>> motion_adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM") >>> vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16) >>> pipe: AnimateDiffControlNetPipeline = AnimateDiffControlNetPipeline.from_pretrained( ... "SG161222/Realistic_Vision_V5.1_noVAE", ... motion_adapter=motion_adapter, ... controlnet=controlnet, ... vae=vae, ... ).to(device="cuda", dtype=torch.float16) >>> pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear") >>> pipe.load_lora_weights( ... "wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora" ... ) >>> pipe.set_adapters(["lcm-lora"], [0.8]) >>> depth_detector = ZoeDetector.from_pretrained("lllyasviel/Annotators").to("cuda") >>> video = load_video( ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-vid2vid-input-1.gif" ... ) >>> conditioning_frames = [] >>> with pipe.progress_bar(total=len(video)) as progress_bar: ... for frame in video: ... conditioning_frames.append(depth_detector(frame)) ... progress_bar.update() >>> prompt = "a panda, playing a guitar, sitting in a pink boat, in the ocean, mountains in background, realistic, high quality" >>> negative_prompt = "bad quality, worst quality" >>> video = pipe( ... prompt=prompt, ... negative_prompt=negative_prompt, ... num_frames=len(video), ... num_inference_steps=10, ... guidance_scale=2.0, ... conditioning_frames=conditioning_frames, ... generator=torch.Generator().manual_seed(42), ... ).frames[0] >>> export_to_gif(video, "animatediff_controlnet.gif", fps=8) ``` c8seZdZdZdZddgZgdZ  dSdeded e d e e Bd e d e ee Bee BeBd ededBdedBffdd Z     dTdejdBdejdBdedBdedBfddZdUddZddZdVdefddZddZ      ! dWd"d#Z dUd$d%Z & &dXd'd(Z e!d)d*Z"e!d+d,Z#e!d-d.Z$e!d/d0Z%e!d1d2Z&e!d3d4Z'e(ddddd5d6dd7d!dddddddd8d9dd d&d!d ddd:gdfd;e)ee)BdedBd?ed@edAe)ee)BdBdBedBdCedDej*eej*BdBd:ejdBdejdBdejdBdEe+dBdFe+dBdGee+dBdHe)dBdIe,dJe-e)e.fdBdKeeeBdLe,dMeeeBdNeeeBdedBdOe/eegdfdBdPee)def6dQdRZ0Z1S)YAnimateDiffControlNetPipelinea Pipeline for text-to-video generation with ControlNet guidance. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). The pipeline also inherits the following loading methods: - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. text_encoder ([`CLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). tokenizer (`CLIPTokenizer`): A [`~transformers.CLIPTokenizer`] to tokenize text. unet ([`UNet2DConditionModel`]): A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. motion_adapter ([`MotionAdapter`]): A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. ztext_encoder->unet->vaefeature_extractor image_encoder)latents prompt_embedsnegative_prompt_embedsNvae text_encoder tokenizerunetmotion_adapter controlnet schedulerc stt|trt||}t|ttfrt|}|j ||||||||| d t |ddr:dt |j j jdnd|_t|jd|_t|jddd |_dS) N) r,r-r.r/r0r1r2r'r(r,rr$)vae_scale_factorTF)r4do_convert_rgb do_normalize)super__init__ isinstancerr from_unet2dlisttuplerregister_modulesgetattrlenr,configblock_out_channelsr4rvideo_processorcontrol_video_processor) selfr,r-r.r/r0r1r2r'r( __class__u/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.pyr8s*  (  z&AnimateDiffControlNetPipeline.__init__r*r+ lora_scale clip_skipc  Cs|durt|tr||_tst|j|nt|j||dur't|tr'd} n|dur5t|tr5t |} n|j d} |durt|t rJ| ||j }|j |d|j jddd} | j} |j |ddd j} | j d | j d krt| | s|j | dd|j jdd f}td |j jd |t|jjd r|jjjr| j|}nd}| dur|j| ||d}|d}n|j| ||dd}|d | d }|jj|}|jdur|jj}n |jdur|jj}n|j}|j||d}|j \}}}|d|d}||||d }|r|dur|durdg| }nC|dur8t |t |ur8t!dt |dt |dt|trB|g}n| t |kr\t"d|dt |d|d| d |}t|t rk| ||j }|j d}|j |d|ddd}t|jjd r|jjjr|j|}nd}|j|j||d}|d}|r|j d}|j||d}|d|d}|| ||d }|jdurt|trtrt#|j|||fS)a Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. lora_scale (`float`, *optional*): A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. Nr$r max_lengthTpt)paddingrK truncationreturn_tensorslongest)rMrOz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: use_attention_mask)attention_mask)rSoutput_hidden_states)dtypedevicez?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)$r9r _lora_scalerrr-rstrr;r?shaper maybe_convert_promptr.model_max_length input_idstorchequal batch_decodeloggerwarninghasattrr@rRrSto text_modelfinal_layer_normrUr/repeatviewtype TypeError ValueErrorr)rDpromptrVnum_images_per_promptdo_classifier_free_guidancenegative_promptr*r+rIrJ batch_size text_inputstext_input_idsuntruncated_ids removed_textrSprompt_embeds_dtypebs_embedseq_len_ uncond_tokensrK uncond_inputrGrGrH encode_prompts+                     z+AnimateDiffControlNetPipeline.encode_promptc Cst|jj}t|tjs|j|ddj}|j ||d}|rH|j|ddj d}|j |dd}|jt |ddj d}|j |dd}||fS||j }|j |dd}t |} || fS) NrL)rOrVrUT)rTrdim)nextr( parametersrUr9r_Tensorr' pixel_valuesre hidden_statesrepeat_interleave zeros_like image_embeds) rDimagerVrnrTrUimage_enc_hidden_statesuncond_image_enc_hidden_statesruncond_image_embedsrGrGrH encode_image~s(    z*AnimateDiffControlNetPipeline.encode_imagecCslg}|rg}|duret|ts|g}t|t|jjjkr/tdt|dt|jjjdt||jjjD],\}} t| t } | ||d| \} } | | dddf|rc| | dddfq7n|D]} |rw| d\} } | | | | qgg}t |D]0\} } t j| g|dd} |rt j|| g|dd} t j| | gdd} | j|d} | | q|S) NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got z images and z IP Adapters.r$rrrrV)r9r;r?r/encoder_hid_projimage_projection_layersrlziprrappendchunk enumerater_catre)rDip_adapter_imageip_adapter_image_embedsrVrnrornegative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsirGrGrHprepare_ip_adapter_image_embedssH        z=AnimateDiffControlNetPipeline.prepare_ip_adapter_image_embedsdecode_chunk_sizec Csd|jjj|}|j\}}}}}|ddddd|||||}g}td|jd|D]} || | |} |j| j} | | q-t |}|dddf||df|jddddddd}| }|S)Nr$rrrrQ) r,r@scaling_factorr[permutereshaperangedecodesamplerr_rfloat) rDr)rrqchannels num_framesheightwidthvideor batch_latentsrGrGrHdecode_latentss"  8z,AnimateDiffControlNetPipeline.decode_latentscCsXdtt|jjjv}i}|r||d<dtt|jjjv}|r*||d<|S)Neta generator)setinspect signaturer2steprkeys)rDrr accepts_etaextra_step_kwargsaccepts_generatorrGrGrHprepare_extra_step_kwargssz7AnimateDiffControlNetPipeline.prepare_extra_step_kwargs?c  sl|ddks |ddkrtd|d|d|dur8tfdd|Ds8tdjd fd d |D|durK|durKtd |d |d|durW|durWtd|durlt|tttfsltdt||dur|durtd|d|d|dur|dur|j|jkrtd|jd|jdtj t rt|trt dt j jdt |dttdotj tjjj} tj ts| rtj jtrttstdtt |krtd|dt nStj t s| rFtj jt rFttrtdtstdtt d|kr5td|dt dtfddDrEtd nJtj tsZ| retj jtret| tsdtd"n?tj t sw| rtj jt rt| trtd#d| Drtd$nt| trt | t j jkrtd%nJt| ttfs| g} t| ttfs| g} t | t | krtd&t | d't | d(tj t rt | t j jkrtd)| d*t | d+t j jd,t j jd t| | D]/\}}||krtd-|d.|d|d/kr%td-|d0|d1kr2td2|d3qdS)4Nr3rz7`height` and `width` have to be divisible by 8 but are z and rXc3s|]}|jvVqdSN_callback_tensor_inputs.0krDrGrH s  z=AnimateDiffControlNetPipeline.check_inputs..z2`callback_on_step_end_tensor_inputs` has to be in z , but found csg|] }|jvr|qSrGrrrrGrH sz>AnimateDiffControlNetPipeline.check_inputs..zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z:`prompt` has to be of type `str`, `list` or `dict` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` z You have z! ControlNets and you have passed z= prompts. The conditionings will be fixed across the prompts.scaled_dot_product_attentionz>For single controlnet, `image` must be of type `list` but got zExcepted image to have length z but got len(video)=zQFor multiple controlnets: `image` must be type list of lists but got type(video)=z$Expected length of image sublist as z but got len(video[0])=c3s$|] }t|tdkVqdS)rN)r?)rimg)rrGrHr;s"zDAll conditioning frame batches for multicontrolnet must be same sizeFzLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.css|]}t|tVqdSr)r9r;)rrrGrGrHrNszEA single batch of multiple conditionings are supported at the moment.zFor multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have the same length as the number of controlnetsz`control_guidance_start` has z* elements, but `control_guidance_end` has zI elements. Make sure to provide the same number of elements to each list.z`control_guidance_start`: z has z elements but there are z- controlnets available. Make sure to provide zcontrol guidance start: z4 cannot be larger or equal to control guidance end: rz can't be smaller than 0.rzcontrol guidance end: z can't be larger than 1.0.)rlallrr9rZr;dictrjr[r1rrbrcr?netsrdFr__dynamo eval_frameOptimizedModuler _orig_modrkanyrr<r)rDrmrrrrpr*r+"callback_on_step_end_tensor_inputsrcontrolnet_conditioning_scalecontrol_guidance_startcontrol_guidance_end is_compiledstartendrG)rDrrH check_inputss                 0   z*AnimateDiffControlNetPipeline.check_inputsc Cs|jr|||||||||| } t|tr(t||kr(tdt|d|d|||||j||jf} | durBt| |||d} n| |} | |j j } | S)Nz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)rrVrU) free_noise_enabled_prepare_latents_free_noiser9r;r?rlr4rrer2init_noise_sigma) rDrqnum_channels_latentsrrrrUrVrr)r[rGrGrHprepare_latentsvs*   z-AnimateDiffControlNetPipeline.prepare_latentsFc Cs|jj|||djtjd}|ddddddd}|jd} | dkr'|} n|} |j| dd}|j||d }|rC| sCt |gd}|S) N)rr)rUrrr$rrrr}) rCpreprocess_videorer_float32rflattenr[rr) rDrrrrqnum_videos_per_promptrVrUro guess_modevideo_batch_size repeat_byrGrGrH prepare_videos  z+AnimateDiffControlNetPipeline.prepare_videocC|jSr_guidance_scalerrGrGrHguidance_scalez,AnimateDiffControlNetPipeline.guidance_scalecCrr) _clip_skiprrGrGrHrJrz'AnimateDiffControlNetPipeline.clip_skipcCs |jdkS)Nr$rrrGrGrHros z9AnimateDiffControlNetPipeline.do_classifier_free_guidancecCrr)_cross_attention_kwargsrrGrGrHcross_attention_kwargsrz4AnimateDiffControlNetPipeline.cross_attention_kwargscCrr)_num_timestepsrrGrGrH num_timestepsrz+AnimateDiffControlNetPipeline.num_timestepscCrr) _interruptrrGrGrH interruptrz'AnimateDiffControlNetPipeline.interrupt2g@r$pilTr)rmrrrnum_inference_stepsrrprrrrrconditioning_frames output_type return_dictrrrrrcallback_on_step_endrc?st|jr |jjn|j}t|tst|trt||g}n3t|ts0t|tr0t||g}n!t|tsQt|tsQt|trDt|jnd}||g||g}}|pZ|jj j |j }|pd|jj j |j }d}|j ||||||| | ||||d ||_ ||_||_d|_|durt|ttfrd}n|durt|trt|}n| jd}|j}t|trt|tr|gt|j}t|tr|j jn|jdj j} |p| }|dur|ddnd}!|jr|j|||||j|| | |!|jd \} } n$|j||||j|| | |!|jd \} } |jrt| | g} | j |dd } |dus|dur+|!||||||j}"t|trF|j"|||||||||j#|j|d }n,t|trpg}#|D]}$|j"|$||||||||j#|j|d }%|#$|%qP|#}nJ|j%j&||d |j%j'|jj j(}&|)|||&|||| j#|| | } |*| | }'|dus|durd |"ind}(g})t+tD]fd dt,||D}*|)$t|tr|*dn|*q|j-r|j.nd}+t+|+D]M},|j-r|/| |,||| j#| \} t|_0t||j%j1}-|j2|j0d}.t3D] \}/|j4rq|jr%t| gdn| }0|j%5|0|/}0|rF|jrF| }1|j%5|1|/}1| 6dd}2n|0}1| }2t|)tr_ddt,||)D}3n|}4t|4trk|4d}4|4|)}3t7|1dd}1|18d|1jd|1jd|1jdf}1|j|1|/|2||3|dd\}5}6|j|0|/| |j9|(|5|6dj:}7|jr|76d\}8}9|8||9|8}7|j%j;|7|/| fi|'j<} |duri}:|D] };t=|;|:|;<q|||/|:}<|<>d| } |<>d| } |<>d| } tdksd|-krd|j%j1dkr|.?t@rtABqWdn 1s&wYq|dkr5| }=n|C| |}>|jDjE|>|d}=|F|sM|=fStG|=dS)u The call function to the pipeline for generation. Args: prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The width in pixels of the generated video. num_frames (`int`, *optional*, defaults to 16): The number of video frames that are generated. Defaults to 16 frames which at 8 frames per seconds amounts to 2 seconds of video. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor is generated by sampling using the supplied random `generator`. Latents should be of shape `(batch_size, num_channel, num_frames, height, width)`. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, text embeddings are generated from the `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. conditioning_frames (`list[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If multiple ControlNets are specified, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated video. Choose between `torch.Tensor`, `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead of a plain tuple. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. Examples: Returns: [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. r$) rmrrrrprr*r+rrrrFNrscale) rmrrVrrorpr*r+rIrJ)r*r+rIrJ)repeatsr) rrrrqrrVrUrorrrcs<g|]\}}dtt|kpdt|kqS)rr$)rr?)rser timestepsrGrHrs*z:AnimateDiffControlNetPipeline.__call__..)totalrcSsg|]\}}||qSrGrG)rcrrGrGrHrsrQrr)encoder_hidden_statescontrolnet_condconditioning_scalerr)rradded_cond_kwargsdown_block_additional_residualsmid_block_additional_residualr)r*r+latent)rr)frames)Hrr1rr9r;r?rrr/r@ sample_sizer4rrrrrrZrr[_execution_devicerrglobal_pool_conditionsgetr_encode_prompt_free_noiserorJr|r_rrrrrUrr2 set_timestepsr in_channelsrrrrfree_init_enabled_free_init_num_iters_apply_free_initrorder progress_barrrscale_model_inputr transposerrrr prev_samplelocalspopupdate XLA_AVAILABLExm mark_steprrBpostprocess_videomaybe_free_model_hooksr%)?rDrmrrrrrrprrrr)r*r+rrrrrrrrrrrJrrrr1multrqrVr text_encoder_lora_scalercond_prepared_videosframe_prepared_videorrrcontrolnet_keepkeepsnum_free_init_itersfree_init_iternum_warmup_stepsrtlatent_model_inputcontrol_model_inputcontrolnet_prompt_embeds cond_scalecontrolnet_cond_scaledown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textcallback_kwargsrcallback_outputsr video_tensorrGrrH__call__sx              "          6 N  z&AnimateDiffControlNetPipeline.__call__)NN)NNNNNr)r)NNNNNrrr)FF)2__name__ __module__ __qualname____doc__model_cpu_offload_seq_optional_componentsrrrrrrrrr;r<rrrrr8r_rrintr|rrrrrrrpropertyrrJrorrrno_gradrZ Generatorr boolrrrr9 __classcell__rGrGrErHr&xs6    +   8.  )                 r&)@rtypingrrr_torch.nn.functionalnn functionalr transformersrrrrimage_processorr loadersr r r r modelsrrrrrr models.lorarmodels.unets.unet_motion_modelr schedulersrutilsrrrrrutils.torch_utilsrrrBrfree_init_utilsr free_noise_utilsr!pipeline_utilsr"r#pipeline_outputr%torch_xla.core.xla_modelcore xla_modelrr get_loggerr:rbEXAMPLE_DOC_STRINGr&rGrGrGrHsD           B