o pi @s`ddlZddlmZmZmZmZmZmZmZddl Z ddl m m Z ddlmZmZmZmZddlmZddlmZmZmZddlmZmZmZmZmZddlm Z dd l!m"Z"dd l#m$Z$dd l%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+m,Z,dd l-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6m7Z7ddl8m9Z9e':e;ZdS)N)AnyCallableDictListOptionalTupleUnion)CLIPImageProcessor CLIPTextModel CLIPTokenizerCLIPVisionModelWithProjection)PipelineImageInput)IPAdapterMixinStableDiffusionLoraLoaderMixinTextualInversionLoaderMixin) AutoencoderKLControlNetModelImageProjectionUNet2DConditionModelUNetMotionModel)adjust_lora_scale_text_encoder) MotionAdapter)KarrasDiffusionSchedulers)USE_PEFT_BACKENDloggingscale_lora_layersunscale_lora_layers)is_compiled_module randn_tensor)VideoProcessor)MultiControlNetModel) FreeInitMixin)AnimateDiffFreeNoiseMixin)DiffusionPipelineStableDiffusionMixin)AnimateDiffPipelineOutputaT Examples: ```py >>> import torch >>> from diffusers import ( ... AnimateDiffControlNetPipeline, ... AutoencoderKL, ... ControlNetModel, ... MotionAdapter, ... LCMScheduler, ... ) >>> from diffusers.utils import export_to_gif, load_video >>> # Additionally, you will need a preprocess videos before they can be used with the ControlNet >>> # HF maintains just the right package for it: `pip install controlnet_aux` >>> from controlnet_aux.processor import ZoeDetector >>> # Download controlnets from https://huggingface.co/lllyasviel/ControlNet-v1-1 to use .from_single_file >>> # Download Diffusers-format controlnets, such as https://huggingface.co/lllyasviel/sd-controlnet-depth, to use .from_pretrained() >>> controlnet = ControlNetModel.from_single_file("control_v11f1p_sd15_depth.pth", torch_dtype=torch.float16) >>> # We use AnimateLCM for this example but one can use the original motion adapters as well (for example, https://huggingface.co/guoyww/animatediff-motion-adapter-v1-5-3) >>> motion_adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM") >>> vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16) >>> pipe: AnimateDiffControlNetPipeline = AnimateDiffControlNetPipeline.from_pretrained( ... "SG161222/Realistic_Vision_V5.1_noVAE", ... motion_adapter=motion_adapter, ... controlnet=controlnet, ... vae=vae, ... ).to(device="cuda", dtype=torch.float16) >>> pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear") >>> pipe.load_lora_weights( ... "wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora" ... ) >>> pipe.set_adapters(["lcm-lora"], [0.8]) >>> depth_detector = ZoeDetector.from_pretrained("lllyasviel/Annotators").to("cuda") >>> video = load_video( ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-vid2vid-input-1.gif" ... ) >>> conditioning_frames = [] >>> with pipe.progress_bar(total=len(video)) as progress_bar: ... for frame in video: ... conditioning_frames.append(depth_detector(frame)) ... progress_bar.update() >>> prompt = "a panda, playing a guitar, sitting in a pink boat, in the ocean, mountains in background, realistic, high quality" >>> negative_prompt = "bad quality, worst quality" >>> video = pipe( ... prompt=prompt, ... negative_prompt=negative_prompt, ... num_frames=len(video), ... num_inference_steps=10, ... guidance_scale=2.0, ... conditioning_frames=conditioning_frames, ... generator=torch.Generator().manual_seed(42), ... ).frames[0] >>> export_to_gif(video, "animatediff_controlnet.gif", fps=8) ``` c8seZdZdZdZddgZgdZ  dQdeded e d e e e fd e d e eeeeeefd edeedeeffdd Z     dRdeejdeejdeedeefddZdSddZddZdTdefddZddZ      ! dUd"d#Z dSd$d%Z! & &dVd'd(Z"e#d)d*Z$e#d+d,Z%e#d-d.Z&e#d/d0Z'e#d1d2Z(e)ddddd3d4dd5d!dddddddd6d7dd d&d!d ddd8gdfd9e e*ee*fd:eed;eeded?ee e*ee*fd@eedAedBee ej+eej+fd8eejdeejdeejdCee,dDee,dEeee,dFee*dGe-dHee.e*e/fdIe eeefdJe-dKe eeefdLe eeefdeedMee0eee.gdfdNee*def6dOdPZ1Z2S)WAnimateDiffControlNetPipelinea Pipeline for text-to-video generation with ControlNet guidance. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). The pipeline also inherits the following loading methods: - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. text_encoder ([`CLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). tokenizer (`CLIPTokenizer`): A [`~transformers.CLIPTokenizer`] to tokenize text. unet ([`UNet2DConditionModel`]): A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. motion_adapter ([`MotionAdapter`]): A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. ztext_encoder->unet->vaefeature_extractor image_encoder)latents prompt_embedsnegative_prompt_embedsNvae text_encoder tokenizerunetmotion_adapter controlnet schedulerc stt|trt||}t|ttfrt|}|j ||||||||| d dt |j j j d|_t|jd|_t|jddd|_dS)N) r/r0r1r2r3r4r5r*r+r!r')vae_scale_factorTF)r6do_convert_rgb do_normalize)super__init__ isinstancerr from_unet2dlisttupler"register_moduleslenr/configblock_out_channelsr6r video_processorcontrol_video_processor) selfr/r0r1r2r3r4r5r*r+ __class__}/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.pyr:s*    z&AnimateDiffControlNetPipeline.__init__r-r. lora_scale clip_skipc  Cs|durt|tr||_tst|j|nt|j||dur't|tr'd} n|dur5t|tr5t |} n|j d} |durt|t rJ| ||j }|j |d|j jddd} | j} |j |ddd j} | j d | j d krt| | s|j | dd|j jdd f}td |j jd |t|jjd r|jjjr| j|}nd}| dur|j| ||d}|d}n|j| ||dd}|d | d }|jj|}|jdur|jj}n |jdur|jj}n|j}|j||d}|j \}}}|d|d}||||d }|r|dur|durdg| }nC|dur8t |t |ur8t!dt |dt |dt|trB|g}n| t |kr\t"d|dt |d|d| d |}t|t rk| ||j }|j d}|j |d|ddd}t|jjd r|jjjr|j|}nd}|j|j||d}|d}|r|j d}|j||d}|d|d}|| ||d }|jdurt|trtrt#|j|||fS)a Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `List[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. lora_scale (`float`, *optional*): A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. Nr'r max_lengthTpt)paddingrL truncationreturn_tensorslongest)rNrPz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: use_attention_mask)attention_mask)rToutput_hidden_states)dtypedevicez?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)$r;r _lora_scalerrr0rstrr=r@shapermaybe_convert_promptr1model_max_length input_idstorchequal batch_decodeloggerwarninghasattrrArSrTto text_modelfinal_layer_normrVr2repeatviewtype TypeError ValueErrorr)rEpromptrWnum_images_per_promptdo_classifier_free_guidancenegative_promptr-r.rJrK batch_size text_inputstext_input_idsuntruncated_ids removed_textrTprompt_embeds_dtypebs_embedseq_len_ uncond_tokensrL uncond_inputrHrHrI encode_prompts+                     z+AnimateDiffControlNetPipeline.encode_promptc Cst|jj}t|tjs|j|ddj}|j ||d}|rH|j|ddj d}|j |dd}|jt |ddj d}|j |dd}||fS||j }|j |dd}t |} || fS) NrM)rPrWrVT)rUrdim)nextr+ parametersrVr;r`Tensorr* pixel_valuesrf hidden_statesrepeat_interleave zeros_like image_embeds) rEimagerWrorUrVimage_enc_hidden_statesuncond_image_enc_hidden_statesruncond_image_embedsrHrHrI encode_imageos(    z*AnimateDiffControlNetPipeline.encode_imagecCslg}|rg}|duret|ts|g}t|t|jjjkr/tdt|dt|jjjdt||jjjD],\}} t| t } | ||d| \} } | | dddf|rc| | dddfq7n|D]} |rw| d\} } | | | | qgg}t |D]0\} } t j| g|dd} |rt j|| g|dd} t j| | gdd} | j|d} | | q|S) NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got z images and z IP Adapters.r'r!rrrW)r;r=r@r2encoder_hid_projimage_projection_layersrmziprrappendchunk enumerater`catrf)rEip_adapter_imageip_adapter_image_embedsrWrorprnegative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsirHrHrIprepare_ip_adapter_image_embedssH        z=AnimateDiffControlNetPipeline.prepare_ip_adapter_image_embedsdecode_chunk_sizec Csd|jjj|}|j\}}}}}|ddddd|||||}g}td|jd|D]} || | |} |j| j} | | q-t |}|dddf||df|jddddddd}| }|S)Nr'rr!r rR) r/rAscaling_factorr\permutereshaperangedecodesamplerr`rfloat) rEr,rrrchannels num_framesheightwidthvideor batch_latentsrHrHrIdecode_latentss"  8z,AnimateDiffControlNetPipeline.decode_latentscCsXdtt|jjjv}i}|r||d<dtt|jjjv}|r*||d<|S)Neta generator)setinspect signaturer5steprkeys)rErr accepts_etaextra_step_kwargsaccepts_generatorrHrHrIprepare_extra_step_kwargssz7AnimateDiffControlNetPipeline.prepare_extra_step_kwargs?c  sp|ddks |ddkrtd|d|d|dur8tfdd|Ds8tdjd fd d |D|durK|durKtd |d |d|durW|durWtd|durnt|tsnt|tsntdt||dur|durtd|d|d|dur|dur|j|jkrtd|jd|jdtjt rt|trt dt jj dt |dttdotjtjjj} tjts| rtjjtrttstdtt |krtd|dt nStjt s | rHtjjt rHttrtdts tdtt d|kr7td|dt dtfddDrGtd nJtjts\| rgtjjtrgt| tsftd"n?tjt sy| rtjjt rt| trtd#d| Drtd$nt| trt | t jj krtd%nJt| ttfs| g} t| ttfs| g} t | t | krtd&t | d't | d(tjt rt | t jj krtd)| d*t | d+t jj d,t jj d t| | D]/\}}||krtd-|d.|d|d/kr'td-|d0|d1kr4td2|d3qdS)4Nrz7`height` and `width` have to be divisible by 8 but are z and rYc3s|]}|jvVqdSN_callback_tensor_inputs.0krErHrI s  z=AnimateDiffControlNetPipeline.check_inputs..z2`callback_on_step_end_tensor_inputs` has to be in z , but found csg|] }|jvr|qSrHrrrrHrI sz>AnimateDiffControlNetPipeline.check_inputs..zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z2`prompt` has to be of type `str` or `list` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` z You have z! ControlNets and you have passed z= prompts. The conditionings will be fixed across the prompts.scaled_dot_product_attentionz>For single controlnet, `image` must be of type `list` but got zExcepted image to have length z but got len(video)=zQFor multiple controlnets: `image` must be type list of lists but got type(video)=z$Expected length of image sublist as z but got len(video[0])=c3s$|] }t|tdkVqdS)rN)r@)rimg)rrHrIr,s"zDAll conditioning frame batches for multicontrolnet must be same sizeFzLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.css|]}t|tVqdSr)r;r=)rrrHrHrIr?szEA single batch of multiple conditionings are supported at the moment.zFor multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have the same length as the number of controlnetsz`control_guidance_start` has z* elements, but `control_guidance_end` has zI elements. Make sure to provide the same number of elements to each list.z`control_guidance_start`: z has z elements but there are z- controlnets available. Make sure to provide zcontrol guidance start: z4 cannot be larger or equal to control guidance end: rz can't be smaller than 0.rzcontrol guidance end: z can't be larger than 1.0.)rmallrr;r[r=rkr\r4r"rcrdr@netsreFr`_dynamo eval_frameOptimizedModuler _orig_modrlanyrr>r)rErnrrrrqr-r."callback_on_step_end_tensor_inputsrcontrolnet_conditioning_scalecontrol_guidance_startcontrol_guidance_end is_compiledstartendrH)rErrI check_inputss                 0   z*AnimateDiffControlNetPipeline.check_inputsc Cs|jr|||||||||| } t|tr(t||kr(tdt|d|d|||||j||jf} | durBt| |||d} n| |} | |j j } | S)Nz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)rrWrV) free_noise_enabled_prepare_latents_free_noiser;r=r@rmr6rrfr5init_noise_sigma) rErrnum_channels_latentsrrrrVrWrr,r\rHrHrIprepare_latentsgs*   z-AnimateDiffControlNetPipeline.prepare_latentsFc Cs|jj|||djtjd}|ddddddd}|jd} | dkr'|} n|} |j| dd}|j||d }|rC| sCt |gd}|S) N)rr)rVrr!r'r rrr~) rDpreprocess_videorfr`float32rflattenr\rr) rErrrrrnum_videos_per_promptrWrVrp guess_modevideo_batch_size repeat_byrHrHrI prepare_videos  z+AnimateDiffControlNetPipeline.prepare_videocC|jSr_guidance_scalerrHrHrIguidance_scalez,AnimateDiffControlNetPipeline.guidance_scalecCrr) _clip_skiprrHrHrIrKrz'AnimateDiffControlNetPipeline.clip_skipcCs |jdkS)Nr'rrrHrHrIrps z9AnimateDiffControlNetPipeline.do_classifier_free_guidancecCrr)_cross_attention_kwargsrrHrHrIcross_attention_kwargsrz4AnimateDiffControlNetPipeline.cross_attention_kwargscCrr)_num_timestepsrrHrHrI num_timestepsrz+AnimateDiffControlNetPipeline.num_timesteps2g@r'pilTr,rnrrrnum_inference_stepsrrqrrrrrconditioning_frames output_type return_dictrrrrrcallback_on_step_endrc?sNt|jr |jjn|j}t|tst|trt||g}n3t|ts0t|tr0t||g}n!t|tsQt|tsQt|trDt|jnd}||g||g}}|pZ|jj j |j }|pd|jj j |j }d}|j ||||||| | ||||d ||_ ||_||_|durt|trd}n|durt|trt|}n| jd}|j}t|trt|tr|gt|j}t|tr|j jn|jdj j} |p| }|dur|ddnd}!|j||||j|| | |!|jd \} } |jrt| | g} |dus|dur|||||||j}"t|tr!|j|||||||||j|j|d }n,t|trKg}#|D]}$|j|$||||||||j|j|d }%|#|%q+|#}nJ|j j!||d |j j"|jj j#}&|$|||&|||| j|| | } |%| | }'|dus~|durd |"ind}(g})t&tD]fd d t'||D}*|)t|tr|*dn|*q|j(r|j)nd}+t&|+D]G},|j(r|*| |,||| j| \} t|_+t||j j,}-|j-|j+d }.t.D]\}/|jrt| gdn| }0|j /|0|/}0|r|jr| }1|j /|1|/}1| 0dd}2n|0}1| }2|2j1|dd}2t|)tr;dd t'||)D}3n|}4t|4trG|4d}4|4|)}3t2|1dd}1|13d|1jd|1jd|1jdf}1|j|1|/|2||3|dd\}5}6|j|0|/| |j4|(|5|6dj5}7|jr|70d\}8}9|8||9|8}7|j j6|7|/| fi|'j7} |duri}:|D] };t8|;|:|;<q|||/|:}<|<9d| } |<9d| } |<9d| } tdksd|-krd|j j,dkr|.:qWdn 1swYq|dkr | }=n|;| |}>|j|d}=|>|s"|=fSt?|=dS)u The call function to the pipeline for generation. Args: prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The width in pixels of the generated video. num_frames (`int`, *optional*, defaults to 16): The number of video frames that are generated. Defaults to 16 frames which at 8 frames per seconds amounts to 2 seconds of video. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor is generated by sampling using the supplied random `generator`. Latents should be of shape `(batch_size, num_channel, num_frames, height, width)`. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, text embeddings are generated from the `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. conditioning_frames (`List[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If multiple ControlNets are specified, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated video. Choose between `torch.Tensor`, `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead of a plain tuple. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`List`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. Examples: Returns: [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. r') rnrrrrqrr-r.rrrrNrscale)r-r.rJrK) rrrrrrrWrVrprFrrcs<g|]\}}dtt|kpdt|kqS)rr')rr@)rser timestepsrHrIrs*z:AnimateDiffControlNetPipeline.__call__..)totalr!rcSsg|]\}}||qSrHrH)rcrrHrHrIrsrRr r)encoder_hidden_statescontrolnet_condconditioning_scalerr)rradded_cond_kwargsdown_block_additional_residualsmid_block_additional_residualr,r-r.latent)rr)frames)@rr4rr;r=r@r"rr2rA sample_sizer6rrrrr[r\_execution_devicerrglobal_pool_conditionsgetr}rprKr`rrrrVrr5 set_timestepsr in_channelsrrrrfree_init_enabled_free_init_num_iters_apply_free_initrorder progress_barrscale_model_inputrr transposerrrr prev_samplelocalspopupdaterrCpostprocess_videomaybe_free_model_hooksr()?rErnrrrrrrqrrrr,r-r.rrrrrrrrrrrKrrrr4multrrrWr text_encoder_lora_scalercond_prepared_videosframe_prepared_videorrrcontrolnet_keepkeepsnum_free_init_itersfree_init_iternum_warmup_stepsrtlatent_model_inputcontrol_model_inputcontrolnet_prompt_embeds cond_scalecontrolnet_cond_scaledown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textcallback_kwargsrcallback_outputsr video_tensorrHrrI__call__sx            "          6 I  z&AnimateDiffControlNetPipeline.__call__)NN)NNNNNr)r)NNNNNrrr)FF)3__name__ __module__ __qualname____doc__model_cpu_offload_seq_optional_componentsrrr r rrrrrrrr"rrr r r:r`rrintr}rrrrrrrpropertyrrKrprrno_gradr[ Generatorrboolrrrr3 __classcell__rHrHrFrIr)js2     +   8.  )            r))?rtypingrrrrrrrr`torch.nn.functionalnn functionalr transformersr r r r image_processorrloadersrrrmodelsrrrrr models.lorarmodels.unets.unet_motion_modelr schedulersrutilsrrrrutils.torch_utilsrrrCr controlnet.multicontrolnetr"free_init_utilsr#free_noise_utilsr$pipeline_utilsr%r&pipeline_outputr( get_loggerr4rcEXAMPLE_DOC_STRINGr)rHrHrHrIs<$           B