o GiPA @sddlZddlmZmZddlZddlZddlZddl m m Z ddl mZmZmZmZmZddlmZddlmZmZddlmZmZddlmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0dd l1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7ddl8m9Z9erddl:m;Z;ddl*mm?Z@dZAndZAe-BeCZDdZE    ddeFdBdeGejHBdBdeIeFdBdeIeJdBfddZKGddde6e7eeeeZLdS)N)AnyCallable)CLIPImageProcessor CLIPTextModelCLIPTextModelWithProjection CLIPTokenizerCLIPVisionModelWithProjection) is_invisible_watermark_available)MultiPipelineCallbacksPipelineCallback)PipelineImageInputVaeImageProcessor)FromSingleFileMixinIPAdapterMixin StableDiffusionXLLoraLoaderMixinTextualInversionLoaderMixin) AutoencoderKLControlNetModelImageProjectionMultiControlNetModelUNet2DConditionModel)adjust_lora_scale_text_encoder)KarrasDiffusionSchedulers)USE_PEFT_BACKEND deprecateloggingreplace_example_docstringscale_lora_layersunscale_lora_layers)is_compiled_moduleis_torch_version randn_tensor)DiffusionPipelineStableDiffusionMixin)StableDiffusionXLPipelineOutput)StableDiffusionXLWatermarker)is_torch_xla_availableTFa/ Examples: ```py >>> # !pip install opencv-python transformers accelerate >>> from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL >>> from diffusers.utils import load_image >>> import numpy as np >>> import torch >>> import cv2 >>> from PIL import Image >>> prompt = "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting" >>> negative_prompt = "low quality, bad quality, sketches" >>> # download an image >>> image = load_image( ... "https://hf.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png" ... ) >>> # initialize the models and pipeline >>> controlnet_conditioning_scale = 0.5 # recommended for good generalization >>> controlnet = ControlNetModel.from_pretrained( ... "diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float16 ... ) >>> vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16) >>> pipe = StableDiffusionXLControlNetPipeline.from_pretrained( ... "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet, vae=vae, torch_dtype=torch.float16 ... ) >>> pipe.enable_model_cpu_offload() >>> # get canny image >>> image = np.array(image) >>> image = cv2.Canny(image, 100, 200) >>> image = image[:, :, None] >>> image = np.concatenate([image, image, image], axis=2) >>> canny_image = Image.fromarray(image) >>> # generate image >>> image = pipe( ... prompt, controlnet_conditioning_scale=controlnet_conditioning_scale, image=canny_image ... ).images[0] ``` num_inference_stepsdevice timestepssigmascKs|dur |dur td|dur>dtt|jjv}|s(td|jd|jd ||d||j}t |}||fS|durpdtt|jjv}|sZtd|jd|jd ||d ||j}t |}||fS|j|fd |i||j}||fS) a Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. Args: scheduler (`SchedulerMixin`): The scheduler to get timesteps from. num_inference_steps (`int`): The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. NzYOnly one of `timesteps` or `sigmas` can be passed. Please choose one to set custom valuesr+zThe current scheduler class zx's `set_timesteps` does not support custom timestep schedules. Please check whether you are using the correct scheduler.)r+r*r,zv's `set_timesteps` does not support custom sigmas schedules. Please check whether you are using the correct scheduler.)r,r*r*) ValueErrorsetinspect signature set_timesteps parameterskeys __class__r+len) schedulerr)r*r+r,kwargsaccepts_timesteps accept_sigmasr-r-l/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.pyretrieve_timestepsys2  r<cPseZdZdZdZgdZgdZ    dqdeded e d e d e d e d e e e Bee BeBdedededBdedeffdd Z            drdededBdejdBdedededBdedBdejdBdejdBdejdBd ejdBd!edBd"edBfd#d$Zdsd%d&Zd'd(Zd)d*Z         + , + dtd-d.Zd/d0Z 1 1dud2d3Z!dsd4d5Z" dsd6d7Z#d8d9Z$d:ej%fd;ejdejfd?d@Z'e(dAdBZ)e(dCdDZ*e(dEdFZ+e(dGdHZ,e(dIdJZ-e(dKdLZ.e(dMdNZ/e0e1e2ddddddOddddPdddd,dddddddddQddd+d1d,d+ddRdddRddddSgf&dee eBdee eBdBdTe3dUedBdVedBdWedXe edYe edZedBd[edee eBdBdee eBdBdedBd\ed]ej4e ej4BdBdSejdBdejdBdejdBdejdBd ejdBd^e3dBd_e ejdBd`edBdaedbe5ee6fdBdcee eBddedeee eBdfee eBdgeeefdheeefdieeefdjeeefdBdkeeefdleeefdBd"edBdme7eegdfe8Be9BdBdne efLdodpZ:Z;S)v#StableDiffusionXLControlNetPipelinea Pipeline for text-to-image generation using Stable Diffusion XL with ControlNet guidance. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). The pipeline also inherits the following loading methods: - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. text_encoder ([`~transformers.CLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). text_encoder_2 ([`~transformers.CLIPTextModelWithProjection`]): Second frozen text-encoder ([laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)). tokenizer ([`~transformers.CLIPTokenizer`]): A `CLIPTokenizer` to tokenize text. tokenizer_2 ([`~transformers.CLIPTokenizer`]): A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): Whether the negative prompt embeddings should always be set to 0. Also see the config of `stabilityai/stable-diffusion-xl-base-1-0`. add_watermarker (`bool`, *optional*): Whether to use the [invisible_watermark](https://github.com/ShieldMnt/invisible-watermark/) library to watermark output images. If not defined, it defaults to `True` if the package is installed; otherwise no watermarker is used. z6text_encoder->text_encoder_2->image_encoder->unet->vae) tokenizer tokenizer_2 text_encodertext_encoder_2feature_extractor image_encoder)latents prompt_embedsnegative_prompt_embedsadd_text_embeds add_time_idsnegative_pooled_prompt_embedsnegative_add_time_idsimageTNvaer@rAr>r?unet controlnetr7force_zeros_for_empty_promptadd_watermarkerrBrCc stt|ttfrt|}|j||||||||| | d t|ddr0dt|j j j dnd|_ t |j dd|_t |j ddd |_| durJ| nt} | rTt|_nd|_|j| d dS) N) rLr@rAr>r?rMrNr7rBrCrLr#T)vae_scale_factordo_convert_rgbF)rSrT do_normalize)rO)super__init__ isinstancelisttuplerregister_modulesgetattrr6rLconfigblock_out_channelsrSrimage_processorcontrol_image_processorr r' watermarkregister_to_config) selfrLr@rAr>r?rMrNr7rOrPrBrCr5r-r;rWs2 (  z,StableDiffusionXLControlNetPipeline.__init__rQpromptprompt_2r*num_images_per_promptdo_classifier_free_guidancenegative_promptnegative_prompt_2rErFpooled_prompt_embedsrI lora_scale clip_skipc! Cs:|p|j}| dur9t|tr9| |_|jdur%tst|j| nt|j| |jdur9ts3t|j| nt|j| t|t rA|gn|}|durLt |}n|j d}|j dur\|j |j gn|j g}|jdurk|j|jgn|jg}|dur|pw|}t|t r|gn|}g}||g}t|||D]\}}}t|tr|||}||d|jddd}|j}||dddj}|j d |j d krt||s||dd|jd d f}td |jd ||||dd }| dur|djdkr|d} | dur|jd}n|j| d }||qtj|d d}|duo|jj}|r6| dur6|r6t|} t| } n|r| dur|pBd}|pG|}t|t rS||gn|}t|t r`||gn|}|durt |t |urt!dt |dt |d|t |krt"d|dt |d|d|d ||g}g}t|||D]E\}}}t|tr|||}|j d }||d|ddd}||j|dd } | dur| djdkr| d} | jd} || qtj|d d} |jdur|j|jj#|d}n |j|j$j#|d}|j \}}} |%d |d }|&|||d }|rR| j d }|jdur9| j|jj#|d} n | j|j$j#|d} | %d |d } | &|||d } | %d |&||d } |rm| %d |&||d } |jdurt|trtrt'|j| |jdurt|trtrt'|j| || | | fS)a\ Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `list[str]`, *optional*): prompt to be encoded prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): torch device num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. pooled_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, pooled text embeddings will be generated from `prompt` input argument. negative_pooled_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. lora_scale (`float`, *optional*): A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. Nr max_lengthTpt)paddingrn truncationreturn_tensorslongest)rprrrQz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: output_hidden_statesr#dimz?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)dtyper*)(_execution_devicerXr _lora_scaler@rrrrAstrr6shaper>r?ziprmaybe_convert_promptmodel_max_length input_idstorchequal batch_decodeloggerwarningtondim hidden_statesappendconcatr]rO zeros_liketype TypeErrorr.r|rMrepeatviewr)!rcrerfr*rgrhrirjrErFrkrIrlrm batch_size tokenizers text_encodersprompt_embeds_listpromptsr>r@ text_inputstext_input_idsuntruncated_ids removed_textzero_out_negative_prompt uncond_tokensnegative_prompt_embeds_listrn uncond_inputbs_embedseq_len_r-r-r; encode_prompt+s :                                  z1StableDiffusionXLControlNetPipeline.encode_promptc Cst|jj}t|tjs|j|ddj}|j ||d}|rH|j|ddj d}|j |dd}|jt |ddj d}|j |dd}||fS||j }|j |dd}t |} || fS) Nro)rrr*r|Trurwrrx)nextrCr3r|rXrTensorrB pixel_valuesrrrepeat_interleaver image_embeds) rcrKr*rgrvr|image_enc_hidden_statesuncond_image_enc_hidden_statesruncond_image_embedsr-r-r; encode_images(    z0StableDiffusionXLControlNetPipeline.encode_imagecCslg}|rg}|duret|ts|g}t|t|jjjkr/tdt|dt|jjjdt||jjjD],\}} t| t } | ||d| \} } | | dddf|rc| | dddfq7n|D]} |rw| d\} } | | | | qgg}t |D]0\} } t j| g|dd} |rt j|| g|dd} t j| | gdd} | j|d} | | q|S) NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got images and z IP Adapters.rQr#rrx)r*)rXrYr6rMencoder_hid_projimage_projection_layersr.rrrrchunk enumeratercatr)rcip_adapter_imageip_adapter_image_embedsr*rgrhrnegative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsir-r-r;prepare_ip_adapter_image_embeds3sH        zCStableDiffusionXLControlNetPipeline.prepare_ip_adapter_image_embedscCsXdtt|jjjv}i}|r||d<dtt|jjjv}|r*||d<|S)Neta generator)r/r0r1r7stepr3r4)rcrr accepts_etaextra_step_kwargsaccepts_generatorr-r-r;prepare_extra_step_kwargsasz=StableDiffusionXLControlNetPipeline.prepare_extra_step_kwargs?c sn|durt|tr |dkrtd|dt|d|dur;tfdd|Ds;tdjdfd d |D|durN|durNtd |d |d |dura|duratd|d |d |durm|durmtd|durt|tst|tstdt||durt|tst|tstdt||dur|durtd|d|d |dur|durtd|d|d |dur|dur|j|jkrtd|jd|jd|dur| durtd|dur| durtdtj t rt|trt dt j jdt |dttdo!tj tjjj}tj ts4|rs  zCStableDiffusionXLControlNetPipeline.check_inputs..z2`callback_on_step_end_tensor_inputs` has to be in z , but found csg|] }|jvr|qSr-rrrr-r; szDStableDiffusionXLControlNetPipeline.check_inputs..zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.z Cannot forward both `prompt_2`: zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z2`prompt` has to be of type `str` or `list` but is z4`prompt_2` has to be of type `str` or `list` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: z)Cannot forward both `negative_prompt_2`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` zIf `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`.zIf `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`.z You have z! ControlNets and you have passed z= prompts. The conditionings will be fixed across the prompts.scaled_dot_product_attentionz5For multiple controlnets: `image` must be type `list`cs|]}t|tVqdSrrXrYrrr-r-r;rzEA single batch of multiple conditionings are supported at the moment.zbFor multiple controlnets: `image` must have the same length as the number of controlnets, but got rz ControlNets.FzLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.csrrrrr-r-r;rrzFor multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have the same length as the number of controlnetsz`control_guidance_start` has z* elements, but `control_guidance_end` has zI elements. Make sure to provide the same number of elements to each list.z`control_guidance_start`: z has z elements but there are z- controlnets available. Make sure to provide zcontrol guidance start: z4 cannot be larger or equal to control guidance end: rz can't be smaller than 0.rzcontrol guidance end: z can't be larger than 1.0.zProvide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined.z:`ip_adapter_image_embeds` has to be of type `list` but is )r zF`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is D)rXintr.rallrrrYrrNrrrr6netshasattrFr_dynamo eval_frameOptimizedModuler _orig_mod check_imageranyfloatrZrr)rcrerfrKcallback_stepsrirjrErFrkrrrIcontrolnet_conditioning_scalecontrol_guidance_startcontrol_guidance_end"callback_on_step_end_tensor_inputs is_compiledimage_startendr-rr; check_inputsrs6              0      z0StableDiffusionXLControlNetPipeline.check_inputsc Cs$t|tjj}t|tj}t|tj}t|to t|dtjj}t|to-t|dtj}t|to:t|dtj} |sP|sP|sP|sP|sP| sPtdt ||rUd} nt |} |duret|t red} n|durst|trst |} n |dur||j d} | dkr| | krt d| d| dSdS)Nrzimage must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is rQzdIf image batch size is not 1, image batch size must be same as prompt batch size. image batch size: z, prompt batch size: )rXPILImagerrnpndarrayrYrrr6rrr.) rcrKrerE image_is_pilimage_is_tensor image_is_npimage_is_pil_listimage_is_tensor_listimage_is_np_listimage_batch_sizeprompt_batch_sizer-r-r;r0sF     z/StableDiffusionXLControlNetPipeline.check_imageFc Csp|jj|||djtjd}|jd} | dkr|} n|} |j| dd}|j||d}|r6| s6t|gd}|S)N)heightwidthr|rrQrxrr#)r` preprocessrrfloat32rrr) rcrKrrrrgr*r|rh guess_moder repeat_byr-r-r; prepare_imageVs z1StableDiffusionXLControlNetPipeline.prepare_imagec Cs||t||jt||jf} t|tr(t||kr(tdt|d|d|dur5t| |||d}n||}||jj }|S)Nz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)rr*r|) rrSrXrYr6r.r"rr7init_noise_sigma) rcrnum_channels_latentsrrr|r*rrDrr-r-r;prepare_latentsus      z3StableDiffusionXLControlNetPipeline.prepare_latentsc Csdt|||}|jjjt||}|jjjj}||kr(td|d|dt j |g|d}|S)Nz7Model expects an added time embedding vector of length z, but a vector of z was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`.r) rYrMr]addition_time_embed_dimr6 add_embeddinglinear_1 in_featuresr.rtensor) rc original_sizecrops_coords_top_left target_sizer|text_encoder_projection_dimrHpassed_add_embed_dimexpected_add_embed_dimr-r-r;_get_add_time_idss z5StableDiffusionXLControlNetPipeline._get_add_time_idscCs tddd|jjtjddS)N upcast_vae1.0.0z`upcast_vae` is deprecated. Please use `pipe.vae.to(torch.float32)`. For more details, please refer to: https://github.com/huggingface/diffusers/pull/12619#issue-3606633695.r)rrLrrrrr-r-r;rs z.StableDiffusionXLControlNetPipeline.upcast_vaeiw embedding_dimr|returncCst|jdks J|d}|d}ttd|d}ttj||d| }||dddf|dddf}tjt |t |gdd}|ddkrZtj j |d}|j|jd |fksfJ|S) a See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 Args: w (`torch.Tensor`): Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings. embedding_dim (`int`, *optional*, defaults to 512): Dimension of the embeddings to generate. dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): Data type of the generated embeddings. Returns: `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`. rQg@@r#g@rNrx)rrQr)r6rrlogr exparangerrsincosnn functionalpad)rcrrr|half_dimembr-r-r;get_guidance_scale_embeddings& z@StableDiffusionXLControlNetPipeline.get_guidance_scale_embeddingcC|jSr)_guidance_scalerr-r-r;guidance_scalez2StableDiffusionXLControlNetPipeline.guidance_scalecCr!r) _clip_skiprr-r-r;rmr$z-StableDiffusionXLControlNetPipeline.clip_skipcCs|jdko |jjjduS)NrQ)r"rMr]time_cond_proj_dimrr-r-r;rhsz?StableDiffusionXLControlNetPipeline.do_classifier_free_guidancecCr!r)_cross_attention_kwargsrr-r-r;cross_attention_kwargsr$z:StableDiffusionXLControlNetPipeline.cross_attention_kwargscCr!r)_denoising_endrr-r-r; denoising_endr$z1StableDiffusionXLControlNetPipeline.denoising_endcCr!r)_num_timestepsrr-r-r; num_timestepsr$z1StableDiffusionXLControlNetPipeline.num_timestepscCr!r) _interruptrr-r-r; interruptr$z-StableDiffusionXLControlNetPipeline.interrupt2g@pil)rrrDrKrrr)r+r,r*r#rrrr output_type return_dictr(rrrrr r r negative_original_sizenegative_crops_coords_top_leftnegative_target_sizecallback_on_step_endrc'Y s |'dd}(|'dd})|(durtddd|)dur tdddt|%ttfr*|%j}&t|jr3|jjn|j}*t|t sHt|t rHt ||g}n3t|t sZt|t rZt ||g}n!t|t s{t|t s{t|*t rnt |*j nd}+|+|g|+|g}}| ||||)| | ||||||||||&| |_|$|_||_| |_d|_|durt|trd},n|durt|t rt |},n|jd },|j}-t|*t rt|tr|gt |*j }t|*tr|*jjn|*j d jj}.|p|.}|jdur|jd dnd}/|j|||-| |j| | |||||/|jd \}}}}|dus|dur"||||-|,| |j}0t|*trD|j ||||,| | |-|*j!|j|d }|jd d\}}n5t|*t rwg}1|D]}2|j |2|||,| | |-|*j!|j|d }2|1"|2qN|1}|d jd d\}}nJt#rd}3n|-}3t$|j%||3|\}t |_&|j'jj(}4|)|,| |4|||j!|-||}d}5|j'jj*durt+,|j-d.|,| }6|j/|6|j'jj*dj0|-|j!d}5|1||}7g}8t2t D]fddt3||D}9|8"t|*tr|9d n|9qt|t r |p |d jd d}n |p|jd d}| p||f} |}:|j4dur.t5|jd};n|j4jj6};|j7||| |j!|;d}<|!durT|#durT|j7|!|"|#|j!|;d}=n|<}=|jrut+j8||gd d}t+j8||:gd d}:t+j8|=||j:durt|j:tr|j:d kr|j:dkrt5t;|j%jj<|j:|j%jj<t t t=fdd}d|t|j'}?t|j}@t>dd}A|j?|d}Bt@D]\}C|jArqt+jBCr |?r |@r |Ar t+jDE|jrt+8|gdn|}D|j%F|D|C}D|:|krid|j%j9d kri|BK|(duri|)d kritL|j%d+d}S|(|S|C|t#rptMNqWdn 1s}wY|d,ks|jOj!t+jPko|jOjjQ}T|Tr|R|0tStT|jOjUVj!}tW|jOjd-o|jOjjXdu}UtW|jOjd.o|jOjjYdu}V|Ur|Vrt+,|jOjjXZdd/dd0|j[|j!}Wt+,|jOjjYZdd/dd0|j[|j!}X||X|jOjj\|W}n||jOjj\}|jOj]|dd0d }|Tr|jOj0t+jPd1n|}|d,ks9|j^dur1|j^_|}|j`ja||d2}|b|sC|fStc|d3S)4uZ5 The call function to the pipeline for generation. Args: prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders. image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. Anything below 512 pixels won't work well for [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) and checkpoints that are not specifically fine-tuned on low resolutions. width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The width in pixels of the generated image. Anything below 512 pixels won't work well for [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) and checkpoints that are not specifically fine-tuned on low resolutions. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. denoising_end (`float`, *optional*): When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be completed before it is intentionally prematurely terminated. As a result, the returned sample will still retain a substantial amount of noise as determined by the discrete timesteps selected by the scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) guidance_scale (`float`, *optional*, defaults to 5.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. This is sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor is generated by sampling using the supplied random `generator`. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, text embeddings are generated from the `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. pooled_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated pooled text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, pooled text embeddings are generated from `prompt` input argument. negative_pooled_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, pooled `negative_prompt_embeds` are generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated image. Choose between `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a plain tuple. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. Examples: Returns: [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, otherwise a `tuple` is returned containing the output images. callbackNrrzjPassing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`zpPassing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`rQFrscale)rErFrkrIrlrm) rKrrrrgr*r|rhrrwcpu)rrcs<g|]\}}dtt|kpdt|kqS)rrQ)rr6)rse)rr+r-r;res*z@StableDiffusionXLControlNetPipeline.__call__..rt)r|r rxcs|kSrr-)ts)discrete_timestep_cutoffr-r;sz>StableDiffusionXLControlNetPipeline.__call__..z>=z2.1)totalr#) text_embedstime_idscSsg|]\}}||qSr-r-)rcr:r-r-r;rs)encoder_hidden_statescontrolnet_condconditioning_scaleradded_cond_kwargsr2cSs g|] }tt||gqSr-)rrr)rdr-r-r;rs r)rC timestep_condr(down_block_additional_residualsmid_block_additional_residualrFr2r2rDrErFrGrIrHrJrKorderlatent latents_mean latents_stdr)r2r)r1)images)dpoprrXr r tensor_inputsr rNrrYr6rrrr"r%r'r)r-rrr}rrr]global_pool_conditionsr(getrrhrmrrr|r XLA_AVAILABLEr<r7r+rM in_channelsrr&rr r#rr rrrangerrArprojection_dimrrrKr*roundnum_train_timestepsfilterr! progress_barrr.cuda is_available _inductorcudagraph_mark_step_beginscale_model_inputrrrlocalsupdater\xm mark_steprLfloat16 force_upcastrriterpost_quant_convr3rrMrNrr*scaling_factordecoderaapply_watermarkr_ postprocessmaybe_free_model_hooksr&)YrcrerfrKrrr)r+r,r*r#rirjrgrrrDrErFrkrIrrr1r2r(rrrrr r r r3r4r5rmr6rr8r7rrNmultrr*rRtext_encoder_lora_scalerrOrtimestep_devicerrHguidance_scale_tensorrcontrolnet_keepkeepsrGr rHrJnum_warmup_stepsis_unet_compiledis_controlnet_compiledis_torch_higher_equal_2_1r[tlatent_model_inputrFcontrol_model_inputcontrolnet_prompt_embedscontrolnet_added_cond_kwargs cond_scalecontrolnet_cond_scaledown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textcallback_kwargsrcallback_outputsstep_idxneeds_upcastinghas_latents_meanhas_latents_stdrMrNr-)r=rr+r;__call__s E              "                       $        6  o &&    z,StableDiffusionXLControlNetPipeline.__call__)TNNN) NNrQTNNNNNNNNr) NNNNNNNNrrrN)FF)<__name__ __module__ __qualname____doc__model_cpu_offload_seq_optional_componentsrrrrrrrrYrZrrboolrrrWrrr*rrrrrrrrrrrrrrr|r propertyr#rmrhr(r*r,r.no_gradrEXAMPLE_DOC_STRINGr Generatordictrrr r r __classcell__r-r-rdr;r=s,     2       p. ?/                        !" #$%&'r=)NNNN)Mr0typingrrnumpyr PIL.Imagerrtorch.nn.functionalrrr transformersrrrrrdiffusers.utils.import_utilsr callbacksr r r_r rloadersrrrrmodelsrrrrr models.lorar schedulersrutilsrrrrrrutils.torch_utilsr r!r"pipeline_utilsr$r%#stable_diffusion_xl.pipeline_outputr&stable_diffusion_xl.watermarkr'r(torch_xla.core.xla_modelcore xla_modelrcrT get_loggerrrrrrr*rYrr<r=r-r-r-r;s`        1    ;