o GiD@sddlZddlZddlmZddlmZmZmZddl m Z ddl m Z m Z mZddlmZdd lmZdd lmZdd lmZdd lmZmZmZe rYddlmmZd ZndZe e!Z"dZ#GdddeeZ$dS)N) CLIPTokenizer) AutoencoderKLControlNetModelUNet2DConditionModel) PNDMScheduler)is_torch_xla_availableloggingreplace_example_docstring) randn_tensor)BlipImageProcessor)Blip2QFormerModel)ContextCLIPTextModel)DeprecatedPipelineMixinDiffusionPipelineImagePipelineOutputTFa Examples: ```py >>> from diffusers.pipelines import BlipDiffusionControlNetPipeline >>> from diffusers.utils import load_image >>> from controlnet_aux import CannyDetector >>> import torch >>> blip_diffusion_pipe = BlipDiffusionControlNetPipeline.from_pretrained( ... "Salesforce/blipdiffusion-controlnet", torch_dtype=torch.float16 ... ).to("cuda") >>> style_subject = "flower" >>> tgt_subject = "teapot" >>> text_prompt = "on a marble table" >>> cldm_cond_image = load_image( ... "https://huggingface.co/datasets/ayushtues/blipdiffusion_images/resolve/main/kettle.jpg" ... ).resize((512, 512)) >>> canny = CannyDetector() >>> cldm_cond_image = canny(cldm_cond_image, 30, 70, output_type="pil") >>> style_image = load_image( ... "https://huggingface.co/datasets/ayushtues/blipdiffusion_images/resolve/main/flower.jpg" ... ) >>> guidance_scale = 7.5 >>> num_inference_steps = 50 >>> negative_prompt = "over-exposure, under-exposure, saturated, duplicate, out of frame, lowres, cropped, worst quality, low quality, jpeg artifacts, morbid, mutilated, out of frame, ugly, bad anatomy, bad proportions, deformed, blurry, duplicate" >>> output = blip_diffusion_pipe( ... text_prompt, ... style_image, ... cldm_cond_image, ... style_subject, ... tgt_subject, ... guidance_scale=guidance_scale, ... num_inference_steps=num_inference_steps, ... neg_prompt=negative_prompt, ... height=512, ... width=512, ... ).images >>> output[0].save("image.png") ``` c#s<eZdZdZdZdZ   d8dededed e d e d e d e d e dedeedeeffdd ZddZd9ddZd:ddZd:ddZ d;ddZeee  ! ! "  #   $ %dtext_encoder->unet->vaer N tokenizer text_encodervaeunet schedulerqformer controlnetimage_processor ctx_begin_posmeanstdc s8t|j||||||||d|j| | | ddS)N)rrrrrrrr)rrr)super__init__register_modulesregister_to_config) selfrrrrrrrrrrr __class__u/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.pyr ts  z(BlipDiffusionControlNetPipeline.__init__cCs|j||ddS)NF) image_input text_input return_dict)r)r# input_image src_subjectr&r&r'get_query_embeddingssz4BlipDiffusionControlNetPipeline.get_query_embeddings?c CsNg}t||D]\}}d|d|}|d|gt||q|S)Nza  z, )zipstripappendjoinint)r#prompts tgt_subjectsprompt_strength prompt_repsrvprompt tgt_subjectr&r&r' _build_prompts  z-BlipDiffusionControlNetPipeline._build_promptc Cst||||f} t|trt||krtdt|d|d|dur+t| |||d}n|j||d}||jj}|S)Nz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.) generatordevicedtyper?r@) isinstancelistlen ValueErrorr torinit_noise_sigma) r# batch_size num_channelsheightwidthr@r?r>latentsshaper&r&r'prepare_latentss   z/BlipDiffusionControlNetPipeline.prepare_latentsc Csp|p|j}|jjjj}||jjj8}|j|dd|dd|}|j d}|jj g|}|j|j ||dd}|S)N max_lengthTpt)padding truncationrOreturn_tensorsr) input_idsctx_embeddingsr) _execution_devicer text_modelconfigmax_position_embeddingsrnum_query_tokensrrFrMrrT) r# query_embedsr;r?max_lentokenized_promptrHrtext_embeddingsr&r&r' encode_prompts,   z-BlipDiffusionControlNetPipeline.encode_promptFc Csx|jj|||ddddddd|}|jd} | dkr |} n|} |j| dd }|j||d }|r:t|gd }|S) N)rKrJTFrP)size do_rescaledo_center_crop do_normalizerS pixel_valuesr)dimrAr )r preprocessrFrMrepeat_interleavetorchcat) r#imagerKrJrHnum_images_per_promptr?r@do_classifier_free_guidanceimage_batch_size repeat_byr&r&r'prepare_control_images*  z5BlipDiffusionControlNetPipeline.prepare_control_image@2pilTr;reference_imagecondtioning_imagesource_subject_categorytarget_subject_categoryrLguidance_scalerJrKnum_inference_stepsr> neg_promptr8r9 output_typer*c% Cs`|j}|jj||jj|jjddd}||}t|tr |g}t|tr(|g}t|tr0|g}t |}|j ||| |d}| ||}| |||}|dk}|rv|j jjj}|j| g|d|dd}|j |j|dd d }t||g}d t |jjjd }|j||jjj||| || ||jj|d }i}|jj| fi||j|| ||d ||jj|d}t||jjD]N\}}|dk}|rt|gd n|}|j||||dd\}} |j||||| dd}!|r|! d \}"}#|"||#|"}!|j!|!||d}t"r t#$q|j%j&||j%jj'ddd }$|jj(|$|d}$|)|s+|$fSt*|$dS)a Function invoked when calling the pipeline for generation. Args: prompt (`list[str]`): The prompt or prompts to guide the image generation. reference_image (`PIL.Image.Image`): The reference image to condition the generation on. condtioning_image (`PIL.Image.Image`): The conditioning canny edge image to condition the generation on. source_subject_category (`list[str]`): The source subject category. target_subject_category (`list[str]`): The target subject category. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor will be generated by random sampling. guidance_scale (`float`, *optional*, defaults to 7.5): Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. height (`int`, *optional*, defaults to 512): The height of the generated image. width (`int`, *optional*, defaults to 512): The width of the generated image. seed (`int`, *optional*, defaults to 42): The seed to use for random generation. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. neg_prompt (`str`, *optional*, defaults to ""): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_strength (`float`, *optional*, defaults to 1.0): The strength of the prompt. Specifies the number of times the prompt is repeated along with prompt_reps to amplify the prompt. prompt_reps (`int`, *optional*, defaults to 20): The number of times the prompt is repeated along with prompt_strength to amplify the prompt. Examples: Returns: [`~pipelines.ImagePipelineOutput`] or `tuple` rP) image_mean image_stdrSrd)r6r7r8r9r.rO)rQrOrSN)rTrUrr re)rHrIrJrKr>rLr@r?)rkrKrJrHrlr?r@rmF)encoder_hidden_statescontrolnet_condr*)timesteprdown_block_additional_residualsmid_block_additional_residualsample prev_sample)r*)r})images)+rVrrgrXrrrFrBstrrDr=r-r_rrWrYrrTrirjrblock_out_channelsrN in_channelsr@r set_timestepsrpr enumerate progress_bar timestepschunkstep XLA_AVAILABLExm mark_steprdecodescaling_factor postprocessmaybe_free_model_hooksr)%r#r;rvrwrxryrLrzrJrKr{r>r|r8r9r}r*r?rHr[r^rmrO uncond_inputuncond_embeddingsscale_down_factorextra_set_kwargs cond_imageitlatent_model_inputdown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textrkr&r&r'__call__sF            z(BlipDiffusionControlNetPipeline.__call__)r NN)r.r/)N)F) NrqrrrrrsNrtr.r/ruT)#__name__ __module__ __qualname____doc___last_supported_versionmodel_cpu_offload_seqrrrrrrrr r5rCfloatr r-r=rNr_rprino_gradr EXAMPLE_DOC_STRINGrPILImageTensor Generatorboolr __classcell__r&r&r$r'rUs       $ $     r)% PIL.Imagerri transformersrmodelsrrr schedulersrutilsrr r utils.torch_utilsr $blip_diffusion.blip_image_processingr blip_diffusion.modeling_blip2r blip_diffusion.modeling_ctx_cliprpipeline_utilsrrrtorch_xla.core.xla_modelcore xla_modelrr get_loggerrloggerrrr&r&r&r's$       .