o Gi]@s"ddlmZddlmZddlmZddlZddlZddl m Z m Z ddl m Z ddlmZdd lmZmZmZmZmZdd lmZd d lmZd dlmZer]ddlmmZdZ ndZ e!e"Z#e$e%ddde$e%dddd dZ&dZ'eGdddeZ(Gdddee Z)dS)) dataclass)ceil)CallableN) CLIPTextModel CLIPTokenizer)StableDiffusionLoraLoaderMixin)DDPMWuerstchenScheduler) BaseOutput deprecateis_torch_xla_availableloggingreplace_example_docstring) randn_tensor)DiffusionPipeline)WuerstchenPriorTF?gUUUUUU?g a Examples: ```py >>> import torch >>> from diffusers import WuerstchenPriorPipeline >>> prior_pipe = WuerstchenPriorPipeline.from_pretrained( ... "warp-ai/wuerstchen-prior", torch_dtype=torch.float16 ... ).to("cuda") >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet" >>> prior_output = pipe(prompt) ``` c@s"eZdZUdZejejBed<dS)WuerstchenPriorPipelineOutputz Output class for WuerstchenPriorPipeline. Args: image_embeddings (`torch.Tensor` or `np.ndarray`) Prior image embeddings for text prompt image_embeddingsN) __name__ __module__ __qualname____doc__torchTensornpndarray__annotations__r"r"l/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.pyr;s  rc$seZdZdZdZdZdZgdZddgZ   d6d e de de d e d e d e d e ddffdd ZddZ    d7dejdBdejdBfddZ  d8ddZeddZeddZeddZeeedd d d!dd"dddd#ddd$d%dd&gfd'eeeBdBd(ed)ed*ed+ee d,e d-eeeBdBdejdBdejdBd.edBd/ejeejBdBd&ejdBd0edBd1ed2e eegdfdBd3eef d4d5Z!Z"S)9WuerstchenPriorPipelinea Pipeline for generating image prior for Wuerstchen. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) The pipeline also inherits the following loading methods: - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights Args: prior ([`Prior`]): The canonical unCLIP prior to approximate the image embedding from the text embedding. text_encoder ([`CLIPTextModelWithProjection`]): Frozen text-encoder. tokenizer (`CLIPTokenizer`): Tokenizer of class [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). scheduler ([`DDPMWuerstchenScheduler`]): A scheduler to be used in combination with `prior` to generate image embedding. latent_mean ('float', *optional*, defaults to 42.0): Mean value for latent diffusers. latent_std ('float', *optional*, defaults to 1.0): Standard value for latent diffusers. resolution_multiple ('float', *optional*, defaults to 42.67): Default resolution for multiple images generated. prior text_encoderztext_encoder->prior)latentstext_encoder_hidden_statesnegative_prompt_embedsE@r(\UE@ tokenizer scheduler latent_mean latent_stdresolution_multiplereturnNcs0t|j||||d|j|||ddS)N)r,r&r%r-)r.r/r0)super__init__register_modulesregister_to_config)selfr,r&r%r-r.r/r0 __class__r"r#r3ls  z WuerstchenPriorPipeline.__init__cCsR|dur t||||d}n|j|krtd|jd|||}||j}|S)N) generatordevicedtypezUnexpected latents shape, got z , expected )rshape ValueErrortoinit_noise_sigma)r6r<r;r:r9r'r-r"r"r#prepare_latentss   z'WuerstchenPriorPipeline.prepare_latents prompt_embedsr)c Csj|dur t|tr d}n|durt|trt|}n|jd}|dur|j|d|jjddd} | j} | j} |j|dddj} | jd | jd krt | | s|j | dd|jjdd f} t d |jjd | | ddd|jjf} | ddd|jjf} |j| || |d }|j}|j|jj|d }|j|dd}|dur|r|durdg|}n;t|t|urtdt|dt|dt|tr|g}n|t|krtd|dt|d|d|d |}|j|d|jjddd}|j|j||j|d }|j}|r1|jd}|j|jj|d }|d|d}||||d }||fS)Nrr max_lengthTpt)paddingrB truncationreturn_tensorslongest)rDrFz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: )attention_mask)r;r:)dimz?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.) isinstancestrlistlenr<r,model_max_length input_idsrIrequal batch_decodeloggerwarningr&r>last_hidden_stater;repeat_interleavetype TypeErrorr=repeatview)r6r:num_images_per_promptdo_classifier_free_guidancepromptnegative_promptrAr) batch_size text_inputstext_input_idsrIuntruncated_ids removed_texttext_encoder_output uncond_tokens uncond_input*negative_prompt_embeds_text_encoder_outputseq_lenr"r"r# encode_prompts        z%WuerstchenPriorPipeline.encode_promptcCs|dur|durtd|d|d|dur|durtd|dur6t|ts6t|ts6tdt||durI|durItd|d|d|durd|durd|j|jkrdtd|jd |jd t|tsstd t|d dS) NzCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z2`prompt` has to be of type `str` or `list` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` rLz5'num_inference_steps' must be of type 'int', but got zo In Case you want to provide explicit timesteps, please use the 'timesteps' argument.)r=rMrNrOrYr<intrZ)r6r_r`num_inference_stepsr^rAr)r"r"r# check_inputss<   z$WuerstchenPriorPipeline.check_inputscC|jSN_guidance_scaler6r"r"r#guidance_scalez&WuerstchenPriorPipeline.guidance_scalecCs |jdkS)Nrrqrsr"r"r#r^s z3WuerstchenPriorPipeline.do_classifier_free_guidancecCrorp)_num_timestepsrsr"r"r# num_timestepsruz%WuerstchenPriorPipeline.num_timestepsi<g @rrCTr'r_heightwidthrm timestepsrtr`r]r9 output_type return_dictcallback_on_step_end"callback_on_step_end_tensor_inputsc& s|dd}|dd}|durtddd|dur tddd|durAtfdd|DsAtd jd fd d |Dj}|_|durSt|trSd }n|durat|t rat |}n|j d}|durt|t st|trx|g}n t dt |djr|durt|t st|tr|g}n t dt |dj|||j|| dj||| j||| d\}} | durt|| gn|}|j}t|jj}t|jj}jjj}| ||||f}|durjj||djj}t |}n jj||djj}|||| | j} t |dd_t|ddD]\}}| | !d"|}jjrFt| gdn| jrSt|gdn||d}jrk|#d\} }!t$|!| j%}jj&||| | dj'} |duri}"|D] }#t(|#|"|#<q||||"}$|$d| } |$d|}|$d| } |dur||dkr|t)jdd }%||%|| t*rt+,q)| jj-jj.} /| dkr| 012} |s| fSt3| S)a Function invoked when calling the pipeline for generation. Args: prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. height (`int`, *optional*, defaults to 1024): The height in pixels of the generated image. width (`int`, *optional*, defaults to 1024): The width in pixels of the generated image. num_inference_steps (`int`, *optional*, defaults to 60): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 8.0): Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598). `decoder_guidance_scale` is defined as `w` of equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `decoder_guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor will be generated by sampling using the supplied random `generator`. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` (`np.array`) or `"pt"` (`torch.Tensor`). return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. Examples: Returns: [`~pipelines.WuerstchenPriorPipelineOutput`] or `tuple` [`~pipelines.WuerstchenPriorPipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated image embeddings. callbackNcallback_stepsz1.0.0zhPassing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`znPassing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`c3s|]}|jvVqdSrp_callback_tensor_inputs.0krsr"r# s  z3WuerstchenPriorPipeline.__call__..z2`callback_on_step_end_tensor_inputs` has to be in z , but found csg|] }|jvr|qSr"rrrsr"r# sz4WuerstchenPriorPipeline.__call__..rrz2'prompt' must be of type 'list' or 'str', but got rLz;'negative_prompt' must be of type 'list' or 'str', but got )rAr))r_r:r]r^r`rAr))r{r:)r:rHr)rc) model_outputtimestepsampler9r'r(r)orderr)4popr allr=r_execution_devicerrrMrNrOrPr<rZrYr^rnrkrcatr;rconfigr0r%c_inr- set_timestepsr{r@rv enumerate progress_barexpandsizer>chunklerprtstep prev_samplelocalsgetattr XLA_AVAILABLExm mark_stepr.r/maybe_free_model_hookscpufloatnumpyr)&r6r_ryrzrmr{rtr`rAr)r]r9r'r|r}r~rkwargsrrr:rar(r; latent_height latent_width num_channelseffnet_features_shapeitratiopredicted_image_embeddingpredicted_image_embedding_text predicted_image_embedding_uncondcallback_kwargsrcallback_outputsstep_idxr"rsr#__call__!s V              z WuerstchenPriorPipeline.__call__)r*rr+)NNNN)NN)#rrrr unet_nametext_encoder_namemodel_cpu_offload_seqr_lora_loadable_modulesrrrr rr3r@rrrkrnpropertyrtr^rwno_gradrEXAMPLE_DOC_STRINGrNrOrl Generatorboolrr __classcell__r"r"r7r#r$Is  e )        r$)* dataclassesrmathrtypingrrrr transformersrrloadersr schedulersr utilsr r r r rutils.torch_utilsrpipeline_utilsrmodeling_wuerstchen_priorrtorch_xla.core.xla_modelcore xla_modelrr get_loggerrrUrOlinspaceDEFAULT_STAGE_C_TIMESTEPSrrr$r"r"r"r#s,         ,