o pi|V@sddlZddlmZmZmZmZddlZddlmZ ddl m Z m Z ddl mZddlmZmZmZddlmZdd lmZdd lmZd d lmZmZd dlmZeeZ GdddeZ!dS)N)ListOptionalTupleUnion) functional)CLIPTextModelWithProjection CLIPTokenizer)CLIPTextModelOutput)PriorTransformerUNet2DConditionModel UNet2DModel)UnCLIPScheduler)logging) randn_tensor)DiffusionPipelineImagePipelineOutput)UnCLIPTextProjModelc seZdZUdZdgZeed<eed<eed<e ed<e ed<e ed<e ed<e ed <e ed <e ed <d Z dedede de dede de d e d e d e ffd d ZddZ  d,deeeefdeejfddZe               d-deeeeefdeded ed!ed"eeejeejfd#eejd$eejd%eejdeeeefdeejd&ed'ed(eed)efd*d+ZZS).UnCLIPPipelineaE Pipeline for text-to-image generation using unCLIP. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). Args: text_encoder ([`~transformers.CLIPTextModelWithProjection`]): Frozen text-encoder. tokenizer ([`~transformers.CLIPTokenizer`]): A `CLIPTokenizer` to tokenize text. prior ([`PriorTransformer`]): The canonical unCLIP prior to approximate the image embedding from the text embedding. text_proj ([`UnCLIPTextProjModel`]): Utility class to prepare and combine the embeddings before they are passed to the decoder. decoder ([`UNet2DConditionModel`]): The decoder to invert the image embedding into an image. super_res_first ([`UNet2DModel`]): Super resolution UNet. Used in all but the last step of the super resolution diffusion process. super_res_last ([`UNet2DModel`]): Super resolution UNet. Used in the last step of the super resolution diffusion process. prior_scheduler ([`UnCLIPScheduler`]): Scheduler used in the prior denoising process (a modified [`DDPMScheduler`]). decoder_scheduler ([`UnCLIPScheduler`]): Scheduler used in the decoder denoising process (a modified [`DDPMScheduler`]). super_res_scheduler ([`UnCLIPScheduler`]): Scheduler used in the super resolution denoising process (a modified [`DDPMScheduler`]). priordecoder text_proj text_encoder tokenizersuper_res_firstsuper_res_lastprior_schedulerdecoder_schedulersuper_res_schedulerzAtext_encoder->text_proj->decoder->super_res_first->super_res_lastc s,t|j||||||||| | d dS)N) rrrrrrrrrr )super__init__register_modules) selfrrrrrrrrrr  __class__h/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/diffusers/pipelines/unclip/pipeline_unclip.pyr"Qs  zUnCLIPPipeline.__init__cCsR|dur t||||d}n|j|krtd|jd|||}||j}|S)N) generatordevicedtypezUnexpected latents shape, got z , expected )rshape ValueErrortoinit_noise_sigma)r$r,r+r*r)latents schedulerr'r'r(prepare_latentsms   zUnCLIPPipeline.prepare_latentsNtext_model_outputtext_attention_maskcCs(|duryt|tr t|nd}|j|d|jjddd}|j} |j|} |j|dddj} | j d| j dkrjt | | sj|j | dd|jjddf} t d |jjd | | ddd|jjf} || |} | j}| j}n|d j d }|d |d}}|} |j|d d }|j|d d }| j|d d } |rd g|}|j|d|jjddd}|j|}||j|}|j}|j}|j d}|d|}||||}|j d}|d|d}||||d}|j|d d }t ||g}t ||g}t || g} ||| fS)Nr max_lengthTpt)paddingr5 truncationreturn_tensorslongest)r7r9z\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: rdim) isinstancelistlenrmodel_max_length input_idsattention_maskboolr.r,torchequal batch_decodeloggerwarningr text_embedslast_hidden_staterepeat_interleaverepeatviewcat)r$promptr*num_images_per_promptdo_classifier_free_guidancer3r4 batch_size text_inputstext_input_ids text_maskuntruncated_ids removed_texttext_encoder_output prompt_embedstext_enc_hid_states uncond_tokens uncond_inputuncond_text_mask*negative_prompt_embeds_text_encoder_outputnegative_prompt_embedsuncond_text_enc_hid_statesseq_lenr'r'r(_encode_promptxsz       zUnCLIPPipeline._encode_promptr@ @pilTrQrRprior_num_inference_stepsdecoder_num_inference_stepssuper_res_num_inference_stepsr) prior_latentsdecoder_latentssuper_res_latentsprior_guidance_scaledecoder_guidance_scale output_type return_dictc2Cs|durt|tr d}nt|trt|}ntdt|| djd}|j}||}| dkp4| dk}|||||| | \}}}|j j ||d|j j }|j j j}|||f|j||||j }t||D]J\}}|rvt|gdn|}|j |||||dj}|r|d\}}|| ||}|d|jdkrd}n||d}|j j|||||d j}qh|j |}|}|j||||d \}} |jd kr|tj}tj||jjdfdd }!|!tj}!n tj||jjdfd d }!|j j ||d|j j }"|j!j j"}#|j!j j#}$|j!j j#}%|||#|$|%f|j||||j }t||"D]o\}}|r1t|gdn|}|j!|||| |!dj$}&|rq|&d\}'}(|'j%|jddd\}'})|(j%|jddd\}(}*|'| |(|'}&tj|&|*gdd}&|d|"jdkr~d}n|"|d}|j j|&||||dj}q"|&dd}|}+|j'j ||d|j'j },|j(j j"d}-|j(j j#}$|j(j j#}%|||-|$|%f|+j||| |j'} |jd krtj)|+|$|%gd}.ni}/dt*+tj)j,vrd |/d<tj)|+f|$|%gddd|/}.t||,D]E\}}||,jddkr|j-}0n|j(}0tj| |.gdd}|0||dj$}&|d|,jdkr/d}n|,|d}|j'j|&|| ||dj} q| }1|.|1dd}1|1&dd}1|1/0dddd12}1|dkrm|3|1}1|ss|1fSt4|1dS)a The call function to the pipeline for generation. Args: prompt (`str` or `List[str]`): The prompt or prompts to guide image generation. This can only be left undefined if `text_model_output` and `text_attention_mask` is passed. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. prior_num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps for the prior. More denoising steps usually lead to a higher quality image at the expense of slower inference. decoder_num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps for the decoder. More denoising steps usually lead to a higher quality image at the expense of slower inference. super_res_num_inference_steps (`int`, *optional*, defaults to 7): The number of denoising steps for super resolution. More denoising steps usually lead to a higher quality image at the expense of slower inference. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prior_latents (`torch.Tensor` of shape (batch size, embeddings dimension), *optional*): Pre-generated noisy latents to be used as inputs for the prior. decoder_latents (`torch.Tensor` of shape (batch size, channels, height, width), *optional*): Pre-generated noisy latents to be used as inputs for the decoder. super_res_latents (`torch.Tensor` of shape (batch size, channels, super res height, super res width), *optional*): Pre-generated noisy latents to be used as inputs for the decoder. prior_guidance_scale (`float`, *optional*, defaults to 4.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. decoder_guidance_scale (`float`, *optional*, defaults to 4.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. text_model_output (`CLIPTextModelOutput`, *optional*): Pre-defined [`CLIPTextModel`] outputs that can be derived from the text encoder. Pre-defined text outputs can be passed for tasks like text embedding interpolations. Make sure to also pass `text_attention_mask` in this case. `prompt` can the be left `None`. text_attention_mask (`torch.Tensor`, *optional*): Pre-defined CLIP text attention mask that can be derived from the tokenizer. Pre-defined text attention masks are necessary when passing `text_model_output`. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated image. Choose between `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. Returns: [`~pipelines.ImagePipelineOutput`] or `tuple`: If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated images. Nrz2`prompt` has to be of type `str` or `list` but is rg?)r*r)timestepproj_embeddingencoder_hidden_statesrD)rtsampler) prev_timestep)image_embeddingsr[text_encoder_hidden_statesrSmps)valueT)rwrtrv class_labelsrDr<)rxr)r;)size antialiasbicubicF)r~mode align_corners)rwrtg?r ri)images)5r?strr@rAr-typer,_execution_devicerdr set_timesteps timestepsrconfig embedding_dimr2r+ enumerate progress_barrFrPpredicted_image_embeddingchunkstep prev_samplepost_process_latentsrintFpadclip_extra_context_tokensrErr in_channels sample_sizerwsplitclampr r interpolateinspect signature parametersrmaybe_free_model_hookscpupermutefloatnumpy numpy_to_pilr)2r$rQrRrjrkrlr)rmrnror3r4rprqrrrsrTr*rSr[r\rWprior_timesteps_tensorritlatent_model_inputr predicted_image_embedding_uncondpredicted_image_embedding_textrxryadditive_clip_time_embeddingsdecoder_text_maskdecoder_timesteps_tensornum_channels_latentsheightwidth noise_prednoise_pred_uncondnoise_pred_text_predicted_variance image_smallsuper_res_timesteps_tensorchannelsimage_upscaledinterpolate_antialiasunetimager'r'r(__call__s>E                               zUnCLIPPipeline.__call__)NN)NrrererfNNNNNNrgrhriT) __name__ __module__ __qualname____doc___exclude_from_cpu_offloadr __annotations__r rrrr rmodel_cpu_offload_seqr"r2rrr rrFTensorrdno_gradrrr GeneratorrrEr __classcell__r'r'r%r(r"s     [     r)"rtypingrrrrrFtorch.nnrr transformersrr&transformers.models.clip.modeling_clipr modelsr r r schedulersrutilsrutils.torch_utilsrpipeline_utilsrrrr get_loggerrrIrr'r'r'r(s