o piB@sddlmZddlmZmZmZddlZddlmZddlm Z m Z ddl m Z dd l mZdd lmZmZmZdd lmZmZmZdd lmZdd lmZe eZeGdddeZGdddee ZGdddeZ dS)) dataclass)DictOptionalUnionN)nn) ConfigMixinregister_to_config)logging)AttentionProcessor) BaseOutputTuple zero_module)-HunyuanCombinedTimestepTextSizeStyleEmbedding PatchEmbedPixArtAlphaTextProjection) ModelMixin)HunyuanDiTBlockc@seZdZUeejed<dS)HunyuanControlNetOutputcontrolnet_block_samplesN)__name__ __module__ __qualname__rtorchTensor__annotations__rra/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/diffusers/models/controlnet_hunyuan.pyr$s rcseZdZe          d,dedededeedeedededededededededeffdd Z e de ee ffd d!Z d"ee e ee fffd#d$Ze d-d%d&Z '        d.d(ejd)efd*d+ZZS)/HunyuanDiT2DControlNetModelXNgelu-approximate (@MTconditioning_channelsnum_attention_headsattention_head_dim in_channels patch_size activation_fntransformer_num_layers mlp_ratiocross_attention_dimcross_attention_dim_t5pooled_projection_dimtext_len text_len_t5"use_style_cond_and_image_meta_sizecst|_||_t| | ddd_ttj ||tj d_ t |||||dd_ t|| || |d_tg_tfddt| d d D_tt||_ttjD]}t||}t|}j|qhdS) N silu_fp32) in_features hidden_size out_featuresact_fn)dtype)heightwidthr/ embed_dimr0pos_embed_type)r6seq_lenr4r9c s2g|]}tjjjtjdddqS)TF)dimr-r1 ff_inner_dimr4qk_normskip)r inner_dimconfigr-int).0layerr1r4r3selfrr cs  z8HunyuanDiT2DControlNetModel.__init__..rr )super__init__ num_headsrJr text_embedderr Parameterrrandnfloat32text_embedding_paddingr pos_embedrtime_extra_emb ModuleListcontrolnet_blocksrangeblocksrLinear input_blocklenappend)rPr,r-r.r/r0r1 sample_sizer=r2r3r4r5r6r7r8r9_controlnet_block __class__rOrrS*sN      z$HunyuanDiT2DControlNetModel.__init__returncsLi}dtdtjjdtttfffdd |D] \}}|||q|S)z Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with indexed by its weight name. namemodule processorscsLt|dr|jdd||d<|D]\}}|d|||q|S)N get_processorT)return_deprecated_lora .processor.)hasattrrmnamed_children)rjrkrlsub_namechildfn_recursive_add_processorsrrrvs zPHunyuanDiT2DControlNetModel.attn_processors..fn_recursive_add_processors)strrrModulerr rr)rPrlrjrkrrurattn_processorsvs & z+HunyuanDiT2DControlNetModel.attn_processors processorcst|j}t|tr"t||kr"tdt|d|d|ddtdtjj ffdd | D] \}}|||q3d S) a2 Sets the attention processor to use to compute attention. Parameters: processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): The instantiated processor class or a dictionary of processor classes that will be set as the processor for **all** `Attention` layers. If `processor` is a dict, the key needs to define the path to the corresponding cross attention processor. This is strongly recommended when setting trainable attention processors. z>A dict of processors was passed, but the number of processors z0 does not match the number of attention layers: z. Please make sure to pass z processor classes.rjrkcsbt|drt|ts||n |||d|D]\}}|d|||qdS)N set_processorrorp)rq isinstancedictr{poprr)rjrkrzrsrtfn_recursive_attn_processorrrrs   zSHunyuanDiT2DControlNetModel.set_attn_processor..fn_recursive_attn_processorN) rbrykeysr|r} ValueErrorrwrrrxrr)rPrzcountrjrkrrrset_attn_processors   z.HunyuanDiT2DControlNetModel.set_attn_processorcCs|j}|j}|j}|j}|j} |j} |j} |j} |j} |j }|j }|j }|j }|}|p-|j }||||||| | | | | ||||d}|rU|j|dd}td|d|S)N)r,r2r1r.r4r5r=r/r3r-r0rdr7r8F)strictz0controlnet load from Hunyuan-DiT. missing_keys: r)rKr1r.r4r5r=r/r3r-r0rdr7r8r2load_state_dict state_dictloggerwarning)cls transformerr,r2load_weights_from_transformerrKr1r.r4r5r=r/r3r-r0rdr7r8 controlnetkeyrrrfrom_transformersF z,HunyuanDiT2DControlNetModel.from_transformer?controlnet_condconditioning_scalec s>|jdd\} }||}||||}|j||| | |jd}|j\}}}||d|jd}|||d}tj||gdd}tj||gdd}| d }t |||j }d}t |jD]\}}||||| d }||f}qdd}t||jD] \}}||}||f}q~fd d |D}| s|fSt|d S)  The [`HunyuanDiT2DControlNetModel`] forward method. Args: hidden_states (`torch.Tensor` of shape `(batch size, dim, height, width)`): The input tensor. timestep ( `torch.LongTensor`, *optional*): Used to indicate denoising step. controlnet_cond ( `torch.Tensor` ): The conditioning input to ControlNet. conditioning_scale ( `float` ): Indicate the conditioning scale. encoder_hidden_states ( `torch.Tensor` of shape `(batch size, sequence len, embed dims)`, *optional*): Conditional embeddings for cross attention layer. This is the output of `BertModel`. text_embedding_mask: torch.Tensor An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. This is the output of `BertModel`. encoder_hidden_states_t5 ( `torch.Tensor` of shape `(batch size, sequence len, embed dims)`, *optional*): Conditional embeddings for cross attention layer. This is the output of T5 Text Encoder. text_embedding_mask_t5: torch.Tensor An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. This is the output of T5 Text Encoder. image_meta_size (torch.Tensor): Conditional embedding indicate the image sizes style: torch.Tensor: Conditional embedding indicate the style image_rotary_emb (`torch.Tensor`): The image rotary embeddings to apply on query and key tensors during attention calculation. return_dict: bool Whether to return a dictionary. N) hidden_dtyper )rFrr)tembencoder_hidden_statesimage_rotary_embcsg|]}|qSrr)rMsamplerrrrQ0sz7HunyuanDiT2DControlNetModel.forward..)r)shaperZrar[r@rUviewrcat unsqueezeboolwhererY enumerater_zipr]r)rP hidden_statestimesteprrrtext_embedding_maskencoder_hidden_states_t5text_embedding_mask_t5image_meta_sizestyler return_dictrArBr batch_sizesequence_lengthreblock_res_samplesrNblockcontrolnet_block_res_samplesblock_res_samplerfrrrforwards@/      z#HunyuanDiT2DControlNetModel.forward)r r!r"NNr#r$r%r&r'r(r)r(r*r+T)r NT rNNNNNNNT)rrrr rLrrwfloatrrSpropertyrr ryrr classmethodrrrr __classcell__rrrgrr)s    K  .rcsHeZdZdZfddZ         d dejdefd d ZZ S) HunyuanDiT2DMultiControlNetModela `HunyuanDiT2DMultiControlNetModel` wrapper class for Multi-HunyuanDiT2DControlNetModel This module is a wrapper for multiple instances of the `HunyuanDiT2DControlNetModel`. The `forward()` API is designed to be compatible with `HunyuanDiT2DControlNetModel`. Args: controlnets (`List[HunyuanDiT2DControlNetModel]`): Provides additional conditioning to the unet during the denoising process. You must set multiple `HunyuanDiT2DControlNetModel` as a list. cstt||_dS)N)rRrSrr\nets)rP controlnetsrgrrrSEs z)HunyuanDiT2DMultiControlNetModel.__init__rNTrrc Csvtt|||jD]/\} \}}}|||||||||| | | | d }| dkr'|}q ddt|d|dD}|f}q |S)r) rrrrrrrrrrrrrcSsg|]\}}||qSrr)rMcontrol_block_sample block_samplerrrrQsz.)rrr)rPrrrrrrrrrrrriimagescaler block_samplescontrol_block_samplesrrrrIs, .z(HunyuanDiT2DMultiControlNetModel.forwardr) rrr__doc__rSrrrrrrrrgrr8s"  r)! dataclassesrtypingrrrrrconfiguration_utilsrr utilsr attention_processorr rr rr embeddingsrrrmodeling_utilsr#transformers.hunyuan_transformer_2dr get_loggerrrrrrrrrrs$