o GiDf @sddlmZddlZddlZddlmZddlmZm Z ddl m Z m Z m Z ddlmZmZmZddlmZddlmZd d lmZmZd d lmZd d lmZmZmZmZd d l m!Z!d dl"m#Z#d dl$m%Z%m&Z&m'Z'ddl(m)Z)m*Z*e+e,Z-Gdddej.Z/Gdddej.Z0Gdddej.Z1Gdddej.Z2Gdddej.Z3eGdddej.Z4eGdddej.Z5Gd d!d!e#ee e e ee Z6dS)")AnyN) ConfigMixinregister_to_config)FluxTransformer2DLoadersMixinFromOriginalModelMixinPeftAdapterMixin)apply_lora_scale deprecatelogging)is_torch_npu_available)maybe_allow_in_graph)AttentionMixin FeedForward) CacheMixin) FluxPosEmbedPixArtAlphaTextProjection Timestepsget_timestep_embedding)Transformer2DModelOutput) ModelMixin)CombinedTimestepLabelEmbeddings FP32LayerNormRMSNorm) FluxAttentionFluxAttnProcessorcseZdZdZddededBffdd Z    dd ejd ejdBd ejdBd ej dBd ejdBde ejejejejejff ddZ Z S)ChromaAdaLayerNormZeroPruned Norm layer adaptive layer norm zero (adaLN-Zero). Parameters: embedding_dim (`int`): The size of each embedding vector. num_embeddings (`int`): The size of the embeddings dictionary. N layer_normT embedding_dimnum_embeddingscspt|durt|||_nd|_|dkr"tj|ddd|_dS|dkr0t|ddd|_dStd|d) Nr Fư>elementwise_affineepsfp32_layer_norm)r%biasUnsupported `norm_type` (@) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'.) super__init__rembnn LayerNormnormr ValueError)selfr!r" norm_typer( __class__d/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/models/transformers/transformer_chroma.pyr,0s  z%ChromaAdaLayerNormZeroPruned.__init__xtimestep class_labels hidden_dtyper-returnc Csx|jdur |j|||d}|ddjddd\}}}} } } ||d|dddf|dddf}||| | | fS)N)r;rrdim)r-flattenchunkr0) r2r8r9r:r;r- shift_msa scale_msagate_msa shift_mlp scale_mlpgate_mlpr6r6r7forward@s ".z$ChromaAdaLayerNormZeroPruned.forward)Nr T)NNNN) __name__ __module__ __qualname____doc__intr,torchTensor LongTensordtypetuplerH __classcell__r6r6r4r7r's(rc sbeZdZdZd deffdd Z ddejd ejdBd eejejejejejffd d Z Z S)"ChromaAdaLayerNormZeroSinglePrunedrr Tr!cs8t|dkrtj|ddd|_dStd|d)Nr Fr#r$r)r*)r+r,r.r/r0r1)r2r!r3r(r4r6r7r,Xs  z+ChromaAdaLayerNormZeroSinglePruned.__init__Nr8r-r<cCsR|ddjddd\}}}||d|dddf|dddf}||fS)Nrrrr>)r@rAr0)r2r8r-rBrCrDr6r6r7rHbs.z*ChromaAdaLayerNormZeroSinglePruned.forward)r TN) rIrJrKrLrMr,rNrOrRrHrSr6r6r4r7rTOs rTcsPeZdZdZ    ddedeffdd Zd ejd ejd ejfd d ZZ S)"ChromaAdaLayerNormContinuousPruneda Adaptive normalization layer with a norm layer (layer_norm or rms_norm). Args: embedding_dim (`int`): Embedding dimension to use during projection. conditioning_embedding_dim (`int`): Dimension of the input condition. elementwise_affine (`bool`, defaults to `True`): Boolean flag to denote if affine transformation should be applied. eps (`float`, defaults to 1e-5): Epsilon factor. bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use. norm_type (`str`, defaults to `"layer_norm"`): Normalization layer to use. Values supported: "layer_norm", "rms_norm". Th㈵>r r!conditioning_embedding_dimcsPt|dkrt|||||_dS|dkr!t||||_dStd|)Nr rms_normzunknown norm_type )r+r,r.r/r0rr1)r2r!rXr%r&r(r3r4r6r7r,{s z+ChromaAdaLayerNormContinuousPruned.__init__r8r-r<cCsbtj|dd|jddd\}}||d|dddddf|dddddf}|S)Nrrr>)rNrAr@torQr0)r2r8r-shiftscaler6r6r7rHs$:z*ChromaAdaLayerNormContinuousPruned.forward)TrWTr ) rIrJrKrLrMr,rNrOrHrSr6r6r4r7rVls$rVcs<eZdZdedeffdd ZdejdejfddZZS) (ChromaCombinedTimestepTextProjEmbeddings num_channelsout_dimcsXtt|ddd|_t|ddd|_|jdtt|dd|ddddd dS) NTr)r^flip_sin_to_cosdownscale_freq_shiftmod_projr)r`raF) persistent) r+r,r time_proj guidance_projregister_bufferrrNarange)r2r^r_r4r6r7r,s  z1ChromaCombinedTimestepTextProjEmbeddings.__init__r9r<c Cs|jjd}|jd}||j|jd}|tdg|j|j|jd}|jj|j|jd |dd}tj ||gdd d d|d}tj ||gdd}||jS)Nr)rQ)rQdevicerr>) rbshapererZrQrfrNtensorrirepeatcat unsqueeze) r2r9mod_index_length batch_sizetimesteps_projrfrbtimestep_guidance input_vecr6r6r7rHs    z0ChromaCombinedTimestepTextProjEmbeddings.forward) rIrJrKrMr,rNrOrHrSr6r6r4r7r]sr]c s8eZdZd dedededeffdd Zdd ZZS) ChromaApproximatorin_dimr_ hidden_dimn_layerscsjttj|dd|_tfddt|D|_tfddt|D|_t||_ dS)NTr(csg|] }tddqS)silu)act_fn)r.0_rxr6r7 sz/ChromaApproximator.__init__..csg|]}tqSr6)r.rr}rr6r7rs) r+r,r.Linearin_proj ModuleListrangelayersnormsout_proj)r2rwr_rxryr4rr7r,s zChromaApproximator.__init__cCs<||}t|j|jD] \}}||||}q ||SrU)rziprrr)r2r8layerrr6r6r7rHs  zChromaApproximator.forward)rv)rIrJrKrMr,rHrSr6r6r4r7rus  ruc seZdZ ddedededeffdd Z   dd ejd ejd eejejfdBd ejdBd e e e fdBdejf ddZ Z S)ChromaSingleTransformerBlock@r?num_attention_headsattention_head_dim mlp_ratioc stt|||_t||_t||j|_tj dd|_ t||j||_ t r?ddl m}d}tdd||}nt}t||||d|d dd |_dS) Ntanh) approximater)FluxAttnProcessor2_0_NPUzDefaulting to FluxAttnProcessor2_0_NPU for NPU devices will be removed. Attention processors should be set explicitly using the `set_attn_processor` method. npu_processorz0.34.0Tr#) query_dimdim_headheadsr_r( processorr&pre_only)r+r,rMmlp_hidden_dimrTr0r.rproj_mlpGELUact_mlpproj_outr attention_processorrr rrattn)r2r?rrrrdeprecation_messagerr4r6r7r,s.     z%ChromaSingleTransformerBlock.__init__N hidden_statestembimage_rotary_embattention_maskjoint_attention_kwargsr<c Cs|}|j||d\}}|||} |pi}|dur3|ddddddf|ddddddf}|jd|||d|} tj| | gdd}|d}|||}||}|jtj krd| dd}|S) Nr-)rrrrr>r r6) r0rrrrNrnrorrQfloat16clip) r2rrrrrresidualnorm_hidden_statesgatemlp_hidden_states attn_outputr6r6r7rHs(0   z$ChromaSingleTransformerBlock.forward)rNNN)rIrJrKrMfloatr,rNrOrRdictstrrrHrSr6r6r4r7rs6)rcseZdZ  ddededededef fdd Z dd ejd ejd ejde ejejfd Bdejd Bde ee fd Bde ejejffddZ Z S)ChromaTransformerBlockrYr#r?rrqk_normr&c stt||_t||_t|||||ddt|d |_tj |ddd|_ t ||dd|_ tj |ddd|_ t ||dd|_dS)NFT) radded_kv_proj_dimrrr_context_pre_onlyr(rr&r#r$zgelu-approximate)r?dim_out activation_fn)r+r,rnorm1 norm1_contextrrrr.r/norm2rff norm2_context ff_context)r2r?rrrr&r4r6r7r,s$    zChromaTransformerBlock.__init__Nrencoder_hidden_statesrrrrr<cCs|ddddf|ddddf}}|j||d\} } } } } |j||d\}}}}}|p0i}|durM|ddddddf|ddddddf}|jd | |||d|}t|dkre|\}}n t|dkrp|\}}}| d|}||}||} | d| dddf| dddf} || }| d|}||}t|dkr||}|d|}||}||}|d|dddf|dddf}||}||d|}|j t j kr| dd}||fS) Nr=r)rrrrrrrrrr6) rrrlenrorrrrrQrNrr)r2rrrrrrtemb_imgtemb_txtrrDrErFrGnorm_encoder_hidden_states c_gate_msa c_shift_mlp c_scale_mlp c_gate_mlpattention_outputsrcontext_attn_outputip_attn_output ff_outputcontext_ff_outputr6r6r7rH3sL* 0     (   (   zChromaTransformerBlock.forward)rYr#r)rIrJrKrMrrr,rNrOrRrrrHrSr6r6r4r7rs@#rcseZdZdZdZddgZddgZddgZe       d.de de de d Bde de de de de de e dfde de de ffdd Z e d!  "d/d#e jd$e jd%e jd&e jd'e jd(e jd!eeefd Bd)ed*ed+e jeBfd,d-ZZS)0ChromaTransformer2DModela The Transformer model introduced in Flux, modified for Chroma. Reference: https://huggingface.co/lodestones/Chroma1-HD Args: patch_size (`int`, defaults to `1`): Patch size to turn the input data into small patches. in_channels (`int`, defaults to `64`): The number of channels in the input. out_channels (`int`, *optional*, defaults to `None`): The number of channels in the output. If not specified, it defaults to `in_channels`. num_layers (`int`, defaults to `19`): The number of layers of dual stream DiT blocks to use. num_single_layers (`int`, defaults to `38`): The number of layers of single stream DiT blocks to use. attention_head_dim (`int`, defaults to `128`): The number of dimensions to use for each attention head. num_attention_heads (`int`, defaults to `24`): The number of attention heads to use. joint_attention_dim (`int`, defaults to `4096`): The number of dimensions to use for the joint attention (embedding/channel dimension of `encoder_hidden_states`). axes_dims_rope (`tuple[int]`, defaults to `(16, 56, 56)`): The dimensions to use for the rotary positional embeddings. Trr pos_embedr0r@N&8rrv patch_size in_channels out_channels num_layersnum_single_layersrrjoint_attention_dimaxes_dims_rope.approximator_num_channelsapproximator_hidden_dimapproximator_layersc st|p|__td| d_t| dd|d|dd_t| j| | d_ t |j_ t |j_ t fd d t|D_t fd d t|D_tjjd d d_t j j||jdd_d _dS)Ni')thetaaxes_dimr r)r^r_)rwr_rxrycg|] }tjdqS)r?rr)r inner_dimr}rrr2r6r7rz5ChromaTransformer2DModel.__init__..crr)rrr}rr6r7rrFr#r$Trz)r+r,rrrrr]time_text_embedrudistilled_guidance_layerr.rcontext_embedder x_embedderrrtransformer_blockssingle_transformer_blocksrVnorm_outrgradient_checkpointing) r2rrrrrrrrrrrrr4rr7r,s>      z!ChromaTransformer2DModel.__init__rFrrr9img_idstxt_idsr return_dictcontrolnet_blocks_repeatr<c  Cs||}||jd}||} || } ||}|jdkr*td|d}|jdkr8td|d}t j ||fdd}| |}|dur_d|vr_| d}| |}|d |it|jD]\}}dt|j}|d t|j}|d |}|d |}t j | dd||d f| dd||d ffd d}t r|jr|||||||\}}n |||||||d \}}|durt|jt|}tt|}| r|||t|}qd||||}qdt j ||gd d}t|jD]d\}}d|}| dd||df}t r|jr|||||}n ||||||d }| dur]t|jt| }tt|}|dd|jd ddf| |||dd|jd ddf<q|dd|jd ddf}| ddddf}|||}||}| s|fSt|dS)a The [`FluxTransformer2DModel`] forward method. Args: hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): Input `hidden_states`. encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. timestep ( `torch.LongTensor`): Used to indicate denoising step. block_controlnet_hidden_states: (`list` of `torch.Tensor`): A list of tensors that if specified are added to the residuals of transformer blocks. joint_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain tuple. Returns: If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a `tuple` where the first element is the sample tensor. rcrzrPassing `txt_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch TensorrzrPassing `img_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch Tensorr>Nip_adapter_image_embedsip_hidden_statesr=r)rrrrrr)rrrrr.)sample)rrZrQrrrndimloggerwarningrNrnrpopencoder_hid_projupdate enumeraterrris_grad_enabledr_gradient_checkpointing_funcrMnpceilrkrrr)r2rrr9rrrrcontrolnet_block_samplescontrolnet_single_block_samplesrrrt pooled_tembidsrrr index_blockblock img_offset txt_offsetimg_modulationtext_modulationrinterval_control start_idxoutputr6r6r7rHs (                   z ChromaTransformer2DModel.forward) rrNrrrrrrrrrv) NNNNNNNNTF)rIrJrKrL _supports_gradient_checkpointing_no_split_modules_repeated_blocks _skip_layerwise_casting_patternsrrMrRr,r rNrOrPrrrboolrrHrSr6r6r4r7rrs      @   r)7typingrnumpyrrNtorch.nnr.configuration_utilsrrloadersrrrutilsr r r utils.import_utilsr utils.torch_utilsr attentionrr cache_utilsr embeddingsrrrrmodeling_outputsrmodeling_utilsr normalizationrrrtransformer_fluxrr get_loggerrIrModulerrTrVr]rurrrr6r6r6r7sF        (, G ^