o Gi @sddlZddlmZddlZddlZddlmZddlmm Z ddl m Z m Z ddlmZmZmZddlmZmZddlmZdd lmZmZdd lmZmZmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.e/e0Z1d&d'ddZ2d&d'ddZ3d&d'ddZ4GdddZ5Gdddejj6Z7Gdddejj6eZ8eGdddej6Z9eGd d!d!ej6Z:Gd"d#d#ej6Z;Gd$d%d%e*e eeee!e Zz@ requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.zJ`scale` should be a list of integers with the same length as `num_tokens`.c  g|] }tjddqST)biasdevicerZnnLinear.0_rjrnrZrir0r1 z7FluxIPAdapterAttnProcessor.__init__..c rkrlrorrrur0r1rvrw)superrDr9r?r@rArBrirj isinstancetuplelistlen ValueErrorscalerp ModuleListrangeto_k_ipto_v_ip)rCrirj num_tokensr~rnrZrArur1rDs0       z#FluxIPAdapterAttnProcessor.__init__rrr(r)rErFip_hidden_statesip_adapter_masksrGc Csh|jd}t|||\} } } } } }| d|jdf} | d|jdf} | d|jdf} || } || } | }|dur|| d|jdf} | d|jdf} |d|jdf}|| } || } tj | | gdd} tj | | gdd} tj || gdd} |durt | |dd} t | |dd} t | | | |dd|j |j d}|d d }|| j}|dur2|j|jd|jd|jdgdd\}}|jd|}|jd|}||}t|}t||j|j|jD]G\}}}}||}||}||d|j|j}||d|j|j}t |||ddd|j |j d}||d|j|j}||j}|||7}q|||fS|S) Nrr3rHr4rIF)rK dropout_p is_causalrLrMr r)r\r<rNrOrPrQrRrSrTrUrrrVrWrXrYrZr[r]r_ zeros_likezipr~rrviewhead_dimreshape)rCrr(r)rErFrr batch_sizer*r+r,r-r.r/ip_queryip_attn_outputcurrent_ip_hidden_statesr~rrip_keyip_valuer0r0r1r`s            z#FluxIPAdapterAttnProcessor.__call__)rfrhNN)NNNNN) rBrbrc__doc__rVrWintrDrTrdr{r` __classcell__r0r0rr1res@'  recseZdZeZeegZ             dded ed ed ed e d edBde dBde dedede dBde de ffdd Z   dde j de j dBde j dBde j dBde j f ddZ ZS) r@rFNTh㈵> query_dimrOdim_headdropoutrmr#added_proj_biasout_biasepsout_dimcontext_pre_onlypre_onlyelementwise_affinecst||_| dur| n|||_||_||_||_| dur"| n||_| |_| |_ | dur3| |n||_ ||_ ||_ t jj|| | d|_t jj|| | d|_t jj||j|d|_t jj||j|d|_t jj||j|d|_|j st jg|_|jt jj|j|j|d|jt j||durt jj|| d|_t jj|| d|_t jj||j|d|_t jj||j|d|_t jj||j|d|_t jj|j||d|_|dur| }|!|dS)N)rrrm)r)"rxrDr inner_dimruse_biasrrrrrOr#rrTrpRMSNormrPrQrqr r!r"rr]appendDropoutrRrSr$r%r&r__default_processor_cls set_processor)rCrrOrrrmr#rrrrrrr processorrr0r1rDs> zFluxAttention.__init__r(r)rErFrGc stt|jjjddhfdd|D}t|dkr1t d|d|jj j dfd d |D}|j|||||fi|S) Nrrcs$g|]\}}|vr|vr|qSr0r0)rskrtattn_parametersquiet_attn_parametersr0r1rvZs$z)FluxAttention.forward..rzjoint_attention_kwargs z are not expected by z and will be ignored.csi|] \}}|vr||qSr0r0)rsrw)rr0r1 _sz)FluxAttention.forward..) setinspect signaturerr` parameterskeysitemsr|loggerwarningrArB)rCr(r)rErFkwargs unused_kwargsr0rr1forwardPs zFluxAttention.forward) rrrFNTTrNNFTNra)rBrbrcr=rre_available_processorsrfloatboolrDrTrdrrr0r0rr1rsv     9cseZdZddedededeffdd Z  dd ejd ejd ejd eejejfdBd e e e fdBdeejejff ddZ Z S)FluxSingleTransformerBlock@r5num_attention_headsattention_head_dim mlp_ratioc sttt|||_t||_t||j|_tj dd|_ t||j||_ t ||||dt ddd|_dS)Ntanh) approximateTư>)rrrOrrmrrr)rxrDrmlp_hidden_dimrnormrprqproj_mlpGELUact_mlpproj_outrr=r)rCr5rrrrr0r1rDes   z#FluxSingleTransformerBlock.__init__Nr(r)tembrFjoint_attention_kwargsrGc Cs|jd}tj||gdd}|}|j||d\}} |||} |p$i}|jd||d|} tj| | gdd}| d} | ||}||}|j tj krU| dd}|ddd|f|dd|df}}||fS) NrHr4emb)r(rFr r0) r\rTrUrrrr unsqueezerrZfloat16clip) rCr(r)rrFr text_seq_lenresidualnorm_hidden_statesgatemlp_hidden_states attn_outputr0r0r1rys(    *z"FluxSingleTransformerBlock.forward)rNN)rBrbrcrrrDrTrdrzdictstrrrrr0r0rr1rcs" rcseZdZ ddededededef fdd Z dd ejd ejd ejde ejejfd Bde ee fd Bde ejejff ddZ Z S)FluxTransformerBlockrms_normrr5rrqk_normrc stt||_t||_t|||||ddt|d |_tj |ddd|_ t ||dd|_ tj |ddd|_ t ||dd|_dS)NFT) rr#rrOrrrmrrrrrzgelu-approximate)r5dim_out activation_fn)rxrDrnorm1 norm1_contextrr=rrp LayerNormnorm2rff norm2_context ff_context)rCr5rrrrrr0r1rDs$    zFluxTransformerBlock.__init__Nr(r)rrFrrGcCs||j||d\}}}} } |j||d\} } } }}|pi}|jd|| |d|}t|dkr3|\}}n t|dkr>|\}}}|d|}||}||}|d| dddf|dddf}||}| d|}||}t|dkr|||}| d|}||}||} | d|dddf| dddf} || }||d|}|j t j kr| dd}||fS) Nr)r(r)rFr rrHrrr0) rrrr|rrrrrrZrTrr)rCr(r)rrFrrgate_msa shift_mlp scale_mlpgate_mlpnorm_encoder_hidden_states c_gate_msa c_shift_mlp c_scale_mlp c_gate_mlpattention_outputsrcontext_attn_outputr ff_outputcontext_ff_outputr0r0r1rsD     (   (   zFluxTransformerBlock.forward)rrr)rBrbrcrrrrDrTrdrzrrrrr0r0rr1rs8rcs@eZdZdedeeffdd ZdejdejfddZZ S) FluxPosEmbedthetaaxes_dimcst||_||_dSr)rxrDrr)rCrrrr0r1rDs  zFluxPosEmbed.__init__idsrGc Cs|jd}g}g}|}|jjdk}|jjdk}|s|r tjntj}t|D]"} t|j | |dd| f|j dd|d\} } | | | | q'tj |dd |j} tj |dd |j} | | fS)Nr3mpsnpuT)rrepeat_interleave_realuse_real freqs_dtyper4)r\rrntyperTfloat32float64rrrrrrUrY)rCrn_axescos_outsin_outposis_mpsis_npuricossin freqs_cos freqs_sinr0r0r1rs*      zFluxPosEmbed.forward) rBrbrcrr{rDrTrdrrr0r0rr1rsrcsNeZdZdZdZddgZddgZddgZeddd d eddd d ed d d d ed d d d d e ddddZ e          d3de de de dBde de de de d e d!e d"e d#ee e e fffd$d% Zed&           d4d'ejd(ejd)ejd*ejd+ejd,ejd-ejd&eeefdBd.e d/e d0ejeBfd1d2ZZS)5FluxTransformer2DModela( The Transformer model introduced in Flux. Reference: https://blackforestlabs.ai/announcing-black-forest-labs/ Args: patch_size (`int`, defaults to `1`): Patch size to turn the input data into small patches. in_channels (`int`, defaults to `64`): The number of channels in the input. out_channels (`int`, *optional*, defaults to `None`): The number of channels in the output. If not specified, it defaults to `in_channels`. num_layers (`int`, defaults to `19`): The number of layers of dual stream DiT blocks to use. num_single_layers (`int`, defaults to `38`): The number of layers of single stream DiT blocks to use. attention_head_dim (`int`, defaults to `128`): The number of dimensions to use for each attention head. num_attention_heads (`int`, defaults to `24`): The number of attention heads to use. joint_attention_dim (`int`, defaults to `4096`): The number of dimensions to use for the joint attention (embedding/channel dimension of `encoder_hidden_states`). pooled_projection_dim (`int`, defaults to `768`): The number of dimensions to use for the pooled projection. guidance_embeds (`bool`, defaults to `False`): Whether to use guidance embeddings for guidance-distilled variant of the model. axes_dims_rope (`tuple[int]`, defaults to `(16, 56, 56)`): The dimensions to use for the rotary positional embeddings. Trr pos_embedrrHrF) split_dim expected_dims split_outputrr )r(r)img_idstxt_ids) gather_dimr)rrN&8r$ patch_size in_channels out_channels num_layersnum_single_layersrrjoint_attention_dimpooled_projection_dimguidance_embedsaxes_dims_ropec st|p|__td| d_| rtnt} | j| d_t |j_ t |j_ t fddt|D_t fddt|D_tjjddd _t j j||jd d _d_dS) Ni')rr) embedding_dimr+cg|] }tjdqS)r5rr)rrrrrrrCr0r1rvcz3FluxTransformer2DModel.__init__..cr/r0)rrrrr1r0r1rvnr2FrrTr)rxrDr'rrrrrtime_text_embedrprqcontext_embedder x_embedderrrtransformer_blockssingle_transformer_blocksrnorm_outrgradient_checkpointing) rCr%r&r'r(r)rrr*r+r,r-text_time_guidance_clsrr1r1rDCs0       zFluxTransformer2DModel.__init__rr(r)pooled_projectionstimesteprrguidance return_dictcontrolnet_blocks_repeatrGc  Cs@||}||jd}|dur||jd}|dur#|||n||||} ||}|jdkr=td|d}|jdkrKtd|d}tj ||fdd}| |}|durrd|vrr| d}| |}| d |it|jD]L\}}tr|jr||||| ||\}}n |||| ||d \}}| durt|jt| }tt|}| r|| |t| }qw|| ||}qwt|jD]@\}}tr|jr||||| ||\}}n |||| ||d \}}| dur t|jt| }tt|}|| ||}q||| }||}| s|fSt|d S) as The [`FluxTransformer2DModel`] forward method. Args: hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): Input `hidden_states`. encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. pooled_projections (`torch.Tensor` of shape `(batch_size, projection_dim)`): Embeddings projected from the embeddings of input conditions. timestep ( `torch.LongTensor`): Used to indicate denoising step. block_controlnet_hidden_states: (`list` of `torch.Tensor`): A list of tensors that if specified are added to the residuals of transformer blocks. joint_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain tuple. Returns: If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a `tuple` where the first element is the sample tensor. iNrzrPassing `txt_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch TensorrzrPassing `img_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch Tensorr4ip_adapter_image_embedsr)r(r)rrFr)sample)r5rYrZr3r4ndimrrrTrUrpopencoder_hid_projupdate enumerater6is_grad_enabledr9_gradient_checkpointing_funcr|rnpceilr7r8rr)rCr(r)r;r<rrr=rcontrolnet_block_samplescontrolnet_single_block_samplesr>r?rrrFr@r index_blockblockinterval_controloutputr0r0r1r}s +                 zFluxTransformer2DModel.forward) rHrNrrrrr r!Fr") NNNNNNNNNTF)rBrbrcr _supports_gradient_checkpointing_no_split_modules _skip_layerwise_casting_patterns_repeated_blocksr r_cp_planrrrrzrDr rTrd LongTensorrrrrrrr0r0rr1r s           9   rr)rr)=rtypingrnumpyrIrTtorch.nnrptorch.nn.functional functionalr?configuration_utilsrrloadersrrrutilsr r utils.torch_utilsr _modeling_parallelr r attentionrrrattention_dispatchr cache_utilsr embeddingsrrrrmodeling_outputsrmodeling_utilsr normalizationrrr get_loggerrBrr2r:r<r=Modulererrrrrr0r0r0r1sP           CP5T