o GiS@s6ddlZddlmZddlZddlmZddlmmZddl m Z m Z ddl m Z ddlmZddlmZmZdd lmZdd lmZdd lmZmZmZmZdd lmZdd lmZddl m!Z!m"Z"m#Z#e$e%Z&Gdddej'Z(GdddZ)Gdddej'Z*Gdddej'Z+Gdddee e eZ,dS)N)Any) ConfigMixinregister_to_config)PeftAdapterMixin)FromOriginalModelMixin)apply_lora_scalelogging)LuminaFeedForward) Attention)TimestepEmbedding Timestepsapply_rotary_embget_1d_rotary_pos_embed)Transformer2DModelOutput) ModelMixin)LuminaLayerNormContinuousLuminaRMSNormZeroRMSNormc sheZdZ    ddedededed d f fd d Zd ejdejdejd eejejffddZ Z S)'Lumina2CombinedTimestepCaptionEmbeddingh㈵> hidden_size cap_feat_dimfrequency_embedding_sizenorm_epsreturnNcsTtt|ddd|_t|t|dd|_tt ||dtj ||dd|_ dS)NTg) num_channelsflip_sin_to_cosdownscale_freq_shift) in_channelstime_embed_dimeps)bias) super__init__r time_projr mintimestep_embeddernn SequentialrLinearcaption_embedder)selfrrrr __class__e/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/models/transformers/transformer_lumina2.pyr*&s   z0Lumina2CombinedTimestepCaptionEmbedding.__init__ hidden_statestimestepencoder_hidden_statescCs,|||}||}||}||fSN)r+type_asr-r1)r2r7r8r9 timestep_proj time_embed caption_embedr5r5r6forward;s  z/Lumina2CombinedTimestepCaptionEmbedding.forward)rrrr) __name__ __module__ __qualname__intfloatr*torchTensortupler? __classcell__r5r5r3r6r%s2rc@s\eZdZdZddZ   ddedejdejdejdBd ejdBd edBd ejfd d Z dS)Lumina2AttnProcessor2_0z Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is used in the Lumina2Transformer2DModel model. It applies normalization and RoPE on query and key vectors. cCsttds tddS)Nscaled_dot_product_attentionzPAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.)hasattrF ImportError)r2r5r5r6r*Js z Lumina2AttnProcessor2_0.__init__Nattnr7r9attention_maskimage_rotary_embbase_sequence_lengthrcCs|j\}}} ||} ||} ||} | jd} | jd}| |j}| j}||}| |d|j|} | |d||} | |d||} |jdurN|| } |jdurX|| } |durjt | |dd} t | |dd} | || |} } |durt t |||j}n|j}|j|}|dkr| dddd|ddd} | dddd|ddd} |dur||ddd}| dd} | dd} | dd} tj| | | ||d}|dd|d|j|}|| }|jd|}|jd|}|S) NF)use_realrr ) attn_maskscaler)shapeto_qto_kto_vheadsdtypeviewnorm_qnorm_krtomathsqrtlogrV unsqueezerepeatflattenbool transposerLrJreshaper;to_out)r2rNr7r9rOrPrQ batch_sizesequence_length_querykeyvalue query_dim inner_dimhead_dimr\kv_heads softmax_scalen_repr5r5r6__call__NsP                  z Lumina2AttnProcessor2_0.__call__)NNN) r@rArB__doc__r*r rErFrCrwr5r5r5r6rIDs* rIcsreZdZ ddededededededed d ffd d Z dd ejdejdejdejd Bd ejf ddZ Z S)Lumina2TransformerBlockTdimnum_attention_heads num_kv_heads multiple_offfn_dim_multiplierr modulationrNc st|||_||_t|d||d||dddtd |_t|d|||d|_|r5t ||dd|_ nt ||d |_ t ||d |_ t ||d |_ t ||d |_dS) Nrms_normrF) rqcross_attention_dimdim_headqk_normr[rtr'r(out_bias processor)rzrrr}r~T) embedding_dimrnorm_elementwise_affiner&)r)r*rsrr rIrNr feed_forwardrnorm1r ffn_norm1norm2 ffn_norm2)r2rzr{r|r}r~rrr3r5r6r*s>   z Lumina2TransformerBlock.__init__r7rOrPtembc Cs|jrC|||\}}}}|j||||d} ||d|| }|||d|d} ||d|| }|S||}|j||||d} ||| }|||} ||| }|S)N)r7r9rOrPrT) rrrNrdtanhrrrr) r2r7rOrPrnorm_hidden_statesgate_msa scale_mlpgate_mlp attn_output mlp_outputr5r5r6r?s. zLumina2TransformerBlock.forward)Tr:) r@rArBrCrDrgr*rErFr?rHr5r5r3r6rys>  4ryc seZdZddedeedeedeffdd Zdeedeeded eejfd d Zd ejd ejfd dZ dejdejfddZ Z S)Lumina2RotaryPosEmbedi,rr thetaaxes_dim axes_lens patch_sizecs6t||_||_||_||_|||||_dSr:)r)r*rrrr_precompute_freqs_cis freqs_cis)r2rrrrr3r5r6r*s zLumina2RotaryPosEmbed.__init__rc CsXg}tjjr tjntj}tt||D]\}\}}t|||j |d} | | q|S)N)r freqs_dtype) rEbackendsmps is_availablefloat32float64 enumerateziprrappend) r2rrrrrideembr5r5r6rs  z+Lumina2RotaryPosEmbed._precompute_freqs_cisidsc Cs|j}|jjdkr|d}g}tt|jD]<}|j||j}|dddd||dfdd|jdt j }| t j | d|jdddd|dqt j|dd|S)NrcpurTrRr)rzindexrz)devicetyper`rangelenrrrerWrEint64rgatherrdcat)r2rrresultrfreqsrr5r5r6_get_freqs_ciss  6.z$Lumina2RotaryPosEmbed._get_freqs_cisr7rOcs|j\}}}}|j}||||}} || |j} |jd} |jdd} fdd| D} t| }tj||dtj| d}t t | | D]W\}\}}tj |tj| d||d|df<|||||df<tj |tj| d dd d| }tj | tj| d dd |d}|||||df<|||||d f<qF||}tj|| |jd| |jd }tj||jd| |jd }t t | | D]"\}\}}||d|f||d|f<||||f||df<q| ||||| |dd d dd dddd }||||| | fS) NrTrcsg|]}|qSr5r5).0 cap_seq_len image_seq_lenr5r6 sz1Lumina2RotaryPosEmbed.forward..r)r\rrrRr )rr\r)rWrrsumtolistmaxrEzerosint32rraranger]rerfrr\permute)r2r7rOrkchannelsheightwidthppost_patch_heightpost_patch_widthrencoder_seq_lenl_effective_cap_len seq_lengths max_seq_len position_idsrrseq_lenrow_idscol_idsr cap_freqs_cis img_freqs_cisr5rr6r?sR   zLumina2RotaryPosEmbed.forward)rr ) r@rArBrClistr*rErFrrr?rHr5r5r3r6rs ((  rc$seZdZdZdZdgZddgZe          d/dedededed Bdedededededede d Bde d e d!e eeefd"e eeefd#ed$d f"fd%d& Z e d' d0d(e jd)e jd*e jd+e jd'eeefd Bd,ed$e jeBfd-d.ZZS)1Lumina2Transformer2DModela Lumina2NextDiT: Diffusion model with a Transformer backbone. Parameters: sample_size (`int`): The width of the latent images. This is fixed during training since it is used to learn a number of position embeddings. patch_size (`int`, *optional*, (`int`, *optional*, defaults to 2): The size of each patch in the image. This parameter defines the resolution of patches fed into the model. in_channels (`int`, *optional*, defaults to 4): The number of input channels for the model. Typically, this matches the number of channels in the input images. hidden_size (`int`, *optional*, defaults to 4096): The dimensionality of the hidden layers in the model. This parameter determines the width of the model's hidden representations. num_layers (`int`, *optional*, default to 32): The number of layers in the model. This defines the depth of the neural network. num_attention_heads (`int`, *optional*, defaults to 32): The number of attention heads in each attention layer. This parameter specifies how many separate attention mechanisms are used. num_kv_heads (`int`, *optional*, defaults to 8): The number of key-value heads in the attention mechanism, if different from the number of attention heads. If None, it defaults to num_attention_heads. multiple_of (`int`, *optional*, defaults to 256): A factor that the hidden size should be a multiple of. This can help optimize certain hardware configurations. ffn_dim_multiplier (`float`, *optional*): A multiplier for the dimensionality of the feed-forward network. If None, it uses a default value based on the model configuration. norm_eps (`float`, *optional*, defaults to 1e-5): A small value added to the denominator for numerical stability in normalization layers. scaling_factor (`float`, *optional*, defaults to 1.0): A scaling factor applied to certain parameters or layers in the model. This can be used for adjusting the overall scale of the model's operations. Try x_embeddernormr N rr? rrrr# sample_sizerr$ out_channelsr num_layersnum_refiner_layersr{r|r}r~rscaling_factor axes_dim_roperrrcst|p||_td|||d|_tj|||d|_t|d|_ t fddt |D|_ t fddt |D|_ t fddt |D|_ttd d d d |||jd |_d |_dS)Ni')rrrr) in_features out_features)rrrc "g|] }tddqSTrryrrmr~rr}rr{r|r5r6r z6Lumina2Transformer2DModel.__init__..c r)Frrrrr5r6rrc rrrrrr5r6rrr#Fgư>T)rconditioning_embedding_dimelementwise_affiner'r(out_dim)r)r*rr rope_embedderr.r0rrtime_caption_embed ModuleListr noise_refinercontext_refinerlayersrr,norm_outgradient_checkpointing)r2rrr$rrrrr{r|r}r~rrrrrr3rr6r*msB       z"Lumina2Transformer2DModel.__init__attention_kwargsr7r8r9encoder_attention_mask return_dictc Cs|j\}}} } ||||\} }|||\}} } }}}||}|jD]}|||| }q$|jD] }||d| | }q0t|}tt|dk}|j ||t j d}| |||j j }tt||D]&\}\}}d||d|f<||d|f||d|f<||||||f<q_|}|jD]"}t r|jr||||r|nd|| }q|||r|nd|| }q||| }|j j}g}tt||D],\}\}}|||||| || ||||jdddddddddqt j|dd}|s|fSt|d S) NrT)r\Trrr rr)sample)rWrrrrrrrset new_zerosrErgconfigrrrris_grad_enabledr_gradient_checkpointing_funcrrrr]rrrfstackr)r2r7r8r9rrrrkrmrrrcontext_rotary_embnoise_rotary_emb rotary_embencoder_seq_lengthsrlayerruse_maskrOjoint_hidden_statesrrrroutputr5r5r6r?sZ        z!Lumina2Transformer2DModel.forward)rr rNrrr rrrNrrrrr#)NT)r@rArBrx _supports_gradient_checkpointing_no_split_modules _skip_layerwise_casting_patternsrrCrDrGr*rrErFdictstrrrgrr?rHr5r5r3r6rEs#       \r)-ratypingrrEtorch.nnr.torch.nn.functional functionalrLconfiguration_utilsrrloadersrloaders.single_file_modelrutilsrr attentionr attention_processorr embeddingsr rrrmodeling_outputsrmodeling_utilsr normalizationrrr get_loggerr@loggerModulerrIryrrr5r5r5r6s*         TQ\