o GiM@sddlZddlZddlmZddlmmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZddlmZdd lmZdd lmZmZe eZGd d d ejZGd ddejZGdddejZGdddZ GdddejZ!GdddeeZ"dS)N) ConfigMixinregister_to_config)logging) Attention)TimestepEmbedding Timestepsget_2d_sincos_pos_embed)Transformer2DModelOutput) ModelMixin) AdaLayerNormRMSNormcs<eZdZdedeffdd ZdejdejfddZZS) OmniGenFeedForward hidden_sizeintermediate_sizecs@ttj|d|dd|_tj||dd|_t|_dS)NrFbias)super__init__nnLinear gate_up_proj down_projSiLU activation_fn)selfrr __class__e/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/models/transformers/transformer_omnigen.pyr"s zOmniGenFeedForward.__init__ hidden_statesreturncCs4||}|jddd\}}|||}||S)Nrdim)rchunkrr)rr! up_statesgaterrr forward)s  zOmniGenFeedForward.forward) __name__ __module__ __qualname__intrtorchTensorr) __classcell__rrrr r!srcseZdZ       dded ed ed ed ed edeffdd ZddZdej dedej fddZ ddej dedej dej fddZ Z S)OmniGenPatchEmbedrT@ patch_size in_channels embed_dimrinterpolation_scalepos_embed_max_size base_sizec sttj||||f||d|_tj||||f||d|_||_||_||_t ||j||jdd}|j d| ddddS)N) kernel_sizestriderpt)r<r: output_type pos_embedrT) persistent) rrrConv2doutput_image_projinput_image_projr7r:r;r register_bufferfloat unsqueeze) rr7r8r9rr:r;r<rArrr r1s$ zOmniGenPatchEmbed.__init__cCs|jdur td||j}||j}||jkr$td|d|jd||jkr5td|d|jd|j|d}|j|d}|jd|j|jd }|dd||||||ddf}|dd |jd }|S) z2Crops positional embeddings for SD3 compatibility.Nz.`pos_embed_max_size` must be set for cropping.zHeight (z/) cannot be greater than `pos_embed_max_size`: .zWidth (rr4r#)r; ValueErrorr7rAreshapeshape)rheightwidthtopleftspatial_pos_embedrrr _cropped_pos_embedQs$     (z$OmniGenPatchEmbed._cropped_pos_embedr!is_input_imager"cCs0|r||}n||}|ddd}|S)Nrr4)rErDflatten transpose)rr!rSrrr _patch_embeddingshs   z#OmniGenPatchEmbed._patch_embeddingsNpadding_latentc Cst|trM|durdgt|}g}t||D]3\}}|jdd\}}|||}|||} || }|durEtj|| |j gdd}| |q|S|jdd\}}|||} |||}|| }|S)Nr$) isinstancelistlenziprLrVrRr.cattodeviceappend) rr!rSrWpatched_latents sub_latentpaddingrMrNrArrr r)ps$      zOmniGenPatchEmbed.forward)rr2r3Tr4r5r6)N) r*r+r,r-boolrGrrRr.r/rVr)r0rrrr r10sF  r1cs(eZdZ d fdd ZddZZS) OmniGenSuScaledRotaryEmbedding'Ncstt||_||_||_d|jtjd|jdtjd|j}|j d|dd|d|_ |d |_ ||_ dS) N?rr)dtypeinv_freqF)tensorrB short_factor long_factor) rrr%max_position_embeddingsbaser.arangeint64rGrFrmrn original_max_position_embeddings)rr%rorsrp rope_scalingrkrrr rs *   z'OmniGenSuScaledRotaryEmbedding.__init__cCst|d}||jkrtj|jtj|jd}n tj|jtj|jd}tjd|j dtj |jd |j }d||j ||_ |j ddddf |jddd}|dddddf }|jj}t|tro|dkro|nd}tj|d d K| | dd} tj| | fdd d} |j|j} | dkrd} ntdt| t|j} | | } | | }Wd| |fS1swY| |fS) Nr4)rjr_rrrir#mpscpuF) device_typeenabledr$)r.maxrsrlrnfloat32r_rmrqr%rrrGrprkexpandrLtyperYstrautocastrUr]romathsqrtlogcossin)rr! position_idsseq_len ext_factorsinv_freq_shapeinv_freq_expandedposition_ids_expandedrwfreqsembscalescaling_factorrrrrr r)s0 "(      z&OmniGenSuScaledRotaryEmbedding.forward)rfrgrhN)r*r+r,rr)r0rrrr resrec @sReZdZdZddZ  d dedejdejdejdBd ejdBd ejf d d ZdS)OmniGenAttnProcessor2_0z Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is used in the OmniGen model. cCsttds tddS)Nscaled_dot_product_attentionzPAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.)hasattrF ImportErrorrrrr rs z OmniGenAttnProcessor2_0.__init__Nattnr!encoder_hidden_statesattention_maskimage_rotary_embr"cCs|j\}}}||} ||} ||} | \} } }| jd}||j}||}| |d|j|dd} | |d||dd} | |d||dd} |durgddlm }|| |dd} || |dd} t j | | | |d}|dd | }| | | |j}|jd|}|S) Nr#r4r)apply_rotary_embrX)use_real_unbind_dim) attn_maskr)rLto_qto_kto_vsizeheadsviewrU embeddingsrrrtype_asrKout_dimto_out)rrr!rrr batch_sizesequence_length_querykeyvaluebszq_len query_dim inner_dimhead_dimkv_headsrrrr __call__s(       z OmniGenAttnProcessor2_0.__call__)NN) r*r+r,__doc__rrr.r/rrrrr rs$ rc sXeZdZdedededededdf fdd Zd ejd ejd ejdejfd dZZ S) OmniGenBlockrnum_attention_headsnum_key_value_headsr rms_norm_epsr"Nc sXtt||d|_t||||||d|dtd |_t||d|_t|||_ dS)NepsF) rcross_attention_dimdim_headrrrrout_bias processor) rrrinput_layernormrr self_attnpost_attention_layernormrmlp)rrrrrrrrr rs  zOmniGenBlock.__init__r!rrcCsD||}|j||||d}||}||}||}||}|S)N)r!rrr)rrrr)rr!rrnorm_hidden_states attn_output ff_outputrrr r)s   zOmniGenBlock.forward) r*r+r,r-rGrr.r/r)r0rrrr rs.rc(s4eZdZdZdZdgZgdZe            d7dededede dededededededed ed!ed"e d#ed$ed%e d&ed'e f&fd(d) Z d*ejd+eejd,e d-ejdBfd.d/Z d8d0ejd1ee BejBd*ejd+eejd,e eeefd2ejd3ejd4e d-eeejBfd5d6ZZS)9OmniGenTransformer2DModelaF The Transformer model introduced in OmniGen (https://huggingface.co/papers/2409.11340). Parameters: in_channels (`int`, defaults to `4`): The number of channels in the input. patch_size (`int`, defaults to `2`): The size of the spatial patches to use in the patch embedding layer. hidden_size (`int`, defaults to `3072`): The dimensionality of the hidden layers in the model. rms_norm_eps (`float`, defaults to `1e-5`): Eps for RMSNorm layer. num_attention_heads (`int`, defaults to `32`): The number of heads to use for multi-head attention. num_key_value_heads (`int`, defaults to `32`): The number of heads to use for keys and values in multi-head attention. intermediate_size (`int`, defaults to `8192`): Dimension of the hidden layer in FeedForward layers. num_layers (`int`, default to `32`): The number of layers of transformer blocks to use. pad_token_id (`int`, default to `32000`): The id of the padding token. vocab_size (`int`, default to `32064`): The size of the vocabulary of the embedding vocabulary. rope_base (`int`, default to `10000`): The default theta value to use when creating RoPE. rope_scaling (`dict`, optional): The scaling factors for the RoPE. Must contain `short_factor` and `long_factor`. pos_embed_max_size (`int`, default to `192`): The maximum size of the positional embeddings. time_step_dim (`int`, default to `256`): Output dimension of timestep embeddings. flip_sin_to_cos (`bool`, default to `True`): Whether to flip the sin and cos in the positional embeddings when preparing timestep embeddings. downscale_freq_shift (`int`, default to `0`): The frequency shift to use when downscaling the timestep embeddings. timestep_activation_fn (`str`, default to `silu`): The activation function to use for the timestep embeddings. Tr)patch_embedding embed_tokensnormr2r h㈵> }@}rfrgrhNr5rsilur8r7rrrrr num_layers pad_token_id vocab_sizerors rope_basertr; time_step_dimflip_sin_to_cosdownscale_freq_shifttimestep_activation_fncst||_||_t|||d|_t||||_t|||_ t|||_ t | | |_ t| | | |d|_t fddt|D|_td|_tdddd |_t j|||jd d |_d|_dS) N)r7r8r9r;)rorsrprtcsg|] }tqSr)r).0rrrrrrrr xsz6OmniGenTransformer2DModel.__init__..rFgư>r4)norm_elementwise_affinenorm_eps chunk_dimTr)rrr8 out_channelsr1rr time_projr time_token t_embedderr Embeddingrrerope ModuleListrangelayersrrr norm_outrproj_outgradient_checkpointing)rr8r7rrrrrrrrrorsrrtr;rrrrrrr rHs:  z"OmniGenTransformer2DModel.__init__ input_idsinput_img_latentsinput_image_sizesr"c s|durdSfdd|D}|}d}j|dd}|D]}||D]\}} |||j|||| f<|d7}q'q!|S)Ncsg|]}|jqSr)r^rj)rxrrr rszHOmniGenTransformer2DModel._get_multimodal_embeddings..rTrSr4)rrkeysr^rj) rrrrcondition_tokens input_img_inxinput_image_tokensb_inx start_inxend_inxrrr _get_multimodal_embeddingss   z4OmniGenTransformer2DModel._get_multimodal_embeddingsr!timesteprr return_dictc Cs|j\} } } } |jj} | | | | }}|j|dd}|d}|||}||d}| |}| |||}|durKt j |||gdd}n t j ||gdd}|d}| d|}|dur|dkr|j}t |j}d||}|d|}|||}|jD]}t r|jr|||||}q||||d}q||}|dd| df}|j||d}||}|| ||| | d}|d d ddd d d d d d}|s|fSt|d S)NFrr4r$r#r)rr)tembrrr2)sample)rLconfigr7rrrrrrHrrr.r]rlongr%rjfinfominrris_grad_enabledr_gradient_checkpointing_funcrrrrKpermuterTr )rr!rrrrrrrr num_channelsrMrNppost_patch_heightpost_patch_widthnum_tokens_for_output_image timestep_projrrr seq_lengthrj min_dtyperblockoutputrrr r)sF          $ z!OmniGenTransformer2DModel.forward)r2rrrrrrrrrrfrgrhNr5rTrr)T)r*r+r,r _supports_gradient_checkpointing_no_split_modules _skip_layerwise_casting_patternsrr-rGdictrdr}rr.r/rZr FloatTensorr tupler)r0rrrr rs(     ;    r)#rr.torch.nnrtorch.nn.functional functionalrconfiguration_utilsrrutilsrattention_processorrrrr r modeling_outputsr modeling_utilsr normalizationr r get_loggerr*loggerModulerr1rerrrrrrr s$      X32.