o Gi @sddlZddlmZddlmZddlZddlmZddlmm Z ddl m Z m Z ddlmZmZddlmZmZddlmZd d lmZmZd d lmZmZmZd d lmZd d lm Z d dl!m"Z"m#Z#m$Z$d dl%m&Z&d dl'm(Z(d dl)m*Z*e+e,Z-ddZ.ddZ/dej0dej0fddZ1dddej0dej0fddZ2Gdd d ej3Z4Gd!d"d"Z5Gd#ddejj3eZ6Gd$d%d%ej3Z7Gd&d'd'ej3Z8eGd(d)d)ej3Z9Gd*d+d+e(e eee eZ:dS),N) lru_cache)Any) ConfigMixinregister_to_config)FromOriginalModelMixinPeftAdapterMixin)apply_lora_scalelogging)maybe_allow_in_graph)ContextParallelInputContextParallelOutput)AttentionMixinAttentionModuleMixin FeedForward)dispatch_attention_fn) CacheMixin)PixArtAlphaTextProjectionTimestepEmbedding Timesteps)Transformer2DModelOutput) ModelMixin) FP32LayerNormc Csj|j\}}}}}|\}}} ||||} ||||} | || | } tjjj|d| d| d| fddS)Nr replicate)mode)shapetorchnn functionalpad) x kernel_sizebcthwptphpwpad_tpad_hpad_wr.d/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/models/transformers/transformer_helios.pypad_for_3d_conv(s   r0cCstjjj|||dS)N)stride)rrr avg_pool3d)r!r"r.r.r/center_down_sample_3d1sr3 hidden_states freqs_ciscCs|ddd\}}|djddd\}}t|}||ddddf||ddddf|ddddf<||ddddf||ddddf|ddddf<||S) N)r6r r dim.r) unflattenunbind unsqueezechunkr empty_liketype_as)r4r5x_1x_2cossinoutr.r.r/apply_rotary_emb_transposed5s  :: rFattnHeliosAttentionencoder_hidden_statescCs|dur|}|jr,|js||jddd\}}}n!||}||jddd\}}n||}||}||}|||fS)Nrr6r8r )fused_projectionsis_cross_attentionto_qkvr>to_qto_kvto_kto_v)rGr4rIquerykeyvaluer.r.r/_get_qkv_projectionsAs     rTcsFeZdZd dededeffdd Zdejd ejd efd d Z Z S)HeliosOutputNormư>Fr9epselementwise_affinecs<tttdd||d|_t||dd|_dS)Nr:r ?FrX) super__init__r Parameterrrandnscale_shift_tablernorm)selfr9rWrX __class__r.r/r\Vs zHeliosOutputNorm.__init__r4temboriginal_context_lengthcCs|dd| dddf}|jd|j|djddd\}}|d|j|d|j}}|dd| dddf}||d|||}|S)Nrr r8r:) r_r=todevicer>squeezer`floatr@)rar4rdreshiftscaler.r.r/forward[s ,& zHeliosOutputNorm.forward)rVF) __name__ __module__ __qualname__intriboolr\rTensorrl __classcell__r.r.rbr/rUUs"rUc@sleZdZdZdZddZ    ddddejdejdBdejdBd eejejfdBd e d ejfd d Z dS)HeliosAttnProcessorNcCsttds tddS)Nscaled_dot_product_attentionzeHeliosAttnProcessor requires PyTorch 2.0. To use it, please upgrade PyTorch to version 2.0 or higher.)hasattrF ImportErrorrar.r.r/r\hs zHeliosAttnProcessor.__init__rGrHr4rIattention_mask rotary_embrereturnc Csbt|||\}}} ||}||}|d|jdf}|d|jdf}| d|jdf} |dur      6 zHeliosAttnProcessor.__call__NNNN) rmrnrorrr\rrrtuplerprr.r.r.r/rtds. rtcseZdZeZegZ          dded ed ed ed ed edBdedBffdd ZddZ e ddZ    dde j de j dBde j dBdee j e j fdBdede j f ddZZS)rH@h㈵>rNFr~r9rdim_headrWdropoutadded_kv_proj_dimcross_attention_dim_headc st|||_||_||_||_|dur|jn|||_tjj ||jdd|_ tjj ||jdd|_ tjj ||jdd|_ tj tjj |j|ddtj|g|_tjj|||dd|_tjj|||dd|_d|_|_|durtjj ||jdd|_tjj ||jdd|_tjj|||d|_| dur| |_n|du|_||| |_| r| dkrttd|_n| dkrtt||_ntd| | |_d |_dSdS) NTbias)rWrX)rWscalarr:r~zUnknown history_scale_mode: g$@)r[r\ inner_dimrrr kv_inner_dimrrLinearrMrOrP ModuleListDropoutrRMSNormrr add_k_proj add_v_proj norm_added_krK set_processorrr]onesr ValueErrorrr) rar9rrrWrrr processorrKrrrbr.r/r\sF       zHeliosAttention.__init__cCst|ddrdS|js]t|jjj|jjj|jjjg}t|jj j|jj j|jj jg}|j \}}t dt j ||dd|_Wdn1sKwY|jj||ddddnIt|jjj|jjjg}t|jj j|jj jg}|j \}}t dt j ||dd|_Wdn1swY|jj||dddd|jdurt|jjj|jjjg}t|jj j|jj jg}|j \}}t dt j ||dd|_Wdn1swY|jj||ddddd|_dS)NrJFmetaTr)weightr)strictassign)getattrrKrrrMrdatarOrPrrrgrrrLload_state_dictrNrrr to_added_kvrJ)raconcatenated_weightsconcatenated_bias out_features in_featuresr.r.r/fuse_projectionss@ ""           z HeliosAttention.fuse_projectionscCsVt|ddsdSt|drt|dt|drt|dt|dr&t|dd|_dS)NrJFrLrNr)rrvdelattrrJryr.r.r/unfuse_projectionss        z"HeliosAttention.unfuse_projectionsr4rIrzr{rer|cKs|j||||||fi|SN)r)rar4rIrzr{rekwargsr.r.r/rls zHeliosAttention.forward) rrrrNNNNFr~r)rmrnrort_default_processor_cls_available_processorsrprir\rrno_gradrrrrrlrsr.r.rbr/rHs`:# csReZdZdedededeffdd Z  dd ejd ejdBd efd d ZZ S)HeliosTimeTextEmbeddingr9 time_freq_dim time_proj_dimtext_embed_dimcsTtt|ddd|_t||d|_t|_t |||_ t ||dd|_ dS)NTr) num_channelsflip_sin_to_cosdownscale_freq_shift) in_channelstime_embed_dim gelu_tanh)act_fn) r[r\rtimesteps_projr time_embedderrSiLUrr time_projr text_embedder)rar9rrrrbr.r/r\$s  z HeliosTimeTextEmbedding.__init__NTtimesteprIis_return_encoder_hidden_statescCs|||}tt|jj}|j|kr|tjkr||}|| |}| | |}|dur9|r9| |}|||fSr) rnextiterr parametersdtyperint8rfr@rrr)rarrIrtime_embedder_dtyperd timestep_projr.r.r/rl3s     zHeliosTimeTextEmbedding.forward)NT) rmrnrorpr\rrrrqrlrsr.r.rbr/r#s&rcs^eZdZfddZddZeddZeeddd d Z ed d Z Z S) HeliosRotaryPosEmbedcslt|\|_|_|_||_|jd||jdd|jd||jdd|jd||jdddS)N freqs_base_tF) persistent freqs_base_y freqs_base_x)r[r\DTDYDXthetaregister_buffer_get_freqs_base)rarope_dimrrbr.r/r\Hs zHeliosRotaryPosEmbed.__init__cCs.d|jtjd|dtjdd|d|S)Nr}rr )r)rrarangefloat32)rar9r.r.r/rPs.z$HeliosRotaryPosEmbed._get_freqs_basecCs,td||}|jddd}||fS)Nz d,bthw->dbthwr rr8)reinsumrepeat_interleaverCrD)ra freqs_baseposfreqsr.r.r/get_frequency_batchedSsz*HeliosRotaryPosEmbed.get_frequency_batched )maxsizec CsJt|}tj||tjd}tj||tjd}tj||dd\}}||fS)Nrgrij)indexing)rrgrrmeshgrid) raheightwidth device_strrg grid_y_coords grid_x_coordsgrid_ygrid_xr.r.r/_get_spatial_meshgridYs z*HeliosRotaryPosEmbed._get_spatial_meshgridcCs|jd}|jd}|j|tjd}|||t|\}}|ddddddf||||} |ddddddf||dd} |ddddddf||dd} ||j| \} } ||j | \}}||j | \}}tj | ||| ||gdd}| dddddS) Nrr:rr6r8r r) rrfrrrstrexpandrrrrrpermute)ra frame_indicesrrrg batch_size num_framesrrgrid_t grid_y_batch grid_x_batch freqs_cos_t freqs_sin_t freqs_cos_y freqs_sin_y freqs_cos_x freqs_sin_xresultr.r.r/rlbs  $$$zHeliosRotaryPosEmbed.forward) rmrnror\rrrrrrrlrsr.r.rbr/rGs   rcseZdZ       ddededed ed ed ed edBd ededeffdd Z ddej dej dej dej dedej f ddZ Z S)HeliosTransformerBlockrms_norm_across_headsFrVNr~r9ffn_dim num_headsqk_normcross_attn_normrWrguidance_cross_attnrrc stt||dd|_t|||||dt| | d|_t||||||||td|_|r7t||ddnt |_ t ||dd|_ t||dd|_ ttdd ||d |_||_dS) NFrZ)r9rrrWrrrr)r9rrrWrrrTzgelu-approximate)r activation_fnr:rY)r[r\rnorm1rHrtattn1attn2rIdentitynorm2rffnnorm3r]rr^r_r) rar9rrrrrWrrrrrbr.r/r\ys4    zHeliosTransformerBlock.__init__r4rIrdr{rer|cCs|jdkr9|jd|jddd\}}}} } } |d}|d}|d}| d} | d} | d} n|j|jddd\}}}} } } ||d|||} || dd||} || ||}|j r|j d|}t j |||gdd\}}| ||} || |dd|} || }t j||gdd}n| ||} || |dd|} || }||d| | |} || }||| |}|S)Nrrrr r8r:)ndimr_r=rir>rhrr@rrrrsplitrrrrr)rar4rIrdr{re shift_msa scale_msagate_msa c_shift_msa c_scale_msa c_gate_msanorm_hidden_states attn_outputrhistory_hidden_states ff_outputr.r.r/rlsh            zHeliosTransformerBlock.forward)rFrVNFFr~r) rmrnrorprrqrir\rrrrlrsr.r.rbr/rwsV   7rc,seZdZdZdZgdZddgZgdZdgZdgZ de d d d d ie d d d d e d d d d de d d ddZ e                   dHeliosTransformer3DModela A Transformer model for video-like data used in the Helios model. Args: patch_size (`tuple[int]`, defaults to `(1, 2, 2)`): 3D patch dimensions for video embedding (t_patch, h_patch, w_patch). num_attention_heads (`int`, defaults to `40`): Fixed length for text embeddings. attention_head_dim (`int`, defaults to `128`): The number of channels in each head. in_channels (`int`, defaults to `16`): The number of channels in the input. out_channels (`int`, defaults to `16`): The number of channels in the output. text_dim (`int`, defaults to `512`): Input dimension for text embeddings. freq_dim (`int`, defaults to `256`): Dimension for sinusoidal time embeddings. ffn_dim (`int`, defaults to `13824`): Intermediate dimension in feed-forward network. num_layers (`int`, defaults to `40`): The number of layers of transformer blocks to use. window_size (`tuple[int]`, defaults to `(-1, -1)`): Window size for local attention (-1 indicates global attention). cross_attn_norm (`bool`, defaults to `True`): Enable cross-attention normalization. qk_norm (`bool`, defaults to `True`): Enable query/key normalization. eps (`float`, defaults to `1e-6`): Epsilon value for normalization layers. add_img_emb (`bool`, defaults to `False`): Whether to use img_emb. added_kv_proj_dim (`int`, *optional*, defaults to `None`): The number of channels to use for the added key and value projections. If `None`, no projection is used. T)patch_embedding patch_short patch_mid patch_longcondition_embedderr`rrU)rr_rrrr norm_added_qr4r:rF) split_dim expected_dims split_outputr)rdr{) gather_dimr3)zblocks.0zblocks.*z blocks.39r:r r (6rrVN,*r?@r~ patch_size.num_attention_headsattention_head_dimr out_channelstext_dimfreq_dimr num_layersrrrWrr rope_thetarzero_history_timestephas_multi_term_memory_patchrrr|c s2t||p |}t||d|_tj|||d|_||_|r]tj|||d|_tj|t dd|Dt dd|Dd|_ tj|t dd|Dt dd|Dd|_ t |d|d |_ t f d d t| D|_td d |_t|t||_d |_dS)N)rr)r"r1cs|]}d|VqdSr Nr..0pr.r.r/ az4HeliosTransformer3DModel.__init__..csrKrLr.rMr.r.r/rPbrQcsrKrNr.rMr.r.r/rPgrQcsrKrRr.rMr.r.r/rPhrQr)r9rrrcs(g|]}t d qS))rrr)r)rN_ rrrWrrrrrrBrr.r/ us z5HeliosTransformer3DModel.__init__..FrZ)r[r\rroperConv3dr,rIr-rr.r/rr0rrangeblocksrUnorm_outrmathprodproj_outgradient_checkpointing)rarArBrCrrDrErFrrGrrrWrrrHrrIrJrrrbrTr/r\9sD   z!HeliosTransformer3DModel.__init__attention_kwargsrrI return_dictc % Csv|jd} |jj\}}}||}|j\}}}}}|dur*td|d| d}|d dd}|j ||||j d}|d dd}|jd}|dur|dur| |}|j\}}}}}|d dd}|j ||||j d}|d dd}tj ||gdd}tj ||gdd}| dur|durt| d} || } | d dd} |j |||| j d}t|d}t|d}|d dd}tj | |gdd}tj ||gdd}| dur$|dur$t| d } || } | d dd} |j |||| j d}t|d }t|d }|d dd}tj | |gdd}tj ||gdd}|jd|}|dure|jretjd|j|j d }|j||d d \}}}|d| |d}|dddddd| d|d}|||\} }!}|!dd}!|dur|js|jd}"n|}"| | dd| |"d} |!| ddd| d|"d}!|dur|jrtj || gdd} tj ||!gdd}!|!jdkr|!dddd}!|}|}|}tr|jr|jD] }#||#|||!||}qn|jD] }#|#|||!||}q||| |}| |}|!| ||||||d}|dddddddd}|dddddd}$| s6|$fSt"|$dS)Nrr6r r:)rrrrgr8)r rr)r r r )rrr)rrr)rrgF)r)rr6rrr)sample)#rconfigrAr,rrr=rr transposerVrgr-rr0r.r3r/rIzerosrr0r;rrr contiguousis_grad_enabledr^rY_gradient_checkpointing_funcrZr]reshaper)%rar4rrIindices_hidden_statesindices_latents_history_shortindices_latents_history_midindices_latents_history_longlatents_history_shortlatents_history_midlatents_history_longr`r_rp_tp_hp_wrSpost_patch_num_framespost_patch_heightpost_patch_widthr{reH1W1rotary_emb_history_shortrotary_emb_history_midrotary_emb_history_longhistory_context_length timestep_t0temb_t0timestep_proj_t0rdrmain_repeat_sizeblockoutputr.r.r/rls                       z HeliosTransformer3DModel.forward)r6r7r8r9r9r:r;r<r7TrrVNr=r@TTTFr~) NNNNNNNTN)rmrnro__doc__ _supports_gradient_checkpointing _skip_layerwise_casting_patterns_no_split_modules_keep_in_fp32_modules"_keys_to_ignore_on_load_unexpected_repeated_blocksr r_cp_planrrrprqrrir\r rrr LongTensordictrrlrsr.r.rbr/r+s$           R r+);r[ functoolsrtypingrrtorch.nnrtorch.nn.functionalrrwconfiguration_utilsrrloadersrrutilsr r utils.torch_utilsr _modeling_parallelr r attentionrrrattention_dispatchr cache_utilsr embeddingsrrrmodeling_outputsrmodeling_utilsr normalizationr get_loggerrmloggerr0r3rrrFrTModulerUrtrHrrrr+r.r.r.r/sL             <$0 {