o پi@sddlZddlmZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZddlmZddlmZmZddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/d Z0ej1d d d Z2ej1ddd Z3ej1ddd Z4ej1ddd Z5ee2e3e4e0e5e d+ddZ6Gdddej7Z8Gdddej7Z9Gdddej7Z:Gdddej7Z;Gdddej7ZGd!d"d"ej7Z?Gd#d$d$ej7Z@d%d&ZAd'd(ZBGd)d*d*eZCeCZDdS),N)contextmanager) rearrange) WanVAEConfig)get_sp_parallel_rankget_sp_world_size) get_act_fn)DiagonalGaussianDistributionParallelTiledVAE) AvgDown3DDupUp3DWanCausalConv3d WanRMS_norm WanUpsampleattention_block_forward bind_contextmid_block_forwardresample_forwardresidual_block_forwardresidual_down_block_forwardresidual_up_block_forwardup_block_forward) WanDistAttentionBlockWanDistCausalConv3dWanDistMidBlockWanDistResampleWanDistResidualBlockWanDistResidualDownBlockWanDistResidualUpBlockWanDistUpBlockensure_local_heightgather_and_trim_heightsplit_for_parallel_decodesplit_for_parallel_encodeis_first_frameF)default feat_cachefeat_idx first_chunkc cst|}t|}t|}t|}zdVWt|t|t|t|dSt|t|t|t|wN)r$setr&r'r(reset)first_frame_argfeat_cache_arg feat_idx_argfirst_chunk_argis_first_frame_tokenfeat_cache_tokenfeat_idx_tokenfirst_chunk_tokenr4d/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/runtime/models/vaes/wanvae.pyforward_contextMs          r6c s<eZdZdZd dedededdffdd Zd d ZZS) WanResampleax A custom resampling module for 2D and 3D data. Args: dim (int): The number of input/output channels. mode (str): The resampling mode. Must be one of: - 'none': No resampling (identity operation). - 'upsample2d': 2D upsampling with nearest-exact interpolation and convolution. - 'upsample3d': 3D upsampling with nearest-exact interpolation, convolution, and causal 3D convolution. - 'downsample2d': 2D downsampling with zero-padding and convolution. - 'downsample3d': 3D downsampling with zero-padding, convolution, and causal 3D convolution. Ndimmodeupsample_out_dimreturnc st||_||_|dur|d}|dkr+ttdddtj||ddd|_dS|d krNttdddtj||ddd|_t ||dd d d|_ dS|d krett d tj||ddd|_dS|dkrtt d tj||ddd|_t ||d ddd|_ dSt |_dS)Nr# upsample2d)@r=z nearest-exact) scale_factorr9padding upsample3d)r?r@r@)r@rr downsample2d)rr@rr@)r#r#)stride downsample3d)r#r@r@)rrr)rErB) super__init__r8r9nn SequentialrConv2dresampler time_conv ZeroPad2dIdentity)selfr8r9r: __class__r4r5rHls8       zWanResample.__init__cC t||Sr))rrPxr4r4r5forward zWanResample.forwardr)) __name__ __module__ __qualname____doc__intstrrHrV __classcell__r4r4rQr5r7^s %r7c sDeZdZdZ  ddededededd f fd d Zd d ZZ S)WanResidualBlockaE A custom residual block module. Args: in_dim (int): Number of input channels. out_dim (int): Number of output channels. dropout (float, optional): Dropout rate for the dropout layer. Default is 0.0. non_linearity (str, optional): Type of non-linearity to use. Default is "silu". siluin_dimout_dimdropout non_linearityr;Ncst||_||_t||_t|dd|_t||ddd|_ t|dd|_ t ||_ t||ddd|_||krCt||d|_dSt |_dS)NFimagesr?r@rA)rGrHrbrcr nonlinearityr norm1r conv1norm2rIDropoutrdconv2rO conv_shortcut)rPrbrcrdrerQr4r5rHs   zWanResidualBlock.__init__cCrSr))rrTr4r4r5rVrWzWanResidualBlock.forward)r`ra rXrYrZr[r\floatr]rHrVr^r4r4rQr5r_s r_cs*eZdZdZdfdd ZddZZS) WanAttentionBlockz} Causal self-attention with a single head. Args: dim (int): The number of channels in the input tensor. r;NcsBt||_t||_t||dd|_t||d|_dS)Nr?r@) rGrHr8r normrIrKto_qkvproj)rPr8rQr4r5rHs  zWanAttentionBlock.__init__cCrSr))rrTr4r4r5rVrWzWanAttentionBlock.forwardr;N)rXrYrZr[rHrVr^r4r4rQr5rqs rqc sBeZdZdZ   d dedededeffd d Zd d ZZ S) WanMidBlockz Middle block for WanVAE encoder and decoder. Args: dim (int): Number of input/output channels. dropout (float): Dropout rate. non_linearity (str): Type of non-linearity to use. r`rar@r8rdre num_layerscsvt||_t||||g}g}t|D]}|t||t||||qt||_ t||_ d|_ dS)NF) rGrHr8r_rangeappendrqrI ModuleList attentionsresnetsgradient_checkpointing)rPr8rdrerwr|r{_rQr4r5rHs     zWanMidBlock.__init__cCrSr))rrTr4r4r5rVrWzWanMidBlock.forward)r`rar@ror4r4rQr5rvs rvcs*eZdZ  dfdd ZddZZS)WanResidualDownBlockFc stt|||r dnd|rdndd|_g}t|D] }|t||||}qt||_ |rA|r6dnd} t || d|_ dSd|_ dS)Nr#r@factor_tfactor_srFrDr9) rGrHr avg_shortcutrxryr_rIrzr|r7 downsampler) rPrbrcrdnum_res_blockstemperal_downsample down_flagr|r~r9rQr4r5rHs       zWanResidualDownBlock.__init__cCrSr))rrTr4r4r5rVrWzWanResidualDownBlock.forward)FF)rXrYrZrHrVr^r4r4rQr5rs !rc ReZdZdZ        dd ed ededeffdd ZddZZ S) WanEncoder3dao A 3D encoder module. Args: dim (int): The base number of channels in the first layer. z_dim (int): The dimensionality of the latent space. dim_mult (list of int): Multipliers for the number of channels in each block. num_res_blocks (int): Number of residual blocks in each block. attn_scales (list of float): Scales at which to apply attention mechanisms. temperal_downsample (list of bool): Whether to downsample temporally in each block. dropout (float): Dropout rate for the dropout layers. non_linearity (str): Type of non-linearity to use. r?r@r#rrr#r4TTFr`raF in_channelsre is_residualuse_parallel_encodec  sFt|_||_t|}||_||_t||_t||_t | |_ | |_ t t |dd|_fdddg|D} d} d}trHt}| r[|dkr[t}t}t}t}t}t}n t}t}t}t}t}t}||| dddd|_t g|_!t"t#| dd| ddd d D]f\}\}}| r|j!$||||||t |dkr||nd |t |dkd qt%|D]}|j!$||||| |vr|j!$|||}q|t |dkr||rd nd}|j!$|||d| d} q|||| dd|_&t'|d d|_(|||ddd|_)d |_*d|_+d|_,tr!t|_+t-|_,dSdS)Nr@rcg|]}|qSr4r4.0ur8r4r5 Cz)WanEncoder3d.__init__..?r?rATstrictF)rrrFrDrr=rwrf).rGrHr8z_dimlistdim_multr attn_scalesrrrhrmaxlendownsample_countdistis_initializedrrrrrrrr rr_rqr7rvconv_inrIrz down_blocks enumeratezipryrx mid_blockr norm_outconv_outr} world_sizerankr)rPrr8rrrrrrdrerrdimsscaler CausalConv3dResidualDownBlock ResidualBlockAttentionBlockResampleMidBlockirbrcr~r9rQrr5rH(s~      .   zWanEncoder3d.__init__c Csnd}d}|jr|jdkrt||j|j|j\}}}t}t}|dur|}|ddddt dddddf }|j ddkrk||durkt j ||dddddddddf d|j|gdd}||||}|||<|d7}t|t|n||}|jD]}||}q|jr|jdkrt||}||}||}||}t}t}|dur!|}|ddddt dddddf }|j ddkr||durt j ||dddddddddf d|j|gdd}||||}|||<|d7}t|t|n||}|jr5|jdkr5t||}|SNr@r#rr)rrr"rrr&getr'CACHE_Tcloneshapetorchcat unsqueezetodevicerr*rrrrrhrr ) rPrUexpected_local_heightexpected_height _feat_cache _feat_idxidxcache_xlayerr4r4r5rVsp ,$           ,$     zWanEncoder3d.forward) r?rrrr#r4rr`raFF) rXrYrZr[r\r]boolrHrVr^r4r4rQr5rs.   `rcsPeZdZdZ    ddedededed ed ed effd d ZddZ Z S)WanResidualUpBlocka A block that handles upsampling for the WanVAE decoder. Args: in_dim (int): Input dimension out_dim (int): Output dimension num_res_blocks (int): Number of residual blocks dropout (float): Dropout rate temperal_upsample (bool): Whether to upsample on temporal dimension up_flag (bool): Whether to upsample or not non_linearity (str): Type of non-linearity to use r`Frarbrcrrdtemperal_upsampleup_flagrec st||_||_|rt|||rdnddd|_nd|_g}|} t|dD]} |t| ||||} q(t ||_ |rN|rCdnd} t || |d|_ nd|_ d|_dS)Nr#r@rrCr<)r9r:F)rGrHrbrcr rrxryr_rIrzr|r7 upsamplerr}) rPrbrcrrdrrrer| current_dimr~ upsample_moderQr4r5rHs4        zWanResidualUpBlock.__init__cCrSr))rrTr4r4r5rVrWzWanResidualUpBlock.forward)r`FFra) rXrYrZr[r\rprr]rHrVr^r4r4rQr5rs,.rc sNeZdZdZ   ddedededed edBd ef fd d Zd dZZ S) WanUpBlocka A block that handles upsampling for the WanVAE decoder. Args: in_dim (int): Input dimension out_dim (int): Output dimension num_res_blocks (int): Number of residual blocks dropout (float): Dropout rate upsample_mode (str, optional): Mode for upsampling ('upsample2d' or 'upsample3d') non_linearity (str): Type of non-linearity to use r`Nrarbrcrrdrrec st||_||_g}|}t|dD]} |t|||||}qt||_ d|_ |dur.rr@r?rArTrr#rCr<F)rbrcrrdrrre)rbrcrrdrrerf)(rGrHr8rrrrrrrrhrupsample_countrrrrrrrr rvrrrrrIrz up_blocksrrrryr rrr}rrr)rPr8rrrrrrdrerrrrrrrResidualUpBlockUpBlockrrbrcrrup_blockrQrr5rHRs    &  .      zWanDecoder3d.__init__cCsJd}|jr|jdkrt||j|j|j\}}t}t}|dur|}|ddddt dddddf }|j ddkrh||durht j ||dddddddddf d|j|gdd}||||}|||<|d7}t|t|n||}||}|jD]}||}q||}||}t}t}|dur|}|ddddt dddddf }|j ddkr||durt j ||dddddddddf d|j|gdd}||||}|||<|d7}t|t|n||}|jr#|jdkr#t||}|Sr)rrr!rrr&rr'rrrrrrrrrr*rrrrhrr )rPrUrrrrrrr4r4r5rVsj,$          ,$     zWanDecoder3d.forward) rrrr#r4rr`rar?FF) rXrYrZr[r]r\rrHrVr^r4r4rQr5rCs.    hrcCs\|dkr|S|dkrt|d||d}|S|dkr&t|d||d}|Std|j)Nr@rz b c (h q) (w r) -> b (c r q) h wqrz$b c f (h q) (w r) -> b (c r q) f h wzInvalid input shape: )r8r ValueErrorrrU patch_sizer4r4r5patchifys   rcCsL|dkr|S|dkrt|d||d}|S|dkr$t|d||d}|S)Nr@rz b (c r q) h w -> b c (h q) (w r)rrz$b (c r q) f h w -> b c f (h q) (w r))r8rrr4r4r5 unpatchifys  rc @s$eZdZdZdZdeddfddZd#dd Zd ej dej fd d Z d$d ej dej fd dZ d ej dej fddZ d ej dej fddZ dej dej fddZd$dej dej fddZdej dej fddZdej dej fddZdejdejfddZ  d%dej ded ejdBdej fd!d"ZdS)&AutoencoderKLWanz A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos. Introduced in [Wan 2.1]. Fconfigr;Nc Cs@tj|t|||j|_t|j|_t|jddd|_|jdur*|j }n|j}t|j |_ t|j |_ |j |_ t |dd|_t |dd|_|jrit|j|j |jd|j|j|j|j|j|j|jd |_t|jd|jdd|_t|j|jd|_|jrt||j|j|j|j|j|j|j|j|jd |_|j |_ dS) NrrFrr#) rr8rrrrrrdrrr@) r8rrrrrrdrrr)!rIModulerHr rrrrdecoder_base_dimbase_dim latents_mean latents_std shift_factorgetattrrr load_encoderrrrrrrdrencoderr quant_convpost_quant_conv load_decoderrrdecoderuse_feature_cache)rPrrr4r4r5rH-sT        zAutoencoderKLWan.__init__cCsfdtfdd}|jjr||j|_d|_dg|j|_|jjr1||j|_ d|_ dg|j |_ dSdS)Nr;cSs2d}|D]}t|tst|tr|d7}q|S)Nrr@)modules isinstancer r)modelcountmr4r4r5 _count_conv3des  z3AutoencoderKLWan.clear_cache.._count_conv3dr) r\rrr _conv_num _conv_idx _feat_maprr _enc_conv_num _enc_conv_idx _enc_feat_map)rPrr4r4r5 clear_cachecs  zAutoencoderKLWan.clear_cacherUc Cs|jr||jjdurt||jjd}t|j|jdd|jd}d|dd}t |D]J}t d|dkrQ| |ddddddddddf}q/| |dddddd|ddd|ddddf}t ||gd}q/Wdn1swY||}|ddd|jddddddf|dd|jdddddddf}} t j|| gdd}t|}||S|j jD]} t| tr| jdkrt| jj} d| d<t| | j_qt||}|S) Nrr-r.r#r@rrrrF)rrrrrr6rrrrxr'r*rrrrrrrrr7r9rrM_paddingtupler encode) rPrUtiter_routout_encmulogvarblockrr4r4r5rvs>    .@ R    zAutoencoderKLWan.encodecCst|d ||}Wdn1swY||}|ddd|jddddddf|dd|jdddddddf}}tj||gdd}|S)Nr,r@r)r6rrrrr)rPrU first_framer r r r r4r4r5_encodes   RzAutoencoderKLWan._encodecCr|dddddddddfd}|j|dd}t||}|ddddddf}tj||gdd}|SNrr#T)rr@r)rrr tiled_encoderrrPrUrr r4r4r5r ( zAutoencoderKLWan.tiled_encodecCrr)rrr spatial_tiled_encoderrrr4r4r5rrz%AutoencoderKLWan.spatial_tiled_encodezc CsH|jr||jd}||}t|j|jd[t|D]N}t d|dkrGt d| |dddd||dddddf}qt d| |dddd||dddddf}t ||gd}qWdn1swwY|jjdurt||jjd}|}t j|dd d }||St||}|S) Nr#rrTr@Frrminr)rrrrr6rrrxr'r*r(rrrrrrrpclampr decode)rPrrrUrr r r4r4r5rs2     2 0  zAutoencoderKLWan.decodecCsR||}t|d ||}Wdn1swYtj|ddd}|S)Nrrrr)rr6rrr)rPrrrUr r4r4r5_decodes   zAutoencoderKLWan._decodecCB|jd9_t||}|jd}|dddd|df}|SNr#r@)blend_num_framesr tiled_decodetemporal_compression_ratiorPrdecstart_frame_idxr4r4r5r"   zAutoencoderKLWan.tiled_decodecCs4t||}|jd}|dddd|df}|S)Nr@)r spatial_tiled_decoder#r$r4r4r5r(s  z%AutoencoderKLWan.spatial_tiled_decodecCrr )r!r parallel_tiled_decoder#r$r4r4r5r)r'z&AutoencoderKLWan.parallel_tiled_decodesamplesample_posterior generatorcCs8|}||j}|r|j|d}n|}||}|S)z Args: sample (`torch.Tensor`): Input sample. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`DecoderOutput`] instead of a plain tuple. )r,)r latent_distr*r9r)rPr*r+r,rU posteriorrr%r4r4r5rVs  zAutoencoderKLWan.forwardru)F)FN)rXrYrZr[ _supports_gradient_checkpointingrrHrrTensorrrrrrrr"r( FloatTensorr)r GeneratorrVr4r4r4r5r%s: 6     r)FNNN)E contextvars contextlibrrtorch.distributed distributedrtorch.nnrIeinopsr)sglang.multimodal_gen.configs.models.vaesr8sglang.multimodal_gen.runtime.distributed.parallel_staterr/sglang.multimodal_gen.runtime.layers.activationr0sglang.multimodal_gen.runtime.models.vaes.commonrr Csglang.multimodal_gen.runtime.models.vaes.parallel.wan_common_utilsr r r r rrrrrrrrrAsglang.multimodal_gen.runtime.models.vaes.parallel.wan_dist_utilsrrrrrrrrrr r!r"r ContextVarr$r&r'r(r6rr7r_rqrvrrrrrrrr EntryClassr4r4r4r5sJ      <8 7%#';?1?\