o
    ۷i                     @  s  d dl mZ d dlZd dlZd dlmZ d dlm  mZ ddl	m
Z
mZ ddlmZ ddlmZ ddlmZ dd	lmZ d
dlmZmZmZ eeZg dZg dZeddgeddgdZG dd dejZG dd dejj Z!G dd dej Z"G dd dej Z#G dd dej Z$G dd dej Z%G dd dej Z&G dd  d ej Z'G d!d" d"ej Z(G d#d$ d$Z)G d%d& d&Z*G d'd( d(ej Z+G d)d* d*ej Z,G d+d, d,ej Z-G d-d. d.ej Z.G d/d0 d0ej Z/G d1d2 d2eee
Z0dS )3    )annotationsN   )ConfigMixinregister_to_config)
get_logger)apply_forward_hook   )AutoencoderKLOutput)
ModelMixin   )AutoencoderMixinDecoderOutputIdentityDistribution(   g4t?g}Gug#әr?gչ?gF%.2Ǔ?g ?gu	$'y?g
ˆj?g&.*?gij'-?g	ف4?g癖?g1X?gj!?g	3,?g.$-Ĕ?g5	{?g;,T?g 
~?g=-7?gTjݣ$?g,l?gM53ݨ?gJF?g!!%?gnK?ggI?gg=c?g9m?g԰?gJHZ?go @?g7?goS?gJK?ge:Ɯ?gѴ@?gfTF?g #?g~[?gJn^慅?gCo?g<
~?gڪ?g?gϾ,̠?guס?g'#J9?g2`gJ?gJvSgAj?gsQ?gO?gQgQL?g
$
?g^_>?g=Y{7]?g%5?gdH_RƧ?g+>?gU?gC?g"~͋?gCfx	?g2gkgg[Yͅg窳9gޑ녆g8&g-!g
Tgc#듿gZgprg2ϙg9Qygeѡۘgq!mFzg{nHgY[N?g)WgS
h.`g=J$g[<箿gTgQgÙgzR gDԷ骿gW5_g˺V﫿gW'g(xg<gg$Tg}bs?gϹJIf?gʱicogaHl?gI.A?gi6xx2?g\?g؂9գ?gy@vjp?gBVP?gDEk?gƵ?gbEO3?g'".H?g$Pq[?gzة?g%dsY?g'#|?g]XP?gO(r?g~l?g9ݪҳ?gm,´?g%@?gMIcQd?ga\?gm`?g`?g	^K%?gcCbӳ?gWƳ?gw=$?g^A0t?g<Mg`}T?gஅgOgFۆg,Yg[ݐg4UAVgѢF:g	g$竁gXWgXbjgcbgc9gg`wg8@?g=C?g<q?g6*	?g!6X?gMܹQ?gt7?g@?g?ge؋?gZSv%?g˸-?gEi?gkn[?giUH?g ?g]ӶHg̸gE}q-ggzdgf{xgCg>gRLge,gnXzgޱg33'g{_g66gklFfg~I+g)݆t8g[mNg-RQg`g;P]g~U7,pg+2gkog(Jg
v'gR~Fg\T(rgKH`gV3(gX]g>(ſg_rH?gVW̤?gIG?gw?g2O?gK?gTA
}?g#l4?gaVL?g|__Ş?g+!b?g1?gc=2)?g8pN?g?g[H|gd*"TggAGg9[av&ggR|s.g
1.g㪿g-J?DgOYggno g'@Pg҂@gJZ᧧gC¨g##?g~"G!f?gGƝgᄕLFg:&GgA?#g0æg6,>"ئg)U g.3Ͼgՙg]p멿gt5l)Zg	bfէg3樿gAKi ʨg ?g,wn)tg;VEg;!	5g3CgIɛ{gƹguMؤgm2g{gҚ̀g,DEgUslڵggeY]g/`䣿(   g0$?g6D~?gxG ?gs"I?g+!~Q?g4uhU?g&|ⵁ?g$m?gB=j?g)n?y?gZT8@8x?gE>3?gl1?gz[#ac?g&߮?g{W?gWGG?g(p?g8!?gMN?gd?g |?g%r?gl??gEeT?g`?gǢ?gm??gAvA?g]@?J?gu!?gn|)?gw3/?g y?gv?gg~N?gHA]?g@=?gڶ%?g7?g/w?gyޕ?g%Uf?g ?g4x"&2?gp_8?g%ϳafB?g^.^M?gnˀf?g6??gr_!s?gЎ?gda?g9B66?g]o!U5?g}a!g?gWRա&|?gZua?g#?gȟ?gd\=?gYfQ?gNl ?g9B6?g@ _?g1	8 ?g776?gqI?g4?gL?g>`#?g=1_?g?g"?g9w`G?g<?g]f@b@?g^cf?g j?g[U}g?g<^?gݎ}?g)>i?gi^a:?g	j[AV?guU^p?g9{>?g퀈`^?gR4A?g;!	?g??g|)=?g&!a{?gT
O"?g	?gՇ?gk82?g6fR?gx*o?goW՜?g,g~5?g7b?g ?gn)?gC~
6?gHUC?ggB}D?g@!R?g5T?g)0W?gx?gs0 R?gHDz?gmc?g1!$?g&G*!`?g/Qv?g?gh0x^K?gʑA?g%6a?g_T?g?gEr"?gr",?g%Z?g2Rs@?g2?ge]//?g_?g,n?gٍ_?ge)w?ga:?g/@0?g8a?g{ ?gG?gܿ}.?g +?gfޢ?gX?gsG?g>?gbL?gV?gox'?ggS~?gڊ?g@#)?gao?g/?g(bS?g̲'?g6>?g8^)/?gjxb?gπ8W?gO4r?g`?g[?g%L/?g?g贗?g?gܽy?g?g~?gnm]?g
?gc: ?g(?g򸠥(?g%?g:T^"?gy3E?g?gD?gLՇ??g:B?g.S?g?g?gj5ȑ?g[F̌?gz,!?gǸ?gJ"?g+~?g}R7a?gO_?gO;>R?gP]?gߑ?gdֽ?g=I?g#ܞb?g3t8?g^b?g,?g̥)^?gԲ!|?g)j/n?gB_?gף.!8?gʇↆ?gR?g3!!b?g_ ?gLsD!vd?gd|?gmA?g>[?gA~8?gNE	~Da?gٰo?gK?g[B?gA`a?gc>A?gBӯ ?gM?g?gF]0$?g*ڠ;?g8m?g$k?g=mB?g]]>'w?gg>N?g1?gb?gIK"?gi?gW
?@?gY8Q#?gn?l-?gzX^2?gpp?gQ*v?g̀u?gc-?gA.?gZ2?g_JUEZ?g]$rw?gi}m?gbͽn?g_?gg7
?g>(޷?g F?g ?g(_?g8+&?gSa?g?g;f?      ?)haar	rearrangec                      s<   e Zd Z							dd fddZd fddZ  ZS )CosmosCausalConv3dr   r   r   r   r   r   r   constantin_channelsintout_channelskernel_sizeint | tuple[int, int, int]dilationstridepaddingpad_modestrreturnNonec                   s   t |tr
|||fn|}t |tr|||fn|}t |tr"|||fn|}|\}}	}
|	d dkr5|
d dks7J t j|||||d || _|d |d d  d|d   | _||||f| _d S )Nr   r   )r   r   r   )
isinstancer   super__init__r   temporal_padspatial_pad)selfr   r   r   r   r   r   r   _height_kernel_sizewidth_kernel_size	__class__ i/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/models/autoencoders/autoencoder_kl_cosmos.pyr$   /   s   

"zCosmosCausalConv3d.__init__hidden_statestorch.Tensorc                   sn   |d d d d d ddf  dd| jdd}tj||gdd}tj|g | jddR | jdd}t 	|S )Nr   .r   dimr           modevalue)
repeatr%   torchcatFpadr&   r   r#   forward)r'   r/   hidden_states_prevr+   r-   r.   r<   L   s   ,$zCosmosCausalConv3d.forward)r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r/   r0   r    r0   __name__
__module____qualname__r$   r<   __classcell__r-   r-   r+   r.   r   .   s    r   c                      s*   e Zd Zdd fddZdd
dZ  ZS )CosmosCausalGroupNormr   r   r   
num_groupsc                   s(   t    tj||ddd| _|| _d S )Ngư>T)rE   num_channelsepsaffine)r#   r$   nn	GroupNormnormrE   )r'   r   rE   r+   r-   r.   r$   T   s   

zCosmosCausalGroupNorm.__init__r/   r0   r    c                 C  sh   | j dkr-|d}|ddddddd}| |}|d|dfddddd}|S | |}|S )Nr   r   r   r      )rE   sizepermuteflattenrK   	unflatten)r'   r/   
batch_sizer-   r-   r.   r<   ^   s   




zCosmosCausalGroupNorm.forwardr   )r   r   rE   r   r>   r?   r-   r-   r+   r.   rD   S   s    
rD   c                      sJ   e Zd Zdd fd	d
ZddddZdddZdddZdddZ  ZS )CosmosPatchEmbed3dr   r   
patch_sizer   patch_methodr   r    r!   c                   X   t    || _|| _t| }t|j	d }| j
d|dd | j
d|dd d S Nr   waveletsF)
persistent_aranger#   r$   rU   rV   	_WAVELETSgetcloner8   arangeshaperegister_bufferr'   rU   rV   rY   r`   r+   r-   r.   r$   l      
zCosmosPatchEmbed3d.__init__reflectFr/   r0   r5   c              	   C  s  |j }| j}|jd }|jd }|dddd|dd}|d| j  ddd|dd}	|	j|d}	|j|d}tj	|t
d|d |d |d |d |d |d f|d|}tj||dd|d	d
}
tj||	dd|d	d
}tj|
|dd|dd
}tj|
|	dd|dd
}tj||dd|dd
}tj||	dd|dd
}tj||dd|dd
}tj||	dd|dd
}tj||dd|dd
}tj||	dd|dd
}tj||dd|dd
}tj||	dd|dd
}tj||dd|dd
}tj||	dd|dd
}tj||||||||gdd}|rD|d }|S )Nr   r   rM   dtyper   )r;   r5   r   rL   r   r   r   groupsr   r   r   r   r   r   r   r1   ;f@)rg   rY   ra   flipreshaper7   r[   tor:   r;   maxconv3d	unsqueezer8   r9   )r'   r/   r5   rescalerg   rY   nghlhhxlxhxllxlhxhlxhhxlllxllhxlhlxlhhxhllxhlhxhhlxhhhr-   r-   r.   _dwtx   s:   

":zCosmosPatchEmbed3d._dwtc                 C  sl   t j|d|jd d gdd\}}t j|j| jdd|gdd}ttt	| jD ]	}| j
|dd}q*|S )Nr   r   r1   Trt   )r8   splitra   r9   repeat_interleaverU   ranger   mathlog2r   )r'   r/   xixvr(   r-   r-   r.   _haar   s
   "zCosmosPatchEmbed3d._haarc           
   
   C  s   t j|d|jd d gdd\}}t j|j| jdd|gdd}|j\}}}}}| j}	|||||	 |	||	 |	||	 |	}|dddddddd	dd	 }|S )
Nr   r   r1   r   r         rL      )
r8   r   ra   r9   r   rU   ro   rO   rP   
contiguous)
r'   r/   r   r   rR   rF   
num_framesheightwidthpr-   r-   r.   _arrange   s   "$zCosmosPatchEmbed3d._arrangec                 C  s8   | j dkr
| |S | j dkr| |S td| j  )Nr   r   zUnsupported patch method: )rV   r   r   
ValueErrorr'   r/   r-   r-   r.   r<      s
   



zCosmosPatchEmbed3d.forwardr   r   )rU   r   rV   r   r    r!   )re   F)r/   r0   r5   r   r    r0   r>   )	r@   rA   rB   r$   r   r   r   r<   rC   r-   r-   r+   r.   rT   k   s    
&
rT   c                      sJ   e Zd Zdd fddZddddZdddZdddZdddZ  ZS )CosmosUnpatcher3dr   r   rU   r   rV   r   c                   rW   rX   r\   rc   r+   r-   r.   r$      rd   zCosmosUnpatcher3d.__init__Fr/   r0   rt   boolr    c                 C  sz  |j }|j}| j|}|jd d }|dgddd|ddg}|d| j|  ddd|dd}|j|d}|j|d}t	j
|ddd\}	}
}}}}}}tj|	|dd|d	d
}tj|
|dd|d	d
| }tj||dd|d	d
}tj||dd|d	d
| }tj||dd|d	d
}tj||dd|d	d
| }tj||dd|d	d
}tj||dd|d	d
| }tj||dd|dd
}tj||dd|dd
| }tj||dd|dd
}tj||dd|dd
| }tj||dd|dd
}tj||dd|dd
| }|r;|d }|S )Nr      r   rM   rf   r1   r   r   rl   ri   rL   rk   rh   rm   )devicerg   rY   rp   ra   rn   ro   r7   r[   r8   chunkr:   conv_transpose3drs   )r'   r/   rt   r   rg   hrv   rw   rx   r   r   r   r   r   r   r   r   r{   r|   r}   r~   ry   rz   r-   r-   r.   _idwt   s6   "( """""" zCosmosUnpatcher3d._idwtc                 C  sN   t tt| jD ]	}| j|dd}q
|d d d d | jd d df }|S )NTr   r   .)r   r   r   r   rU   r   )r'   r/   r(   r-   r-   r.   _ihaar   s   "zCosmosUnpatcher3d._ihaarc              
   C  sr   | j }|dd|||f}|dddddddd	}|ddd	ddd}|d d d d |d d d
f }|S )Nr   rM   r   r   r   r   r   r   rL   .)rU   rQ   rO   rP   )r'   r/   r   r-   r-   r.   _irearrange   s    zCosmosUnpatcher3d._irearrangec                 C  s6   | j dkr
| |S | j dkr| |S td| j  )Nr   r   zUnknown patch method: )rV   r   r   r   r   r-   r-   r.   r<     s
   



zCosmosUnpatcher3d.forwardr   )rU   r   rV   r   )F)r/   r0   rt   r   r    r0   r>   )	r@   rA   rB   r$   r   r   r   r<   rC   r-   r-   r+   r.   r      s    
+
r   c                      s(   e Zd Zd fddZdd
dZ  ZS )CosmosConvProjection3dr   r   r   r    r!   c                   s6   t    t||dddd| _t||dddd| _d S )Nr   r   r   r   r   r   r   r   r   r   r   )r#   r$   r   conv_sconv_t)r'   r   r   r+   r-   r.   r$     s   
zCosmosConvProjection3d.__init__r/   r0   c                 C  s   |  |}| |}|S N)r   r   r   r-   r-   r.   r<        

zCosmosConvProjection3d.forward)r   r   r   r   r    r!   r>   r?   r-   r-   r+   r.   r   
  s    r   c                      s.   e Zd Z		dd fddZdddZ  ZS )CosmosResnetBlock3dr3   r   r   r   r   dropoutfloatrE   r    r!   c                   s|   t    |p|}t||| _t||| _t||| _t|| _	t||| _
||kr7t||dddd| _d S t | _d S )Nr   r   r   )r#   r$   rD   norm1r   conv1norm2rI   Dropoutr   conv2r   conv_shortcutIdentity)r'   r   r   r   rE   r+   r-   r.   r$     s   
zCosmosResnetBlock3d.__init__r/   r0   c                 C  s\   |}|  |}| |}t|}| |}| |}t|}| |}| |}|| S r   )r   r   r:   silur   r   r   r   )r'   r/   residualr-   r-   r.   r<   .  s   







zCosmosResnetBlock3d.forward)r3   r   )
r   r   r   r   r   r   rE   r   r    r!   r>   r?   r-   r-   r+   r.   r     s
    r   c                      .   e Zd Z		dd fd	d
ZdddZ  ZS )CosmosDownsample3dTr   r   spatial_downsampler   temporal_downsampler    r!   c                   s   t    || _|| _t | _t | _t | _|r&t	||dddd| _|r2t	||dddd| _|s6|rBt	||dddd| _d S d S )Nr   r   r   r   r   r   r   rh   r   )
r#   r$   r   r   rI   r   r   r   conv3r   )r'   r   r   r   r+   r-   r.   r$   ?  &   






zCosmosDownsample3d.__init__r/   r0   c                 C  s   | j s| js|S | j r'd}tj||ddd}| |}tj|ddd}|| }| jrPtj|d d d d d ddf |gd	d
}| |}tj|ddd}|| }| 	|}|S )N)r   r   r   r   r   r   r   r   r4   r   )r   r   r   .r   r1   rh   )
r   r   r:   r;   r   
avg_pool3dr8   r9   r   r   )r'   r/   r;   conv_outpool_outr-   r-   r.   r<   [  s   
*

zCosmosDownsample3d.forwardTT)r   r   r   r   r   r   r    r!   r>   r?   r-   r-   r+   r.   r   >  
    r   c                      r   )CosmosUpsample3dTr   r   spatial_upsampler   temporal_upsampler    r!   c                   s   t    || _|| _t | _t | _t | _|r&t	||dddd| _|r2t	||dddd| _|s6|rBt	||dddd| _d S d S )Nr   r   r   r   r   r   )
r#   r$   r   r   rI   r   r   r   r   r   )r'   r   r   r   r+   r-   r.   r$   q  r   zCosmosUpsample3d.__init__r/   r0   c                 C  s   | j s| js|S | jr:|d}tdd|dk  }|jt|dd}|d|d d d d d d f }| || }| j rP|jdddjddd}| || }| |}|S )Nr   r   r   r1   .r   rL   )r   r   rN   r   r   r   r   r   )r'   r/   r   time_factorr-   r-   r.   r<     s   
 
zCosmosUpsample3d.forwardr   )r   r   r   r   r   r   r    r!   r>   r?   r-   r-   r+   r.   r   p  r   r   c                      s2   e Zd Z			dd fddZddddZ  ZS )CosmosCausalAttentionr   r3   Nnum_attention_headsr   attention_head_dimrE   r   r   	processorL'CosmosSpatialAttentionProcessor2_0' | 'CosmosTemporalAttentionProcessor2_0'r    r!   c              	     s   t    || _t||d| _t||dddd| _t||dddd| _t||dddd| _t	
g | _| jt||dddd | jt	| || _| jd u rUtdd S )NrE   r   r   r   z+CosmosCausalAttention requires a processor.)r#   r$   r   rD   rK   r   to_qto_kto_vrI   
ModuleListto_outappendr   r   r   )r'   r   r   rE   r   r   r+   r-   r.   r$     s   

zCosmosCausalAttention.__init__r/   r0   attention_masktorch.Tensor | Nonec                 C  s   | j | ||dS )N)r/   r   )r   )r'   r/   r   r-   r-   r.   r<     s   zCosmosCausalAttention.forward)r   r3   N)r   r   r   r   rE   r   r   r   r   r   r    r!   r   )r/   r0   r   r   r    r0   r?   r-   r-   r+   r.   r     s    r   c                   @  "   e Zd Zdd Z	ddddZdS )"CosmosSpatialAttentionProcessor2_0c                 C     t tds	tdd S Nscaled_dot_product_attentionzeCosmosSpatialAttentionProcessor2_0 requires PyTorch 2.0 or higher. To use it, please upgrade PyTorch.hasattrr:   ImportErrorr'   r-   r-   r.   r$     
   
z+CosmosSpatialAttentionProcessor2_0.__init__Nattnr   r/   r0   r   r   r    c                 C  sn  |j \}}}}}|}	||}||}
||}||}|
ddddddddd}
|ddddddddd}|ddddddddd}|
d|jdf	dd}
|d|jdf	dd}|d|jdf	dd}t
j|
|||d}|	dddd|
}|d||fd||f}|ddddd}|jd |}|jd |}||	 S )Nr   r   r   rL   r   rM   	attn_maskra   rK   r   r   r   rO   rP   rQ   r   	transposer:   r   type_asr   r'   r   r/   r   rR   rF   r   r   r   r   querykeyr6   r-   r-   r.   __call__  s&   



"""z+CosmosSpatialAttentionProcessor2_0.__call__r   r   r   r/   r0   r   r   r    r0   r@   rA   rB   r$   r   r-   r-   r-   r.   r         r   c                   @  r   )#CosmosTemporalAttentionProcessor2_0c                 C  r   r   r   r   r-   r-   r.   r$     r   z,CosmosTemporalAttentionProcessor2_0.__init__Nr   r   r/   r0   r   r   r    c                 C  sL  |j \}}}}}|}	||}||}
||}||}|
ddddddd}
|ddddddd}|ddddddd}|
d|jdf	dd}
|d|jdf	dd}|d|jdf	dd}t
j|
|||d}|	dddd|
}|d|||f}|ddddd}|jd |}|jd |}||	 S )Nr   r   rL   r   r   rM   r   r   r   r-   r-   r.   r     s&   



z,CosmosTemporalAttentionProcessor2_0.__call__r   r   r   r-   r-   r-   r.   r     r   r   c                      (   e Zd Zd fddZdddZ  ZS )CosmosDownBlock3dr   r   r   
num_layersr   r   use_attentionr   use_downsampler   r   r    r!   c	              
        t    g g g }	}
}||}}t|D ]6}|	t|||dd |}|rB|
td|d|t d |td|d|t d q|
d  |d  qt	|	| _
t	|
| _t	|| _d | _|rvt	g | _| jt||| d S d S Nr   r   r   r   rE   r   r   )r#   r$   r   r   r   r   r   r   rI   r   resnets
attentionstemp_attentionsdownsamplersr   )r'   r   r   r   r   r   r   r   r   r   r   r   
in_channelout_channelr(   r+   r-   r.   r$     F   

	

zCosmosDownBlock3d.__init__r/   r0   c                 C     t | j| j| jD ]*\}}}||}|d ur||}|d ur3|d}t||| }|||}q	| j	d urC| j	D ]}||}q<|S Nr   )
zipr   r   r   rN   r8   trilnew_onesr   r   )r'   r/   resnet	attentiontemp_attentionr   r   downsamplerr-   r-   r.   r<   A     




zCosmosDownBlock3d.forward)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r>   r?   r-   r-   r+   r.   r         4r   c                      s*   e Zd Zdd fd
dZdddZ  ZS )CosmosMidBlock3dr   r   r   r   r   r   rE   r    r!   c           	   
     s   t    g g g }}}|t|||| t|D ]&}|td|||t d |td|||t d |t|||| qt	|| _
t	|| _t	|| _d S )Nr   r   )r#   r$   r   r   r   r   r   r   rI   r   r   r   r   )	r'   r   r   r   rE   r   r   r   r(   r+   r-   r.   r$   S  s4   
		zCosmosMidBlock3d.__init__r/   r0   c                 C  sr   | j d |}t| j| j| j dd  D ]"\}}}|d}t||| }||}|||}||}q|S )Nr   r   r   )	r   r  r   r   rN   r8   r  r  r   )r'   r/   r  r  r  r   r   r-   r-   r.   r<   r  s   $


zCosmosMidBlock3d.forwardrS   )
r   r   r   r   r   r   rE   r   r    r!   r>   r?   r-   r-   r+   r.   r  R  s    r  c                      r   )CosmosUpBlock3dr   r   r   r   r   r   r   r   use_upsampler   r   r    r!   c	              
     r   r   )r#   r$   r   r   r   r   r   r   rI   r   r   r   r   
upsamplersr   )r'   r   r   r   r   r   r  r   r   r   r  r   r   r   r(   r+   r-   r.   r$     r   zCosmosUpBlock3d.__init__r/   r0   c                 C  r   r  )
r  r   r   r   rN   r8   r  r  r   r  )r'   r/   r  r  r  r   r   	upsamplerr-   r-   r.   r<     r	  zCosmosUpBlock3d.forward)r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r    r!   r>   r?   r-   r-   r+   r.   r    r
  r  c                      @   e Zd Z											
	
d"d# fddZd$d d!Z  ZS )%CosmosEncoder3dr               r  r          rL   r   r3   r   r   r   r   block_out_channelstuple[int, ...]num_resnet_blocksattention_resolutions
resolutionrU   
patch_typer   r   r   spatial_compression_ratiotemporal_compression_ratior    r!   c                   sL  t    ||d  }tt|
tt| }tt|tt| }t||| _t||d | _|| }g }t	t
|d D ];}|| }||d  }||v }d }}|t
|d k rnd}||k }||k }|d }nd}|t||||	|||| qCt|| _t|d d|	dd| _t|d dd	| _t|d || _d| _d S )
Nr   r   r   Fr   TrM   r   r   rE   r   )r#   r$   r   r   r   rT   patch_embedr   conv_inr   lenr   r   rI   r   down_blocksr  	mid_blockrD   norm_outr   gradient_checkpointing)r'   r   r   r  r  r  r  rU   r  r   r   r!  	inner_dimnum_spatial_layersnum_temporal_layerscurrent_resolutionr&  ir   r   r   r   r   r   r+   r-   r.   r$     sH   


zCosmosEncoder3d.__init__r/   r0   c                 C  s   |  |}| |}t r%| jr%| jD ]}| ||}q| | j|}n| jD ]}||}q(| |}| |}t	
|}| |}|S r   )r#  r$  r8   is_grad_enabledr)  r&  _gradient_checkpointing_funcr'  r(  r:   r   r   r'   r/   blockr-   r-   r.   r<     s   








zCosmosEncoder3d.forward)r   r  r  r   r  r  rL   r   r3   r   r   r   r   r   r   r  r  r  r   r  r  r  r   rU   r   r  r   r   r   r   r   r!  r   r    r!   r>   r?   r-   r-   r+   r.   r    s    @r  c                      r  )%CosmosDecoder3dr  r   r  r   r  r  rL   r   r3   r   r   r   r   r  r  r  r  r  rU   r  r   r   r   r   r!  r    r!   c                   s  t    ||d  }tt|
tt| }tt|tt| }tt|}t||d | _t	|d d|	dd| _
|| dt|d   }g }tt|d D ]M}|| }||d  }||v }d }}|t|d k rd}d|  k o|d k n  }|p||k o||k}|d }nd}|t|||d |	|||| qVt|| _t|d dd	| _t|d || _t||| _d| _d S )
Nr   r   r   r"  r   FTrM   r   )r#   r$   r   r   r   listreversedr   r$  r  r'  r%  r   r   r  rI   r   	up_blocksrD   r(  r   r   unpatch_embedr)  )r'   r   r   r  r  r  r  rU   r  r   r   r!  r*  r+  r,  reversed_block_out_channelsr-  r7  r.  r   r   r   r   r   r  r+   r-   r.   r$     sN   


zCosmosDecoder3d.__init__r/   r0   c                 C  sp   |  |}| |}| jD ]}t r| jr| ||}q||}q| |}t	|}| 
|}| |}|S r   )r$  r'  r7  r8   r/  r)  r0  r(  r:   r   r   r8  r1  r-   r-   r.   r<   ^  s   







zCosmosDecoder3d.forward)r  r   r  r   r  r  rL   r   r3   r   r   r3  r>   r?   r-   r-   r+   r.   r4    s    Cr4  c                      s   e Zd ZdZdZedddddddd	d
ddddeefdG fd$d%Z	&	&	&	&	&	&dHdId/d0Z	dJd3d4Z
edKdLd7d8ZdKdMd;d<ZedKdMd=d>Z	?		&dNdOdEdFZ  ZS )PAutoencoderKLCosmosa	  
    Autoencoder used in [Cosmos](https://huggingface.co/papers/2501.03575).

    Args:
        in_channels (`int`, defaults to `3`):
            Number of input channels.
        out_channels (`int`, defaults to `3`):
            Number of output channels.
        latent_channels (`int`, defaults to `16`):
            Number of latent channels.
        encoder_block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512)`):
            Number of output channels for each encoder down block.
        decode_block_out_channels (`tuple[int, ...]`, defaults to `(256, 512, 512, 512)`):
            Number of output channels for each decoder up block.
        attention_resolutions (`tuple[int, ...]`, defaults to `(32,)`):
            list of image/video resolutions at which to apply attention.
        resolution (`int`, defaults to `1024`):
            Base image/video resolution used for computing whether a block should have attention layers.
        num_layers (`int`, defaults to `2`):
            Number of resnet blocks in each encoder/decoder block.
        patch_size (`int`, defaults to `4`):
            Patch size used for patching the input image/video.
        patch_type (`str`, defaults to `haar`):
            Patch type used for patching the input image/video. Can be either `haar` or `rearrange`.
        scaling_factor (`float`, defaults to `1.0`):
            The component-wise standard deviation of the trained latent space computed using the first batch of the
            training set. This is used to scale the latent space to have unit variance when training the diffusion
            model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the
            diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1
            / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image
            Synthesis with Latent Diffusion Models](https://huggingface.co/papers/2112.10752) paper. Not applicable in
            Cosmos, but we default to 1.0 for consistency.
        spatial_compression_ratio (`int`, defaults to `8`):
            The spatial compression ratio to apply in the VAE. The number of downsample blocks is determined using
            this.
        temporal_compression_ratio (`int`, defaults to `8`):
            The temporal compression ratio to apply in the VAE. The number of downsample blocks is determined using
            this.
    Tr   r  r  )r  r  r  r  r  r  r   rL   r   r   r   r   r   r   latent_channelsencoder_block_out_channelsr  decode_block_out_channelsr  r  r   rU   r  r   scaling_factorr   r   r!  latents_meanlist[float] | Nonelatents_stdr    r!   c                   s   t    t|||||||	|
||d
| _t|||||||	|
||d
| _t||ddd| _t||ddd| _d| _	d| _
d| _d| _d| _d| _d| _d| _d| _d	| _d	| _d
| _d S )N)
r   r   r  r  r  r  rU   r  r   r!  r   r   )r   r   Fr  r   r  i  r   )r#   r$   r  encoderr4  decoderr   
quant_convpost_quant_convuse_slicing
use_tilinguse_framewise_encodinguse_framewise_decodingnum_sample_frames_batch_sizenum_latent_frames_batch_sizetile_sample_min_heighttile_sample_min_widthtile_sample_min_num_framestile_sample_stride_heighttile_sample_stride_widthtile_sample_stride_num_frames)r'   r   r   r;  r<  r=  r  r  r   rU   r  r>  r   r!  r?  rA  r+   r-   r.   r$     sN   

zAutoencoderKLCosmos.__init__NrL  
int | NonerM  rN  rO  float | NonerP  rQ  c                 C  sR   d| _ |p| j| _|p| j| _|p| j| _|p| j| _|p| j| _|p%| j| _dS )aX  
        Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
        compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
        processing larger images.

        Args:
            tile_sample_min_height (`int`, *optional*):
                The minimum height required for a sample to be separated into tiles across the height dimension.
            tile_sample_min_width (`int`, *optional*):
                The minimum width required for a sample to be separated into tiles across the width dimension.
            tile_sample_stride_height (`int`, *optional*):
                The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are
                no tiling artifacts produced across the height dimension.
            tile_sample_stride_width (`int`, *optional*):
                The stride between two consecutive horizontal tiles. This is to ensure that there are no tiling
                artifacts produced across the width dimension.
        TN)rG  rL  rM  rN  rO  rP  rQ  )r'   rL  rM  rN  rO  rP  rQ  r-   r-   r.   enable_tiling  s   z!AutoencoderKLCosmos.enable_tilingxr0   c                 C  s   |  |}| |}|S r   )rB  rD  )r'   rU  encr-   r-   r.   _encode  r   zAutoencoderKLCosmos._encodereturn_dictr   c                   s^    j r|jd dkr fdd|dD }t|}n |}t|}|s*|fS t|dS )Nr   r   c                   s   g | ]}  |qS r-   )rW  ).0x_slicer   r-   r.   
<listcomp>  s    z.AutoencoderKLCosmos.encode.<locals>.<listcomp>)latent_dist)rF  ra   r   r8   r9   rW  r   r	   )r'   rU  rX  encoded_slicesr   	posteriorr-   r   r.   encode  s   

zAutoencoderKLCosmos.encodez#DecoderOutput | tuple[torch.Tensor]c                 C  s(   |  |}| |}|s|fS t|dS )Nsample)rE  rC  r   )r'   r`  rX  decr-   r-   r.   _decode  s
   


zAutoencoderKLCosmos._decodec                   sX    j r|jd dkr fdd|dD }t|}n |j}|s'|fS t|dS )Nr   r   c                   s   g | ]}  |jqS r-   )re  rc  )rY  z_slicer   r-   r.   r[  )  s    z.AutoencoderKLCosmos.decode.<locals>.<listcomp>rb  )rF  ra   r   r8   r9   re  rc  r   )r'   r`  rX  decoded_slicesdecodedr-   r   r.   decode&  s   
zAutoencoderKLCosmos.decodeFrc  sample_posterior	generatortorch.Generator | None#tuple[torch.Tensor] | DecoderOutputc           	      C  sJ   |}|  |j}|r|j|d}n| }| |j}|s |fS t|dS )N)rk  rb  )r_  r\  rc  r5   ri  r   )	r'   rc  rj  rX  rk  rU  r^  r`  rd  r-   r-   r.   r<   2  s   
zAutoencoderKLCosmos.forward) r   r   r   r   r;  r   r<  r  r=  r  r  r  r  r   r   r   rU   r   r  r   r>  r   r   r   r!  r   r?  r@  rA  r@  r    r!   )NNNNNN)rL  rR  rM  rR  rN  rR  rO  rS  rP  rS  rQ  rS  r    r!   )rU  r0   r    r0   )T)rU  r0   rX  r   r    r0   )r`  r0   rX  r   r    ra  )FTN)
rc  r0   rj  r   rX  r   rk  rl  r    rm  )r@   rA   rB   __doc__ _supports_gradient_checkpointingr   LATENTS_MEANLATENTS_STDr$   rT  rW  r   r_  re  ri  r<   rC   r-   r-   r+   r.   r:  o  sJ    (P
"r:  )1
__future__r   r   r8   torch.nnrI   torch.nn.functional
functionalr:   configuration_utilsr   r   utilsr   utils.accelerate_utilsr   modeling_outputsr	   modeling_utilsr
   vaer   r   r   r@   loggerrp  rq  tensorr]   Conv3dr   ModulerD   rT   r   r   r   r   r   r   r   r   r   r  r  r  r4  r:  r-   r-   r-   r.   <module>   sD   %PO'20''F.FTU