o
    ۷i
                     @   s~   d dl Z d dlmZ ddlmZ G dd dejZG dd dejZG dd	 d	ejZ	G d
d dejZ
G dd dejZdS )    N   )	Attentionc                       s(   e Zd Z fddZ fddZ  ZS )WuerstchenLayerNormc                    s   t  j|i | d S N)super__init__)selfargskwargs	__class__ o/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.pyr      s   zWuerstchenLayerNorm.__init__c                    s,   | dddd}t |}| ddddS )Nr      r      )permuter   forward)r   xr   r   r   r      s   zWuerstchenLayerNorm.forward__name__
__module____qualname__r   r   __classcell__r   r   r   r   r      s    r   c                       $   e Zd Z fddZdd Z  ZS )TimestepBlockc                    s    t    t||d | _d S )Nr   )r   r   nnLinearmapper)r   c
c_timestepr   r   r   r      s   
zTimestepBlock.__init__c                 C   s<   |  |d d d d d d f jddd\}}|d|  | S )Nr   r   dim)r   chunk)r   r   tabr   r   r   r      s   ,zTimestepBlock.forwardr   r   r   r   r   r          r   c                       s(   e Zd Zd	 fdd	Zd
ddZ  ZS )ResBlockr   r           c              
      sz   t    tj|| |||d |d| _t|ddd| _tt||d t	 t
|d t|t|d || _d S )Nr   )kernel_sizepaddinggroupsFư>elementwise_affineeps   )r   r   r   Conv2d	depthwiser   norm
Sequentialr   GELUGlobalResponseNormDropoutchannelwise)r   r   c_skipr)   dropoutr   r   r   r      s   
4
zResBlock.__init__Nc                 C   sX   |}|d urt j||gdd}| | |dddd}| |dddd}|| S )Nr   r    r   r   r   )torchcatr3   r2   r   r8   )r   r   x_skipx_resr   r   r   r   &   s   zResBlock.forward)r   r   r(   r   r   r   r   r   r   r'      s    	r'   c                       r   )r6   c                    s>   t    ttddd|| _ttddd|| _d S )Nr   )r   r   r   	Parameterr;   zerosgammabeta)r   r!   r   r   r   r   1   s   
zGlobalResponseNorm.__init__c                 C   s@   t j|dddd}||jdddd  }| j||  | j | S )Nr   )r   r   T)pr!   keepdim)r!   rD   r,   )r;   r3   meanrA   rB   )r   r   agg_normstand_div_normr   r   r   r   6   s   zGlobalResponseNorm.forwardr   r   r   r   r   r6   0   r&   r6   c                       s&   e Zd Zd fdd	Zdd Z  ZS )	AttnBlockTr(   c                    sV   t    || _t|ddd| _t|||| |dd| _tt	 t
||| _d S )NFr,   r-   T)	query_dimheadsdim_headr:   bias)r   r   	self_attnr   r3   r   	attentionr   r4   SiLUr   	kv_mapper)r   r   c_condnheadrN   r:   r   r   r   r   =   s
   
zAttnBlock.__init__c                 C   sb   |  |}| |}| jr&|j\}}}}tj|||ddd|gdd}|| j||d }|S )NrE   r   r   r    )encoder_hidden_states)	rQ   r3   rN   shaper;   r<   view	transposerO   )r   r   kvnorm_x
batch_sizechannel_r   r   r   r   E   s   

$zAttnBlock.forward)Tr(   r   r   r   r   r   rI   <   s    rI   )r;   torch.nnr   models.attention_processorr   	LayerNormr   Moduler   r'   r6   rI   r   r   r   r   <module>   s    
