o
    ۷i(                     @   s   d dl Z d dlZd dlZd dlmZ ddlmZmZ ddl	m
Z
 ddlmZmZmZmZ G dd de
eZG d	d
 d
ejZdS )    N   )ConfigMixinregister_to_config)
ModelMixin   )	AttnBlockGlobalResponseNormTimestepBlockWuerstchenLayerNormc                       s   e Zd Zedddddg dg dg dg dg d	d
dddf fdd	Zdd ZdddZdd Zd ddZd ddZ	d!ddZ
  ZS )"WuerstchenDiffNeXt   @      i   )i@  i     r   )
      r   )r   r      r   )CTCTAr   r   )FTTT   r   g?c                    s  t    | _ | _t|ts|gt| }t| | _	t
 fdd|
tt|
 D | _tj ddd| _tt|tj||d  |d dd	t|d ddd| _d fd
d	}t
 | _tt|D ]Y}t
 }|dkr|tt||d  dddtj||d  || ddd t|| D ]#}|	| D ]}|
| r nd}||||| || ||| d qq| j| qmt
 | _ttt|D ]|}t
 }t|| D ]G}t|	| D ]>\}}|t|d k r||  krdkrn n|| nd}||
| r nd7 }||||| || ||| d qq|dkrJ|tt|| dddtj|| ||d  ddd | j| qtt|d dddtj|d d| |d  dd	t|| _| | j d S )Nc                    s$   g | ]}|rt j d dndqS )r   kernel_sizeN)nnConv2d).0inject)c_condeffnet_embd q/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/pipelines/wuerstchen/modeling_wuerstchen_diffnext.py
<listcomp>7   s    z/WuerstchenDiffNeXt.__init__.<locals>.<listcomp>Fư>elementwise_affineepsr   r   r   r   c                    sT   | dkrt |||dS | dkrt| |d|dS | dkr"t|S td|  d)	NC)r   dropoutAT)	self_attnr'   TzBlock type z not supported)ResBlockStageBr   r	   
ValueError)
block_typec_hiddennheadc_skipr'   )r   c_rr   r   r    	get_blockD   s   
z.WuerstchenDiffNeXt.__init__.<locals>.get_block)r   stride)r0   r'   )r   r   )super__init__r1   r   
isinstancelistlenr   Linearclip_mapper
ModuleListreversedeffnet_mappers	LayerNormseq_norm
SequentialPixelUnshuffler   r
   	embeddingdown_blocksrangeappend	up_blocks	enumerateConvTranspose2dPixelShuffleclfapply_init_weights)selfc_inc_outr1   
patch_sizer   r.   r/   blockslevel_configinject_effnetr   	clip_embdr   r'   r2   i
down_block_r-   r0   up_blockjk	__class__)r   r1   r   r   r    r5      sr   


&
:&
zWuerstchenDiffNeXt.__init__c              	   C   s  t |tjtjfrtj|j |jd urtj|jd | j	D ]}|d ur/tjj
|jdd q tjj
| jjdd tj| jd jd tj| jd jd | j| j D ]/}|D ]*}t |trv|jd j jtdt| jj 9  _qZt |trtj|jjd qZqVd S )Nr   g{Gz?)stdr   r   )r6   r   r   r9   initxavier_uniform_weightbias	constant_r=   normal_r:   rB   rJ   rC   rF   r+   channelwisedatanpsqrtsumconfigrQ   r	   mapper)rM   mrj   level_blockblockr   r   r    rL   |   s(   


*
z WuerstchenDiffNeXt._init_weights'  c                 C   s   || }| j d }t||d  }tj||jd |  }|d d d f |d d d f  }tj	|
 | gdd}| j d dkrOtjj|ddd}|j|jdS )	Nr   r   )devicedim)r   r   constant)mode)dtype)r1   mathlogtorcharangero   floatmulexpcatsincosr   
functionalpadtort   )rM   rmax_positionshalf_dimembr   r   r    gen_r_embedding   s   
  z"WuerstchenDiffNeXt.gen_r_embeddingc                 C   s   |  |}| |}|S N)r:   r?   )rM   clipr   r   r    gen_c_embeddings   s   

z#WuerstchenDiffNeXt.gen_c_embeddingsNc              
   C   s   g }t | jD ]i\}}d }|D ]Z}	t|	trO|d u r>| j| d ur>|j}
| j| tjj|	 |j
dd  dddd|
}| j| d urG|nd }|	||}qt|	trZ|	||}qt|	tre|	||}q|	|}q|d| q|S )NbicubicTsizers   	antialiasalign_cornersr   )rG   rC   r6   r+   r=   rt   r   r   interpolatery   shaper   r   r	   insert)rM   xr_embedeffnetr   level_outputsrU   rV   effnet_crm   rt   skipr   r   r    _down_encode   s,   



zWuerstchenDiffNeXt._down_encodec              
   C   s(  |d }t | jD ]\}}d }t |D ]}\}	}
t|
trv|d u rN| jt| j|  d urN|j}| jt| j|  tj	j
| |jdd  dddd|}|	dkrZ|dkrZ|| nd }|d urp|d urntj||gdd}n|}|
||}qt|
tr|
||}qt|
tr|
||}q|
|}qq	|S )Nr   r   r   Tr   r   rp   )rG   rF   r6   r+   r=   r8   rC   rt   r   r   r   ry   r   r   rw   r|   r   r	   )rM   r   r   r   r   r   rU   rX   r   rY   rm   rt   r   r   r   r    
_up_decode   s4   
 


zWuerstchenDiffNeXt._up_decodeMbP?Tc                 C   s   |d urt j||gdd}| |}|d ur| |}|}	| |}| ||||}
| |
|||}| |jddd\}}|	 d|d   | }|rR|	| | S ||fS )Nr   rp   r   )
rw   r|   r   r   rB   r   r   rJ   chunksigmoid)rM   r   r   r   r   x_catr%   return_noiser   x_inr   abr   r   r    forward   s   


zWuerstchenDiffNeXt.forward)rn   r   )NNr   T)__name__
__module____qualname__r   r5   rL   r   r   r   r   r   __classcell__r   r   r[   r    r      s.    _


r   c                       s(   e Zd Zd	 fdd	Zd
ddZ  ZS )r+   r   r           c              
      sz   t    tj||||d |d| _t|ddd| _tt|| |d t	 t
|d t|t|d || _d S )Nr   )r   paddinggroupsFr"   r#   r   )r4   r5   r   r   	depthwiser
   normr@   r9   GELUr   Dropoutrd   )rM   cr0   r   r'   r[   r   r    r5      s   


zResBlockStageB.__init__Nc                 C   sX   |}|  | |}|d urtj||gdd}| |dddddddd}|| S )Nr   rp   r   r   r   )r   r   rw   r|   rd   permute)rM   r   x_skipx_resr   r   r    r      s   "zResBlockStageB.forward)r   r   r   r   )r   r   r   r5   r   r   r   r   r[   r    r+      s    r+   )ru   numpyrf   rw   torch.nnr   configuration_utilsr   r   models.modeling_utilsr   modeling_wuerstchen_commonr   r   r	   r
   r   Moduler+   r   r   r   r    <module>   s    Q