o
    TiF                     @   s@   d dl Z d dlmZ ddlmZ ddlmZ G dd deZdS )	    N)	Parameter   )DSPolicy   DSUNetc                       s>   e Zd Z fddZdd Zdd Zddd	Zd
d Z  ZS )
UNetPolicyc                    s`   t    zdd l}|jjj| _W d S  ty#   |jjjj| _Y d S  t	y/   d | _Y d S w )Nr   )
super__init__	diffusersmodelsunet_2d_conditionUNet2DConditionModel_orig_layer_classAttributeErrorunetsImportError)selfr   	__class__ [/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/module_inject/containers/unet.pyr
      s   
zUNetPolicy.__init__c                 C   s   t || jS N)
isinstancer   r   moduler   r   r   match   s   zUNetPolicy.matchc                 C   s
   t |tS r   )r   r   r   r   r   r   match_replaced   s   
zUNetPolicy.match_replacedTc                 C   s   t ||dS )N)enable_cuda_graphr   )r   r   r   r   r   r   apply   s   zUNetPolicy.applyc                 C   s   |j j}|jj}|jj}|jd |jd kr7ttj|||fdddd}||jd j|jd j	|jd |j
fS ||||jd j|jd j	|jd |j
fS )N   r   )dimF)requires_grad)to_qweightto_kto_vshaper   torchcatto_outbiasheads)r   client_moduleqwkwvwqkvwr   r   r   	attention#   s$   



zUNetPolicy.attention)T)	__name__
__module____qualname__r
   r   r   r   r3   __classcell__r   r   r   r   r      s    

r   )r)   torch.nn.parameterr   policyr   $model_implementations.diffusers.unetr   r   r   r   r   r   <module>   s
   