o
    Ni                     @  sz  d dl mZ d dlmZmZ d dlmZ d dlmZ d dl	Z	d dl	m
Z
mZ d dlmZ d dlmZ d	d
 Zdd Zdd Zddd$ddZddd$ddZG dd deZedkre	dde	ddZeeddd ddZe  e	  ed  d! ed" d! W d   n1 sw   Y  ejd   Z e  de  ded    Z!e	"ejd  e!sJ e#d# dS dS )%    )annotations)CallableAny)deepcopy)partialN)nnTensor)Module)_pytreec                 C  s   | d uS N )valr   r   R/home/ubuntu/.local/lib/python3.10/site-packages/ema_pytorch/ema_pytree_pytorch.pyexists   s   r   c                 C  s   | | dkS Nr   r   )numdenr   r   r   divisible_by   s   r   c                 C  s   | j |kr| S | |S r   )dtypeto)tr   r   r   r   maybe_coerce_dtype   s   

r   Fcoerce_dtypetgtr   srcc                C  s   |rt || j}| | d S r   )r   r   copy_)r   r   r   r   r   r   inplace_copy   s   r   c                C  s    |rt || j}| || d S r   )r   r   lerp_)r   r   weightr   r   r   r   inplace_lerp   s   r    c                      sn   e Zd Z								dd fddZedd Zdd Zdd Zdd Ze	
 dddZdd Z  ZS )	EMAPytreeNH.?d   
         ?UUUUUU?        Fpytreer   
ema_pytreeAny | Callable[[], Any] | Nonec
                   s   t    || _|dk| _|g| _t|st|r| }|| _t| js)t|| _t	
| j\}
}|
D ]}t|tr>|  q3tt|	d| _tt|	d| _|| _|| _|| _|| _|| _|	| _| dtd | dtd d S )Nr%   r   inittedFstepr   )super__init__beta	is_frozenonline_pytreer   callabler)   r   
pytree_pkgtree_flatten
isinstancer   detach_r   r   r    update_everyupdate_after_step	inv_gammapower	min_valuer   register_buffertorchtensor)selfr(   r)   r/   r8   r7   r9   r:   r;   r   ema_tensors_p	__class__r   r   r.   %   s0   




zEMAPytree.__init__c                 C  s
   | j d S r   )r1   )r?   r   r   r   r(   ]   s   
zEMAPytree.pytreec                 C  s`   | j }t| j\}}t| j\}}t||D ]\}}t|tr-t|tr-||j|j qd S r   )	r   r3   r4   r)   r(   zipr5   r   data)r?   copyr@   rA   online_tensors	ma_tensoronline_tensorr   r   r   copy_params_from_pytree_to_emaa   s   z(EMAPytree.copy_params_from_pytree_to_emac                 C  sX   | j | j d jdd}dd|| j  | j   }| dkr!dS |j| j| jd S )N   r'   )minr   )rM   max)r,   r8   clampr9   r:   itemr;   r/   )r?   epochvaluer   r   r   get_current_decayk   s
   zEMAPytree.get_current_decayc                 C  s   | j  }|  j d7  _ | j s!|   | jjtd d S t|| j	}|r4|| j
kr4|   d S |r@| | j| j d S d S )NrL   T)r,   rP   r+   rK   rF   r   r=   r>   r   r7   r8   update_moving_averager)   r(   )r?   r,   should_updater   r   r   updatet   s   

zEMAPytree.updatec           	      C  sx   | j rd S t|s|  }t|\}}t|\}}t||D ]\}}t|tr9t|tr9| |j	|j	d|  q d S )Nr%   )
r0   r   rS   r3   r4   rE   r5   r   r    rF   )	r?   	ma_pytreecurrent_pytreecurrent_decayr@   rA   rH   rI   rJ   r   r   r   rT      s   zEMAPytree.update_moving_averagec                 O  s    t | jr| j|i |S | jS r   )r2   r)   )r?   argskwargsr   r   r   __call__   s   
zEMAPytree.__call__)Nr"   r#   r$   r%   r&   r'   F)r(   r   r)   r*   r   )__name__
__module____qualname__r.   propertyr(   rK   rS   rV   r=   no_gradrT   r\   __classcell__r   r   rC   r   r!   $   s$    8

	r!   __main__   )wbg      ?rL   )r/   r;   r8   r7   re   r%   rf   success)r   r   r   r   )$
__future__r   typingr   r   rG   r   	functoolsr   r=   r   r   torch.nnr	   torch.utilsr
   r3   r   r   r   r   r    r!   r]   randnonline_treeema_treerV   ra   fill_r)   clone	old_ema_w
expected_wallcloseprintr   r   r   r   <module>   sH    v

