o
    Ni9                     @  s   d dl mZ d dlmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlmZmZ d dlmZmZ d dlZd	d
 Zdd Zdd Zdd Zd(ddZddd)ddZddd)ddZdd ZG d d! d!eZd"d# Zd$d% ZG d&d' d'eZdS )*    )annotations)CallableLiteral)Path)deepcopy)partialN)nnTensor)Module
ModuleListc                 C  s   | d uS N )valr   r   L/home/ubuntu/.local/lib/python3.10/site-packages/ema_pytorch/post_hoc_ema.pyexists      r   c                 C  s   t | r| S |S r   )r   )r   dr   r   r   default      r   c                 C  s   | d S Nr   r   )arrr   r   r   first   r   r   c                 C  s   | | dkS r   r   )numdenr   r   r   divisible_by   s   r   mr
   c                 C  s   t |  jS r   )next
parametersdevice)r   r   r   r   get_module_device   s   r   Fauto_move_devicetgtr	   srcc                C  s   |r| | j}| | d S r   )tor   copy_)r"   r#   r!   r   r   r   inplace_copy   s   r&   c                C  s    |r| | j}| || d S r   )r$   r   lerp_)r"   r#   weightr!   r   r   r   inplace_lerp#   s   r)   c                 C  s,   | d }t ddd| d| gj  S )N            )nprootsrealmaxitem)	sigma_reltr   r   r   sigma_rel_to_gamma+   s   $r6   c                
      s   e Zd ZdZddddde e e ddf
d. fddZedd Zedd Zdd Z	dd Z
dd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Ze d*d+ Zd,d- Z  ZS )/	KarrasEMAz
    exponential moving average module that uses hyperparameters from the paper https://arxiv.org/abs/2312.02696
    can either use gamma or sigma_rel from paper
    Nd   Fmodelr
   r4   float | Nonegamma	ema_model$Module | Callable[[], Module] | Noneupdate_everyintfrozenboolparam_or_buffer_names_no_emaset[str]ignore_namesignore_startswith_namesc              
     s|  t    t|t|A sJ dt|rt|}|| _|| _|g| _t|ts/t	|r/| }|| _
t| j
s_zt|| _
W n  ty^ } ztd|  td t  W Y d }~nd }~ww | j
 D ]}|  qddd | j
 D | _dd | j
 D | _tt|
d| _tt|
d| _|| _t|ttfsJ || _|| _|	| _|
| _|| _|  dt!"d	 |  d
t!"d d S )Nzteither sigma_rel or gamma is given. gamma is derived from sigma_rel as in the paper, then beta is dervied from gammaz'Error: While trying to deepcopy model: zNYour model was not copyable. Please make sure you are not using any LazyLinearc                 S  (   h | ]\}}t |st |r|qS r   torchis_floating_point
is_complex).0nameparamr   r   r   	<setcomp>e      ( z%KarrasEMA.__init__.<locals>.<setcomp>c                 S  rF   r   rG   )rK   rL   bufferr   r   r   rN   f   rO   r    inittedFstepr   )#super__init__r   r6   r;   r@   online_model
isinstancer
   callabler<   r   	Exceptionprintexitr   detach_named_parametersparameter_namesnamed_buffersbuffer_namesr   r&   r)   r>   setlistrB   rD   rE   allow_different_devicesmove_ema_to_online_deviceregister_bufferrH   tensor)selfr9   r4   r;   r<   r>   r@   rB   rD   rE   rb   rc   ep	__class__r   r   rT   5   sD   


zKarrasEMA.__init__c                 C  
   t | jS r   )r   rU   rf   r   r   r   r9         
zKarrasEMA.modelc                 C  s    dd| j  d   d| j  S )N      ?)rR   r3   r;   rl   r   r   r   beta   s    zKarrasEMA.betac                 C  s
   | j  S r   )r<   evalrl   r   r   r   rp      s   
zKarrasEMA.evalc                 C  s   | j j}| j| d S r   )rQ   r   r<   r$   )rf   r   r   r   r   restore_ema_model_device   s   z"KarrasEMA.restore_ema_model_devicec                 c  .    |  D ]\}}|| jvrq||fV  qd S r   )r\   r]   )rf   r9   rL   rM   r   r   r   get_params_iter      
zKarrasEMA.get_params_iterc                 c  rr   r   )r^   r_   )rf   r9   rL   rP   r   r   r   get_buffers_iter   rt   zKarrasEMA.get_buffers_iterc                 C  s~   | j }t| | j| | jD ]\\}}\}}||j|j qt| | j| | jD ]\\}}\}}||j|j q-d S r   r&   ziprs   r<   r9   dataru   rf   copy_	ma_paramscurrent_params
ma_bufferscurrent_buffersr   r   r   copy_params_from_model_to_ema      **z'KarrasEMA.copy_params_from_model_to_emac                 C  s~   | j }t| | j| | jD ]\\}}\}}||j|j qt| | j| | jD ]\\}}\}}||j|j q-d S r   rv   ry   r   r   r   copy_params_from_ema_to_model   r   z'KarrasEMA.copy_params_from_ema_to_modelc                 C  sd   | j  }|  j d7  _ || j dkrd S | j s(|   | jjtd | 	| j
| j d S )Nr+   r   T)rR   r3   r>   rQ   r   rx   r%   rH   re   update_moving_averager<   r9   )rf   rR   r   r   r   update   s   

zKarrasEMA.updatec                 #  s    |  | jD ] \ } | jv rqt fdd| jD rq | jv r$q|V  q| | jD ] \ } | jv r8q.t fdd| jD rEq. | jv rKq.|V  q.d S )Nc                      g | ]}  |qS r   
startswithrK   prefixrL   r   r   
<listcomp>       z=KarrasEMA.iter_all_ema_params_and_buffers.<locals>.<listcomp>c                   r   r   r   r   r   r   r   r      r   )rs   r<   rD   anyrE   rB   ru   )rf   r|   	ma_bufferr   r   r   iter_all_ema_params_and_buffers   s$   



z)KarrasEMA.iter_all_ema_params_and_buffersc                   s>  | j rd S | jrt|t|kr|t| | j| j}}| j}t| || |D ]2\\ }\}} | j	v r:q,t
 fdd| jD rGq, | jv rT||j|j q,||j|jd|  q,t| || |D ]2\\ }	\}}
 | j	v rxqjt
 fdd| jD rqj | jv r||
j|	j qj||
j|	jd|  qjd S )Nc                   r   r   r   r   r   r   r   r      r   z3KarrasEMA.update_moving_average.<locals>.<listcomp>rn   c                   r   r   r   r   r   r   r   r      r   )r@   rc   r   r$   r&   r)   ro   rw   rs   rD   r   rE   rB   rx   ru   )rf   ma_modelcurrent_modelrz   lerpcurrent_decayr}   r{   r|   current_bufferr   r   r   r   r      s2   &

&

zKarrasEMA.update_moving_averagec                 O  s   | j |i |S r   )r<   rf   argskwargsr   r   r   __call__   r   zKarrasEMA.__call__)r9   r
   r4   r:   r;   r:   r<   r=   r>   r?   r@   rA   rB   rC   rD   rC   rE   rC   )__name__
__module____qualname____doc__r`   rT   propertyr9   ro   rp   rq   rs   ru   r   r   r   r   rH   no_gradr   r   __classcell__r   r   ri   r   r7   /   s8    O

		
(r7   c           	      C  sX   | | }t | |k || }t | |}|d |d  ||  }|| d | }|| S )Nr+   )rH   wheremaximum)	t_agamma_at_bgamma_bt_ratiot_expt_maxr   r   r   r   r   p_dot_p  s   r   c                 C  sZ   dd }dd }t || |||| ||}t || ||||||}tj||S )Nc                 S     |   ddS )Nr+   doublereshapexr   r   r   <lambda>      zsolve_weights.<locals>.<lambda>c                 S  r   )Nr+   r   r   r   r   r   r   r     r   )r   rH   linalgsolve)t_igamma_it_rgamma_rrvcvAbr   r   r   solve_weights  s
   r   c                      s   e Zd Zdddddejfd, fddZedd Zedd Zedd Z	dd Z
dd Zdd Zdd  Z			d-d.d(d)Zd*d+ Z  ZS )/
PostHocEMANi  z./post-hoc-ema-checkpointsr9   r
   r<   Callable[[], Module] | None
sigma_relstuple[float, ...] | Nonegammascheckpoint_every_num_stepsint | Literal['manual']checkpoint_folderstrcheckpoint_dtypetorch.dtypec           	        s   t    t|t|A sJ t|rttt|}t|dks$J dtt|t|ks2J d | _|| _	t|| _
g| _t fdd|D | _t|| _| jjddd | j scJ || _|| _| _d S )Nr+   zfat least 2 ema models with different gammas in order to synthesize new ema models of a different gammaz$calculated gammas must be all uniquec                   s"   g | ]}t f |d qS ))r<   r;   )r7   )rK   r;   r<   r   r9   r   r   r   3  s   " z'PostHocEMA.__init__.<locals>.<listcomp>T)exist_okparents)rS   rT   r   tuplemapr6   lenr`   maybe_ema_modelr   num_ema_models_modelr   
ema_modelsr   r   mkdiris_dirr   r   
ema_kwargs)	rf   r9   r<   r   r   r   r   r   r   ri   r   r   rT     s"   



zPostHocEMA.__init__c                 C  rk   r   )r   r   rl   r   r   r   r9   =  rm   zPostHocEMA.modelc                 C  s   t | jjS r   )r   r   rR   rl   r   r   r   rR   A  s   zPostHocEMA.stepc                 C  s   | j jS r   )rR   r   rl   r   r   r   r   E  s   zPostHocEMA.devicec                 C     | j D ]}|  qd S r   )r   r   rf   r<   r   r   r   r   I     

z(PostHocEMA.copy_params_from_model_to_emac                 C  r   r   )r   r   r   r   r   r   r   M  r   z(PostHocEMA.copy_params_from_ema_to_modelc                 C  sD   | j D ]}|  q| jdkrd S t| j | jr |   d S d S )Nmanual)r   r   r   r   rR   r3   
checkpointr   r   r   r   r   Q  s   


zPostHocEMA.updatec                   sf    j  }t jD ]&\}}| d| d} j| } fdd|  D }t|t	| q
d S )N.z.ptc                   s$   i | ]\}}||j d  jddqS )cpuT)r   dtyperz   )r$   r   )rK   kvrl   r   r   
<dictcomp>b  s    z)PostHocEMA.checkpoint.<locals>.<dictcomp>)
rR   r3   	enumerater   r   
state_dictitemsrH   saver   )rf   rR   indr<   filenamepathpkgr   rl   r   r   [  s   



zPostHocEMA.checkpointr;   r:   r4   rR   
int | Nonereturnr7   c                 C  s  t |t |A s
J | j}t |rt|}td
| j| j|d| j}| g }g }g | jd}|D ]}	t	t
|	jd\}
}|| j|
  || q3t|t|}|t|ksdJ dt| tj||d}tj||d}tj|g|d}tj|g|d}t||||}|d}td
| j| j|d| j}tt|| D ]3\}\}}|dk}tjt|dd	}|| t| | D ]\}}|r|  |||  qq|S )N)r9   r<   r;   z*.ptr   zJyou can only synthesize for a timestep that is less than the max timestep )r   r   r   T)weights_onlyr   )r   r   r6   r7   r9   r   r   r   globr   r?   stemsplitappendr   r   r2   rH   re   r   squeezer   rw   tolistloadr   load_state_dictr   zero_add_)rf   r;   r4   rR   r   synthesized_ema_modelr   	timestepscheckpointsfile	gamma_indtimestepr   r   r   r   weightstmp_ema_modelr   r   r(   is_firstckpt_state_dictckpt_tensorsynth_tensorr   r   r   synthesize_ema_modeli  sX   

zPostHocEMA.synthesize_ema_modelc                   s   t  fdd| jD S )Nc                 3  s    | ]
}| i V  qd S r   r   )rK   r<   r   r   r   r   	<genexpr>  s    z&PostHocEMA.__call__.<locals>.<genexpr>)r   r   r   r   r  r   r     s   zPostHocEMA.__call__)r9   r
   r<   r   r   r   r   r   r   r   r   r   r   r   )NNN)r;   r:   r4   r:   rR   r   r   r7   )r   r   r   rH   float16rT   r   r9   rR   r   r   r   r   r   r  r   r   r   r   ri   r   r     s.    $



Mr   )r   r
   )r"   r	   r#   r	   )
__future__r   typingr   r   pathlibr   rz   r   	functoolsr   rH   r   r	   torch.nnr
   r   numpyr/   r   r   r   r   r   r&   r)   r6   r7   r   r   r   r   r   r   r   <module>   s,    
 Z