o
    }oiF                     @   s.  d dl Zd dlmZmZmZ d dlmZ d dl	Z	d dl
Z
d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ dZe
j r\e
j Zdd Z dd Z!G dd de
j"j#j$Z%G dd deZ&G dd dZ'G dd dZ(G dd deZ)G dd deZ*dS )    N)AnyDictUnion)CallbackTrainer)MisconfigurationException)STEP_OUTPUT)
DictConfig	OmegaConf)EMA)EMAOptimizer)ModelPT)exp_managerc                 C   s4   dd |j D d }|| t| }|| |S )Nc                 S      g | ]	}t |tr|qS  
isinstancer   .0xr   r   U/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/common/test_ema.py
<listcomp>%       z'extract_ema_weights.<locals>.<listcomp>r   )	callbacksswap_model_weightsextract_weights)	pl_moduletrainerema_callbackweightsr   r   r   extract_ema_weights$   s
   

r    c                 C   s   dd |   D S )Nc                 S   s   g | ]}|   qS r   )detachclone)r   wr   r   r   r   -   s    z#extract_weights.<locals>.<listcomp>)
parameters)r   r   r   r   r   ,   s   r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )RandomDatasetc                 C   s   || _ t||| _d S N)lentorchrandndata)selfsizelengthr   r   r   __init__1   s   zRandomDataset.__init__c                 C   s
   | j | S r&   )r*   )r+   indexr   r   r   __getitem__5   s   
zRandomDataset.__getitem__c                 C   s   | j S r&   )r'   r+   r   r   r   __len__8   s   zRandomDataset.__len__N)__name__
__module____qualname__r.   r0   r2   r   r   r   r   r%   0   s    r%   c                       s   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdeeef fddZdeeef fddZdeeef fddZdd Z  ZS )ExampleModelc                    s<   t i }t | tjjjddd| _tj	d| _
d S )N    )in_featuresout_features)r
   
structuredsuperr.   r(   nnmodulesLinearl1BatchNorm1dbn)r+   argskwargscfg	__class__r   r   r.   =   s   
zExampleModel.__init__c                 C      t dd}tjjj|ddS Nr7         
batch_sizer%   r(   utilsr*   
DataLoaderr+   datasetr   r   r   train_dataloaderC      
zExampleModel.train_dataloaderc                 C   rG   rH   rM   rP   r   r   r   val_dataloaderG   rS   zExampleModel.val_dataloaderc                 C   s8   t dd}tjjj|dd}dd tt|D | _|S )Nr7   rI   rJ   rK   c                 S   s   g | ]}d  |qS )ztest_{}_)format)r   idxr   r   r   r   N   s    z0ExampleModel.test_dataloader.<locals>.<listcomp>)r%   r(   rN   r*   rO   ranger'   _test_names)r+   rQ   dlr   r   r   test_dataloaderK   s   
zExampleModel.test_dataloaderc                 C   s   |  | | S r&   )r?   rA   sum)r+   batchr   r   r   forwardQ      zExampleModel.forwardc                 C   s   | |S r&   r   )r+   r\   	batch_idxr   r   r   training_stepT   s   zExampleModel.training_stepc                 C      | |}| j | |S r&   )validation_step_outputsappendr+   r\   r_   lossr   r   r   validation_stepW      zExampleModel.validation_stepc                 C   ra   r&   )test_step_outputsrc   rd   r   r   r   	test_step\   rg   zExampleModel.test_stepc                 C   s   t jj|  ddS )NgMbP?)lr)r(   optimSGDr$   r1   r   r   r   configure_optimizersa   r^   z!ExampleModel.configure_optimizersc                 C      d S r&   r   r1   r   r   r   list_available_modelsd      z"ExampleModel.list_available_modelstrain_data_configc                 C   rn   r&   r   )r+   rq   r   r   r   setup_training_datag   rp   z ExampleModel.setup_training_dataval_data_configc                 C   rn   r&   r   r+   rs   r   r   r   setup_validation_dataj   rp   z"ExampleModel.setup_validation_datac                 C   rn   r&   r   rt   r   r   r   setup_test_datam   rp   zExampleModel.setup_test_datac                 C   s&   |  dt| j  | j  d S )Nval_loss)logr(   stackrb   meanclearr1   r   r   r   on_validation_epoch_endp   s   z$ExampleModel.on_validation_epoch_end)r3   r4   r5   r.   rR   rT   rZ   r]   r`   rf   ri   rm   ro   r   r	   r   rr   ru   rv   r|   __classcell__r   r   rE   r   r6   <   s    r6   c                   @   st   e Zd Zejjdd Zejjejddd Zejjejddd Z	ejjdd	 Z
ejjd
d ZdS )TestEMAConfigc                 C   s>   t jtdd tdd W d    d S 1 sw   Y  d S )Nzbetween 0 and 1matchrJ   )decay)pytestraisesr   r   r1   r   r   r   test_ema_valuev   s   "zTestEMAConfig.test_ema_valueGPUc           
         s  t j|d}G dd dt}t }|  tddddddd	d gd
	}t|ddit|ddid t	t
 |j|d W d   n1 sHw   Y  t j|d}t }G  fdddt}tdddddd	dd}t|ddit|ddid |j|  |j||d t j|d}	tdddddd	dd}t|ddit|ddid |j||	d t |	 tdddddd	dd}t|dddt|ddid tj	tdd |j||d W d   dS 1 sw   Y  dS )zYTest to ensure that when we re-load the EMA callback, it loads the EMA weights correctly.saved_statec                   @   s   e Zd Zd	ddZdS )
z=TestEMAConfig.test_ema_saved_state.<locals>.TerminateCallbackr   
pl.Trainerr   pl.LightningModulereturnNc                 S   s   t ||| _t|| _tr&   )r    saved_ema_weightsr   pl_module_weights
SystemExit)r+   r   r   r   r   r   on_train_epoch_end   s   
zPTestEMAConfig.test_ema_saved_state.<locals>.TerminateCallback.on_train_epoch_endr   r   r   r   r   N)r3   r4   r5   r   r   r   r   r   TerminateCallback   s    r   rJ      rI   Fg      ?gpu)	
max_epochslimit_val_batcheslimit_train_batchesloggerval_check_intervalenable_checkpointingacceleratordevicesr   enableTfilename{epoch}-{step}emaexplicit_log_dircheckpoint_callback_params)modelNcheckpoints/epoch=0-step=8.ckptc                       s   e Zd Zd	 fddZdS )
z>TestEMAConfig.test_ema_saved_state.<locals>.CheckStateCallbackr   r   r   r   r   Nc                    s   t |}t| jD ]\}}t| | sJ q
t||}t| jD ]\}}t| | s6J q&|jD ]}t	|t
sCJ |jdksJJ q:d S )N   )r   zipr   r(   allclosecpur    r   
optimizersr   r   current_step)r+   r   r   r   r   ycurrent_ema_weights	optimizerterminate_callbackr   r   on_train_start   s   

zMTestEMAConfig.test_ema_saved_state.<locals>.CheckStateCallback.on_train_startr   )r3   r4   r5   r   r   r   r   r   CheckStateCallback   s    r   r   )r   r   r   r   r   r   r   )	ckpt_path#checkpoints/epoch=0-step=8-EMA.ckptr   validate_original_weightsz9Unable to find the associated EMA weights when re-loadingr   )ospathjoinr   r6   r   r   strr   r   r   fitr   rc   remover   )
r+   tmpdircaplog	temp_pathr   r   r   resume_pathr   ema_pathr   r   r   test_ema_saved_state{   s   			
	"z"TestEMAConfig.test_ema_saved_statec           
      C   s   |d }t  }tdddddd}t|dddt|dd	id
 tdd |jD s,J || t||}tj	
|d s@J |d }tj	
|sLJ t t|}t|  |D ]\}}	t| |	 slJ q\dS )zdTest to ensure that the exp manager adds the EMA callback, and we save an additional EMA checkpoint.exp_manager_testr   Fr   )r   r   r   r   r   Tr   r   r   r   c                 s   s    | ]}t |tV  qd S r&   r   )r   callbackr   r   r   	<genexpr>  s    z=TestEMAConfig.test_exp_manager_ema_weights.<locals>.<genexpr>r   r   N)r6   r   r   r   anyr   r   r    r   r   existsload_from_checkpointr   
state_dictvaluesr(   r   r   )
r+   r   tmp_pathr   r   ema_weightsr   duplicate_modelsaved_weight
ema_weightr   r   r   test_exp_manager_ema_weights   s(   

z*TestEMAConfig.test_exp_manager_ema_weightsc                 C   st   |d }t  }d}tddddd}t|ddit|d	|id
 || tt|d |d d d ks8J dS )zHTest to ensure that EMA correctly ensures we only keep topk checkpoints.r      
   Fr   r   r   r   r   r   T
save_top_kr   checkpoints/rJ   Nr6   r   r   r   r   r'   r   listdirr+   r   r   r   r   r   r   r   r   !test_exp_manager_ema_weights_topk  s   
*z/TestEMAConfig.test_exp_manager_ema_weights_topkc                 C   s   |d }t  }d}tddddd}t|ddit|d	|id
 || tt|d |d d d ks8J d}tddddd}t|ddit|dd	|id || tt|d |d d d ksjJ dS )zJTest to ensure that we always keep top_k checkpoints, even after resuming.r   r   r   Fr   r   r   Tr   r   r   rJ   )r   r   resume_if_existsr   Nr   r   r   r   r   (test_exp_manager_ema_weights_topk_resume$  s2   
&
	*z6TestEMAConfig.test_exp_manager_ema_weights_topk_resumeN)r3   r4   r5   r   markunitr   run_only_onr   r   r   r   r   r   r   r   r~   u   s    

w

r~   c                   @   s   e Zd Zejjejdddejdejje	 pe	d dk ddd	gejd
ddgejdddgej
ddd Zejjejd
ddgejdddgdd Zdd Zejjdd ZdS )TestEMATrain	precisionr7   rI   bf16r   r   z(bfloat16 is not supported on this device)reason)marksaccumulate_grad_batchesr   rJ   r   TFr   c                 C   s   | j ||d||d d S )Nr   r   r   r   r   r   run_training_test)r+   test_data_dirr   r   r   r   r   r   r   test_ema_run_cudaM  s   
zTestEMATrain.test_ema_run_cudac                 C   s   | j ||dd|d d S )Nr   r7   r   r   )r+   r   r   r   r   r   r   r   test_ema_run_cpuo  s   
zTestEMATrain.test_ema_run_cpuc                 C   s   t d t }td|ddd|ddd|dd}t|d|dd	t|d
did |jt  |j	dt
  |j|| d d S )N{   r   r   Fr   )r   r   r   r   r   r   num_sanity_val_stepsenable_model_summaryr   r   r   Tg+?)r   r   r   r   r   r   )r   val_dataloaders)plseed_everythingr6   r   r   r   r   rc   EMAAssertCallbackinsertEMAValidationAssertCallbackr   rR   )r+   r   r   r   r   r   r   r   r   r   r   r   {  s2   

	zTestEMATrain.run_training_testc                 C   sL   |d }t  }tdddddd}t|ddit|ddid || d	S )
zTTest to ensure that we save the model correctly when save best model is set to True.r   r   F)r   r   r   r   r   r   Tsave_best_modelr   N)r6   r   r   r   r   )r+   r   r   r   r   r   r   r   !test_ema_run_with_save_best_model  s   z.TestEMATrain.test_ema_run_with_save_best_modelN)r3   r4   r5   r   r   r   parametrizeparamskipifDEVICE_CAPABILITYr   r   r   r   r   r   r   r   r   r   L  s4    
	r   c                   @   s8   e Zd ZdddZddddd	ed
ededdfddZdS )r   r   r   r   r   r   Nc                 C   s>   t |}t||| _t|| jD ]\}}t||sJ qd S r&   )r   r    r   r   r(   r   )r+   r   r   model_weightsr   r   r   r   r   r     s
   z EMAAssertCallback.on_train_startoutputsr\   r_   c                 C   s   |d |j  dkrd S dd |jD d }|j}g }t|}	t| j|	D ]\}
}|
| }||d|  7 }|| q$t||}t||D ]\}}t	||sPJ qD|| _d S )Nr   r   c                 S   r   r   r   r   r   r   r   r     r   z8EMAAssertCallback.on_train_batch_end.<locals>.<listcomp>)
r   r   r   r   r   r   rc   r    r(   r   )r+   r   r   r   r\   r_   r   r   expected_ema_weightsnew_weightsr   
new_weightexpected_ema_weightr   actual_ema_weightr   r   r   on_train_batch_end  s   

z$EMAAssertCallback.on_train_batch_endr   )r3   r4   r5   r   r   r   intr   r   r   r   r   r     s    
r   c                       s(   e Zd Zd fddZdd	d
Z  ZS )r   r   r   r   r   r   Nc                    sv   dd |j D d }t|| _t||| _t || |js5|jr7t	| jt|D ]\}}t
|| q*d S d S d S )Nc                 S   r   r   r   r   r   r   r   r     r   zCEMAValidationAssertCallback.on_validation_start.<locals>.<listcomp>r   )r   r   _original_weightsr    _ema_weightsr;   on_validation_startr   _ema_initializedr   r(   r   )r+   r   r   r   r   module_weightsrE   r   r   r    s   
z/EMAValidationAssertCallback.on_validation_startc                 C   s^   dd |j D d }|js)t|}|jr+t| j|D ]\}}t| |  qd S d S d S )Nc                 S   r   r   r   r   r   r   r   r     r   zAEMAValidationAssertCallback.on_validation_end.<locals>.<listcomp>r   )	r   r   r   r  r   r   r(   r   r   )r+   r   r   r   r   orig_weightsr  r   r   r   on_validation_end  s   z-EMAValidationAssertCallback.on_validation_endr   )r3   r4   r5   r  r  r}   r   r   rE   r   r     s    r   )+os.pathr   typingr   r   r   lightning.pytorchpytorchr   r   r(   r   r   &lightning.pytorch.utilities.exceptionsr   !lightning.pytorch.utilities.typesr   	omegaconfr	   r
   !nemo.collections.common.callbacksr   %nemo.collections.common.callbacks.emar   	nemo.corer   nemo.utils.exp_managerr   r   cudais_availableget_device_capabilityr    r   rN   r*   Datasetr%   r6   r~   r   r   r   r   r   r   r   <module>   s2   

9 X^