o
    wiG                     @   sP  d dl Zd dlmZmZmZ d dlmZ d dlm	Z
 d dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ dZej  rbej! Zej"ddddd Z#dd Z$dd Z%G dd dej&j'j(Z)G dd deZ*G dd dZ+G dd dZ,G dd deZ-G dd  d eZ.dS )!    N)AnyDictUnionpatch)CallbackTrainer)MisconfigurationException)STEP_OUTPUT)
DictConfig	OmegaConf)EMA)EMAOptimizer)ModelPT)exp_managerTmodule)autousescopec                   c   s:    t dd d d V  W d    d S 1 sw   Y  d S )Nz9nemo.lightning.callback_group.CallbackGroup.update_config)return_valuer    r   r   ^/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/tests/collections/common/test_ema.py_mock_onelogger_update_config%   s   "r   c                 C   s4   dd |j D d }|| t| }|| |S )Nc                 S      g | ]	}t |tr|qS r   
isinstancer   .0xr   r   r   
<listcomp>,       z'extract_ema_weights.<locals>.<listcomp>r   )	callbacksswap_model_weightsextract_weights)	pl_moduletrainerema_callbackweightsr   r   r   extract_ema_weights+   s
   

r'   c                 C   s   dd |   D S )Nc                 S   s   g | ]}|   qS r   )detachclone)r   wr   r   r   r   4   s    z#extract_weights.<locals>.<listcomp>)
parameters)r#   r   r   r   r"   3   s   r"   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )RandomDatasetc                 C   s   || _ t||| _d S N)lentorchrandndata)selfsizelengthr   r   r   __init__8   s   zRandomDataset.__init__c                 C   s
   | j | S r-   )r1   )r2   indexr   r   r   __getitem__<   s   
zRandomDataset.__getitem__c                 C   s   | j S r-   )r.   r2   r   r   r   __len__?   s   zRandomDataset.__len__N)__name__
__module____qualname__r5   r7   r9   r   r   r   r   r,   7   s    r,   c                       s   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdeeef fddZdeeef fddZdeeef fddZdd Z  ZS )ExampleModelc                    s<   t i }t | tjjjddd| _tj	d| _
d S )N    )in_featuresout_features)r   
structuredsuperr5   r/   nnmodulesLinearl1BatchNorm1dbn)r2   argskwargscfg	__class__r   r   r5   D   s   
zExampleModel.__init__c                 C      t dd}tjjj|ddS Nr>         
batch_sizer,   r/   utilsr1   
DataLoaderr2   datasetr   r   r   train_dataloaderJ      
zExampleModel.train_dataloaderc                 C   rN   rO   rT   rW   r   r   r   val_dataloaderN   rZ   zExampleModel.val_dataloaderc                 C   s8   t dd}tjjj|dd}dd tt|D | _|S )Nr>   rP   rQ   rR   c                 S   s   g | ]}d  |qS )ztest_{}_)format)r   idxr   r   r   r   U   s    z0ExampleModel.test_dataloader.<locals>.<listcomp>)r,   r/   rU   r1   rV   ranger.   _test_names)r2   rX   dlr   r   r   test_dataloaderR   s   
zExampleModel.test_dataloaderc                 C   s   |  | | S r-   )rF   rH   sum)r2   batchr   r   r   forwardX      zExampleModel.forwardc                 C   s   | |S r-   r   )r2   rc   	batch_idxr   r   r   training_step[   s   zExampleModel.training_stepc                 C      | |}| j | |S r-   )validation_step_outputsappendr2   rc   rf   lossr   r   r   validation_step^      zExampleModel.validation_stepc                 C   rh   r-   )test_step_outputsrj   rk   r   r   r   	test_stepc   rn   zExampleModel.test_stepc                 C   s   t jj|  ddS )NgMbP?)lr)r/   optimSGDr+   r8   r   r   r   configure_optimizersh   re   z!ExampleModel.configure_optimizersc                 C      d S r-   r   r8   r   r   r   list_available_modelsk      z"ExampleModel.list_available_modelstrain_data_configc                 C   ru   r-   r   )r2   rx   r   r   r   setup_training_datan   rw   z ExampleModel.setup_training_dataval_data_configc                 C   ru   r-   r   r2   rz   r   r   r   setup_validation_dataq   rw   z"ExampleModel.setup_validation_datac                 C   ru   r-   r   r{   r   r   r   setup_test_datat   rw   zExampleModel.setup_test_datac                 C   s&   |  dt| j  | j  d S )Nval_loss)logr/   stackri   meanclearr8   r   r   r   on_validation_epoch_endw   s   z$ExampleModel.on_validation_epoch_end)r:   r;   r<   r5   rY   r[   ra   rd   rg   rm   rp   rt   rv   r   r   r   ry   r|   r}   r   __classcell__r   r   rL   r   r=   C   s    r=   c                   @   st   e Zd Zejjdd Zejjejddd Zejjejddd Z	ejjdd	 Z
ejjd
d ZdS )TestEMAConfigc                 C   s>   t jtdd tdd W d    d S 1 sw   Y  d S )Nzbetween 0 and 1matchrQ   )decay)pytestraisesr	   r   r8   r   r   r   test_ema_value}   s   "zTestEMAConfig.test_ema_valueGPUc           
         s  t j|d}G dd dt}t }|  tddddddd	d gd
	}t|ddit|ddid t	t
 |j|d W d   n1 sHw   Y  t j|d}t }G  fdddt}tdddddd	dd}t|ddit|ddid |j|  |j||d t j|d}	tdddddd	dd}t|ddit|ddid |j||	d t |	 tdddddd	dd}t|dddt|ddid tj	tdd |j||d W d   dS 1 sw   Y  dS )zYTest to ensure that when we re-load the EMA callback, it loads the EMA weights correctly.saved_statec                   @   s   e Zd Zd	ddZdS )
z=TestEMAConfig.test_ema_saved_state.<locals>.TerminateCallbackr$   
pl.Trainerr#   pl.LightningModulereturnNc                 S   s   t ||| _t|| _tr-   )r'   saved_ema_weightsr"   pl_module_weights
SystemExit)r2   r$   r#   r   r   r   on_train_epoch_end   s   
zPTestEMAConfig.test_ema_saved_state.<locals>.TerminateCallback.on_train_epoch_endr$   r   r#   r   r   N)r:   r;   r<   r   r   r   r   r   TerminateCallback   s    r   rQ      rP   Fg      ?gpu)	
max_epochslimit_val_batcheslimit_train_batchesloggerval_check_intervalenable_checkpointingacceleratordevicesr    enableTfilename{epoch}-{step}emaexplicit_log_dircheckpoint_callback_params)modelNcheckpoints/epoch=0-step=8.ckptc                       s   e Zd Zd	 fddZdS )
z>TestEMAConfig.test_ema_saved_state.<locals>.CheckStateCallbackr$   r   r#   r   r   Nc                    s   t |}t| jD ]\}}t| | sJ q
t||}t| jD ]\}}t| | s6J q&|jD ]}t	|t
sCJ |jdksJJ q:d S )N   )r"   zipr   r/   allclosecpur'   r   
optimizersr   r   current_step)r2   r$   r#   r&   r   ycurrent_ema_weights	optimizerterminate_callbackr   r   on_train_start   s   

zMTestEMAConfig.test_ema_saved_state.<locals>.CheckStateCallback.on_train_startr   )r:   r;   r<   r   r   r   r   r   CheckStateCallback   s    r   r   )r   r   r   r   r   r   r   )	ckpt_path#checkpoints/epoch=0-step=8-EMA.ckptr   validate_original_weightsz9Unable to find the associated EMA weights when re-loadingr   )ospathjoinr   r=   r   r   strr   r   r   fitr    rj   remover	   )
r2   tmpdircaplog	temp_pathr   r   r$   resume_pathr   ema_pathr   r   r   test_ema_saved_state   s   			
	"z"TestEMAConfig.test_ema_saved_statec           
      C   s   |d }t  }tdddddd}t|dddt|dd	id
 tdd |jD s,J || t||}tj	
|d s@J |d }tj	
|sLJ t t|}t|  |D ]\}}	t| |	 slJ q\dS )zdTest to ensure that the exp manager adds the EMA callback, and we save an additional EMA checkpoint.exp_manager_testr   Fr   )r   r   r   r   r   Tr   r   r   r   c                 s   s    | ]}t |tV  qd S r-   r   )r   callbackr   r   r   	<genexpr>
  s    z=TestEMAConfig.test_exp_manager_ema_weights.<locals>.<genexpr>r   r   N)r=   r   r   r   anyr    r   r'   r   r   existsload_from_checkpointr   
state_dictvaluesr/   r   r   )
r2   r   tmp_pathr   r$   ema_weightsr   duplicate_modelsaved_weight
ema_weightr   r   r   test_exp_manager_ema_weights   s(   

z*TestEMAConfig.test_exp_manager_ema_weightsc                 C   st   |d }t  }d}tddddd}t|ddit|d	|id
 || tt|d |d d d ks8J dS )zHTest to ensure that EMA correctly ensures we only keep topk checkpoints.r      
   Fr   r   r   r   r   r   T
save_top_kr   checkpoints/rQ   Nr=   r   r   r   r   r.   r   listdirr2   r   r   r   r   r$   r   r   r   !test_exp_manager_ema_weights_topk  s   
*z/TestEMAConfig.test_exp_manager_ema_weights_topkc                 C   s   |d }t  }d}tddddd}t|ddit|d	|id
 || tt|d |d d d ks8J d}tddddd}t|ddit|dd	|id || tt|d |d d d ksjJ dS )zJTest to ensure that we always keep top_k checkpoints, even after resuming.r   r   r   Fr   r   r   Tr   r   r   rQ   )r   r   resume_if_existsr   Nr   r   r   r   r   (test_exp_manager_ema_weights_topk_resume+  s2   
&
	*z6TestEMAConfig.test_exp_manager_ema_weights_topk_resumeN)r:   r;   r<   r   markunitr   run_only_onr   r   r   r   r   r   r   r   r   |   s    

w

r   c                   @   s   e Zd Zejjejdddejdejje	 pe	d dk ddd	gejd
ddgejdddgej
ddd Zejjejd
ddgejdddgdd Zdd Zejjdd ZdS )TestEMATrain	precisionr>   rP   bf16r   r   z(bfloat16 is not supported on this device)reason)marksaccumulate_grad_batchesr   rQ   r   TFr   c                 C   s   | j ||d||d d S )Nr   r   r   r   r   r   run_training_test)r2   test_data_dirr   r   r   r   r   r   r   test_ema_run_cudaT  s   
zTestEMATrain.test_ema_run_cudac                 C   s   | j ||dd|d d S )Nr   r>   r   r   )r2   r   r   r   r   r   r   r   test_ema_run_cpuv  s   
zTestEMATrain.test_ema_run_cpuc                 C   s   t d t }td|ddd|ddd|dd}t|d|dd	t|d
did |jt  |j	dt
  |j|| d d S )N{   r   r   Fr   )r   r   r   r   r   r   num_sanity_val_stepsenable_model_summaryr   r   r   Tg+?)r   r   r   r   r   r   )r   val_dataloaders)plseed_everythingr=   r   r   r   r    rj   EMAAssertCallbackinsertEMAValidationAssertCallbackr   rY   )r2   r   r   r   r   r   r   r$   r   r   r   r     s2   

	zTestEMATrain.run_training_testc                 C   sL   |d }t  }tdddddd}t|ddit|ddid || d	S )
zTTest to ensure that we save the model correctly when save best model is set to True.r   r   F)r   r   r   r   r   r   Tsave_best_modelr   N)r=   r   r   r   r   )r2   r   r   r   r$   r   r   r   !test_ema_run_with_save_best_model  s   z.TestEMATrain.test_ema_run_with_save_best_modelN)r:   r;   r<   r   r   r   parametrizeparamskipifDEVICE_CAPABILITYr   r   r   r   r   r   r   r   r   r   S  s4    
	r   c                   @   s8   e Zd ZdddZddddd	ed
ededdfddZdS )r   r$   r   r#   r   r   Nc                 C   s>   t |}t||| _t|| jD ]\}}t||sJ qd S r-   )r"   r'   r   r   r/   r   )r2   r$   r#   model_weightsr   r   r   r   r   r     s
   z EMAAssertCallback.on_train_startoutputsrc   rf   c                 C   s   |d |j  dkrd S dd |jD d }|j}g }t|}	t| j|	D ]\}
}|
| }||d|  7 }|| q$t||}t||D ]\}}t	||sPJ qD|| _d S )Nr   r   c                 S   r   r   r   r   r   r   r   r     r   z8EMAAssertCallback.on_train_batch_end.<locals>.<listcomp>)
r   r    r   r"   r   r   rj   r'   r/   r   )r2   r$   r#   r   rc   rf   r%   r   expected_ema_weightsnew_weightsr   
new_weightexpected_ema_weightr   actual_ema_weightr   r   r   on_train_batch_end  s   

z$EMAAssertCallback.on_train_batch_endr   )r:   r;   r<   r   r
   r   intr  r   r   r   r   r     s    
r   c                       s(   e Zd Zd fddZdd	d
Z  ZS )r   r$   r   r#   r   r   Nc                    sv   dd |j D d }t|| _t||| _t || |js5|jr7t	| jt|D ]\}}t
|| q*d S d S d S )Nc                 S   r   r   r   r   r   r   r   r     r   zCEMAValidationAssertCallback.on_validation_start.<locals>.<listcomp>r   )r    r"   _original_weightsr'   _ema_weightsrB   on_validation_startr   _ema_initializedr   r/   r   )r2   r$   r#   r%   r   module_weightsrL   r   r   r	    s   
z/EMAValidationAssertCallback.on_validation_startc                 C   s^   dd |j D d }|js)t|}|jr+t| j|D ]\}}t| |  qd S d S d S )Nc                 S   r   r   r   r   r   r   r   r     r   zAEMAValidationAssertCallback.on_validation_end.<locals>.<listcomp>r   )	r    r   r"   r
  r   r  r/   r   r   )r2   r$   r#   r%   r   orig_weightsr  r   r   r   on_validation_end  s   z-EMAValidationAssertCallback.on_validation_endr   )r:   r;   r<   r	  r  r   r   r   rL   r   r     s    r   )/os.pathr   typingr   r   r   unittest.mockr   lightning.pytorchpytorchr   r   r/   r   r   &lightning.pytorch.utilities.exceptionsr	   !lightning.pytorch.utilities.typesr
   	omegaconfr   r   !nemo.collections.common.callbacksr   %nemo.collections.common.callbacks.emar   	nemo.corer   nemo.utils.exp_managerr   r   cudais_availableget_device_capabilityfixturer   r'   r"   rU   r1   Datasetr,   r=   r   r   r   r   r   r   r   r   <module>   s8   


9 X^