o }oi3@sddlZddlZddlZddlZddlmZmZmZddlm Z ddl Z ddlm Z ddl mZddlmZGddde Ze dd Zdd d ZGd d d e jjZdS)N)AnyDictIterable)Callback)MisconfigurationException)rank_zero_infoc @seZdZdZ   d(dedededefdd Zd)ddZd)ddZ d)ddZ d)ddZ d)ddZ d d defddZ d d defddZd*d d defdd Zejd+d!d"Zejd+d#d$Zd d d d d%eeefddfd&d'ZdS),EMAaj Implements Exponential Moving Averaging (EMA). When training a model, this callback will maintain moving averages of the trained parameters. When evaluating, we use the moving averages copy of the trained parameters. When saving, we save an additional set of parameters with the prefix `ema`. Args: decay: The exponential decay used when calculating the moving average. Has to be between 0-1. validate_original_weights: Validate the original weights, as apposed to the EMA weights. every_n_steps: Apply EMA every N steps. cpu_offload: Offload weights to CPU. Fdecayvalidate_original_weights every_n_steps cpu_offloadcCs@d|kr dkstdtd||_||_||_||_dS)Nrr z'EMA decay value must be between 0 and 1)rr r r r )selfr r r r rY/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/common/callbacks/ema.py__init__*s z EMA.__init__trainer pl.Trainer pl_modulepl.LightningModulereturnNcs4js|jntdfddjD_dS)Ncpuc s.g|]}t|tst|jjjdqS))devicer r current_step) isinstance EMAOptimizerr r global_step).0optimrrrrr :sz$EMA.on_fit_start..)r rtorch optimizersrrrrrr on_fit_start8s zEMA.on_fit_startcC||r ||dSdSN_should_validate_ema_weightsswap_model_weightsr#rrron_validation_startF zEMA.on_validation_startcCr%r&r'r#rrron_validation_endJr+zEMA.on_validation_endcCr%r&r'r#rrr on_test_startNr+zEMA.on_test_startcCr%r&r'r#rrr on_test_endRr+zEMA.on_test_endcCs|j o||Sr&)r _ema_initializedrrrrrr(Vsz EMA._should_validate_ema_weightscCstdd|jDS)Ncss|]}t|tVqdSr&)rr)r optimizerrrr Zsz'EMA._ema_initialized..)anyr"r0rrrr/YszEMA._ema_initializedsaving_ema_modelcCs(|jD]}t|ts J||qdSr&)r"rrswitch_main_parameter_weights)rrr4r1rrrr)\s  zEMA.swap_model_weightsc cs<|j|ddz dVW|j|dddS|j|ddw)zS Saves an EMA copy of the model + EMA optimizer states for resume. T)r4NF)r)r0rrrsave_ema_modelas "zEMA.save_ema_modelc csV|jD] }t|ts Jd|_qzdVW|jD]}d|_qdS|jD]}d|_q$w)NTF)r"rrsave_original_optimizer_state)rrr1rrrr7ls  z!EMA.save_original_optimizer_state checkpointc Cs|j}|j}|rR|durTdt|jvrV|j}|d|r$tddS||d|}tj |rKt j |t ddd}|d|d<~tddStd |dSdSdS) NNeMoz-EMAzloading EMA based weights. The callback will treat the loaded EMA weights as the main weights and create a new EMA copy when training.rF) map_location weights_onlyoptimizer_stateszEMA state has been restored.z}Unable to find the associated EMA weights when re-loading, training will start with new EMA weights. Expected them to be at: )checkpoint_callback ckpt_pathtype__name__FILE_EXTENSIONendswithrreplaceospathexistsr!loadrr) rrrr8r=r>extema_pathema_state_dictrrron_load_checkpointws*    zEMA.on_load_checkpoint)Fr F)rrrrrNF)rr)r@ __module__ __qualname____doc__floatboolintrr$r*r,r-r.r(r/r) contextlibcontextmanagerr6r7rstrrrKrrrrrsF         rcCs$t||tj||d|ddS)Ng?)alpha)r! _foreach_mul_ _foreach_add_)ema_model_tuplecurrent_model_tupler rrr ema_updates  r[cCs |dur|t|||dSr&) synchronizer[)rYrZr pre_sync_streamrrrrun_ema_update_cpusr^c @seZdZdZ   d*dejjdejdede d e f d d Z d e ej fd dZ d+ddZd efddZeddZddZd,defddZejd-defddZd d!Zd"d#Zd$d%Zd&d'Zd(d)ZdS).ra EMAOptimizer is a wrapper for torch.optim.Optimizer that computes Exponential Moving Average of parameters registered in the optimizer. EMA parameters are automatically updated after every step of the optimizer with the following formula: ema_weight = decay * ema_weight + (1 - decay) * training_weight To access EMA parameters, use ``swap_ema_weights()`` context manager to perform a temporary in-place swap of regular parameters with EMA parameters. Notes: - EMAOptimizer is not compatible with APEX AMP O2. Args: optimizer (torch.optim.Optimizer): optimizer to wrap device (torch.device): device for EMA parameters decay (float): decay factor Returns: returns an instance of torch.optim.Optimizer that computes EMA of parameters Example: model = Model().to(device) opt = torch.optim.Adam(model.parameters()) opt = EMAOptimizer(opt, device, 0.9999) for epoch in range(epochs): training_loop(model, opt) regular_eval_accuracy = evaluate(model) with opt.swap_ema_weights(): ema_eval_accuracy = evaluate(model) H.?r rr1rr r rcCsL||_||_||_||_||_d|_d|_d|_d|_d|_ d|_ d|_ dS)NFTr) r1r rrr r7first_iterationrebuild_ema_paramsstreamthread ema_paramsin_saving_ema_model_context)rr1rr r rrrrrs zEMAOptimizer.__init__rcCsdd|jDS)Ncss"|] }|dD]}|VqqdS)paramsNr)rgroupparamrrrr2s z.EMAOptimizer.all_parameters..) param_groupsrrrrall_parameterszEMAOptimizer.all_parametersNc sjrtddDrtj_d_jr>t }j t fdd|t j dD7_ d_t jddrR|durRjj||d}nj|}r`jd7_|S)Ncss|]}|jVqdSr&)is_cuda)rprrrr2sz$EMAOptimizer.step..Fc3s(|]}t|jjVqdSr&)copydeepcopydatadetachtorrrhrjrrr2s _step_supports_amp_scaling)closure grad_scalerr )joinr`r3rkr!cudaStreamrbralistrdtuplelengetattrr1step_should_update_at_stepupdater)rrvrwkwargs opt_paramslossrrjrrs$    zEMAOptimizer.stepcCs|j|jdkS)Nr)rr rjrrrrrlz#EMAOptimizer._should_update_at_stepcsjdurjtjtjj#tfddD}jjdkr1t j |j Wdn1s;wYjjdkr\t j tj |j jfd_jdSdS)Nc3s"|] }|jjjddVqdS)T) non_blockingN)rqrsrrtrjrrr2s z&EMAOptimizer.update..ryr)targetargs)rb wait_streamr!rycurrent_streamr|rkrr?r[rdr threadingThreadr^rcstart)rcurrent_model_staterrjrrs*     zEMAOptimizer.updatecCs,t|}||||||dSr&)r! empty_likecopy_)rtensor1tensor2tmprrr swap_tensors!s   zEMAOptimizer.swap_tensorsFr4cCs:|||_t||jD] \}}||j|qdSr&)rxreziprkrdrrq)rr4rh ema_paramrrrr5's z*EMAOptimizer.switch_main_parameter_weightsTenabledccs<|r|zdVW|r|dSdS|r|ww)a A context manager to in-place swap regular parameters with EMA parameters. It swaps back to the original regular parameters on context manager exit. Args: enabled (bool): whether the swap should be performed N)r5)rrrrrswap_ema_weights-s   zEMAOptimizer.swap_ema_weightscCs t|j|Sr&)r~r1)rnamerrr __getattr__As zEMAOptimizer.__getattr__cCs0|jdur |j|jdur|jdSdSr&)rbr\rcrxrjrrrrxDs   zEMAOptimizer.joincCsP||jr |jS|js|jnt|}|j||j|j |j d}|S)N)optemarr r ) rxr7r1 state_dictrerdr{rkrr r )rrdrrrrrKs zEMAOptimizer.state_dictcsbj|dtfddt|dD_|d_|d_|d_ d_ dS) Nrc3s|] }|jVqdSr&)rsrrtrjrrr2`sz/EMAOptimizer.load_state_dict..rrr r F) rxr1load_state_dictr|rorprdrr r ra)rrrrjrr\s"    zEMAOptimizer.load_state_dictcCs|j|d|_dS)NT)r1add_param_groupra)r param_grouprrrrfs  zEMAOptimizer.add_param_group)r_r r)NNrL)T)r@rMrNrOr!r OptimizerrrPrRrrTensorrkrrQrno_gradrrr5rSrTrrrxrrrrrrrrs<,    rr&)rSrorDrtypingrrrlightning.pytorchpytorchplr!r&lightning.pytorch.utilities.exceptionsr%lightning.pytorch.utilities.rank_zerorrrr[r^rrrrrrrs     {