o oiNL@s8ddlZddlZddlZddlZddlmZddlmZmZm Z m Z m Z m Z ddl Z ddlmZddlmZddlmZddlmZddlmZddlmZdd lmZmZdd lmZdd l m!Z!m"Z"ej#$d dursdd l%m&Z&ndd l&m&Z&erddl'm(Z)ddl*m+Z+edZ,e-e.Z/d8ddde0de0fddZ1GdddZ2       d9dd ddd!e3d"e3d#e4d$e0d%e e3d&e5de0de e2fd'd(Z6dd de e0effd)d*Z7dd d#e4d%e e3ddfd+d,Z8dd d-e e0efddfd.d/Z9Gd0d1d1eZ:Gd2d3d3eZ;Gd4d5d5eZr2dr*@Fr/r0min_lrmax_lrr(r%early_stop_threshold update_attrc Cs|jr tddS|rt||}tj|jdtd} |j | } | | |j } t |} t||||jr>|jt||||} | |t|| |j || kr`td|j d| j|jdj|jdjd|jj| _t|| |jr|j|j | j| _|r| } | durt!||| td | |j"#| |j $| d |j_%d |jj&_%d|jj&j'_(| S) a1Enables the user to do a range test of good initial learning rates, to reduce the amount of guesswork in picking a good starting learning rate. Args: trainer: A Trainer instance. model: Model to tune. min_lr: minimum learning rate to investigate max_lr: maximum learning rate to investigate num_training: number of learning rates to test mode: Search strategy to update learning rate after each batch: - ``'exponential'``: Increases the learning rate exponentially. - ``'linear'``: Increases the learning rate linearly. early_stop_threshold: Threshold for stopping the search. If the loss at any point is larger than early_stop_threshold*best_loss then the search is stopped. To disable, set to None. update_attr: Whether to update the learning rate attribute or not. attr_name: Name of the attribute which stores the learning rate. The names 'learning_rate' or 'lr' get automatically detected. Otherwise, set the name here. z>Skipping learning rate finder since `fast_dev_run` is enabled.Nz .lr_find_z.ckptzLR finder stopped early after z steps due to diverging loss.r)rrIzLearning rate set to F)) fast_dev_runrr#ospathjoindefault_root_diruuiduuid4r7 broadcastsave_checkpoint global_step__lr_finder_dump_params__lr_finder_reset_paramsprogress_bar_callbackdisabler$rE _try_loop_runrJinfor+update callbacksr^r_fit_looptotal_batch_idxr,__lr_finder_restore_paramsenabler[r_checkpoint_connectorrestoreremove_checkpoint restarting epoch_loopval_loop_combined_loader)r/rryrzr(r%r{r|r ckpt_path start_stepsparams lr_finderrr!r!r"_lr_findsF!       "        rc Cs0|jj|jj|j|j|jj|jt|j dS)N)r8r>rloggers max_stepslimit_val_batchesloop_state_dict) r7r8r>rrrrrr state_dict)r/r!r!r"r9s rcCsVddlm}g|j_t||ddg|_|jdur|nd|_||j|jj _ ||_ dS)Nr) DummyLoggerr2)progress_bar_refresh_rate) lightning.pytorch.loggers.loggerrr7r> _LRCallbackrloggerrrrrr)r/r(r{rr!r!r"rEs  rrcCsj|d|j_|d|j_|d|_|d|_|j}|d|j_|d|_| t |dd|_ d|_ dS) Nr8r>rrrrrF) r7r8r>rrrrrrload_state_dictrr should_stopr/rloopr!r!r"rRs       rc @s~eZdZdZ   ddedeededefd d Zed d d dde deddf ddZ ed d d dde de deddf ddZ dS)raSpecial callback used by the learning rate finder. This callback logs the learning rate before each batch and logs the corresponding loss after each batch. Args: num_training: number of iterations done by the learning rate finder early_stop_threshold: threshold for stopping the search. If the loss at any point is larger than ``early_stop_threshold*best_loss`` then the search is stopped. To disable, set to ``None``. progress_bar_refresh_rate: rate to refresh the progress bar for the learning rate finder beta: smoothing value, the loss being logged is a running average of loss values logged until now. ``beta`` controls the forget rate i.e. if ``beta=0`` all past information is ignored. rxr\(\?r(r{rbetacCs:||_||_||_g|_g|_d|_d|_||_d|_dS)Ng) r(r{rr_r^avg_loss best_lossr progress_bar)r-r(r{rrr!r!r"r.qs z_LRCallback.__init__r/r0 pl_modulerbatch batch_idxrNcCsX|jjd|jdkr dS|jr|jdurtd|jd|_|j|j dj j ddS)zACalled before each training batch, logs the lr that will be used.r2rNzFinding best initial lr)desctotal) rraccumulate_grad_batchesrrrr(r^appendr>rDr)r-r/rrrr!r!r"on_train_batch_starts z _LRCallback.on_train_batch_startoutputsc Cs|jjd|jdkr dS|s|jtddS|jr!|jt|t j r)|n|d}|dus3J| }|j }|j |jd|j ||_|jd|j |d} |jdurp|dkrp| |j|jkrpd|_|jrp|j|j|j|_| |jks|dkr| |_|j| dS)z>Called when the training batch ends, logs the calculated loss.r2rNnanrIT)rrrr_rrsrr isinstancereTensorrkrrrr{rrcloser7r) r-r/rrrr loss_tensor current_loss current_step smoothed_lossr!r!r"on_train_batch_ends.    z_LRCallback.on_train_batch_end)rxrr) rnrorprqrtrrsr.r rrrrr!r!r!r"r`sP   rc leZdZdZddejjdededeffdd Z e d e efd d Z e d eee effd d ZZS)r<a=Linearly increases the learning rate between two boundaries over a number of iterations. Args: optimizer: wrapped optimizer. end_lr: the final learning rate. num_iter: the number of iterations over which the test occurs. last_epoch: the index of last epoch. Default: -1. r?end_lrnum_iter last_epochc||_||_t||dSNrrsuperr.r-r?rrr __class__r!r"r.z_LinearLR.__init__rcJjd}|jjdkrfddjD}ntj}|_|S)Nr2rcsg|] }|j|qSr!r.0base_lrrr-r!r" z$_LinearLR.get_lr..rrbase_lrslist_lrr- curr_itervalr!rr"get_lr    z_LinearLR.get_lrcC|jSrrr-r!r!r"rz _LinearLR.lrrrnrorprqreoptim Optimizerrsrtr.r rrpropertyrr __classcell__r!r!rr"r<$ $r<c r)r=aGExponentially increases the learning rate between two boundaries over a number of iterations. Arguments: optimizer: wrapped optimizer. end_lr: the final learning rate. num_iter: the number of iterations over which the test occurs. last_epoch: the index of last epoch. Default: -1. rr?rrrcrrrrrr!r"r.rz_ExponentialLR.__init__rcr)Nr2rcsg|] }|j|qSr!rrrr!r"rrz)_ExponentialLR.get_lr..rrr!rr"rrz_ExponentialLR.get_lrcCrrrrr!r!r"rrz_ExponentialLR.lrrrr!r!rr"r=rr=cCs*|j}|t|dd|_|dS)NrF)rrrrrunrr!r!r"rs r)r)rvr2rwr*rxFr)> importlibloggingr~rcopyrtypingrrrrrrre lightning_utilities.core.importsr torch.optim.lr_schedulerr typing_extensionsr lightning.pytorchpytorchpllightning.pytorch.callbacksr &lightning.pytorch.utilities.exceptionsr #lightning.pytorch.utilities.parsingrr%lightning.pytorch.utilities.rank_zeror!lightning.pytorch.utilities.typesrrutil find_spec tqdm.autorrSrTr]matplotlib.axesrrR getLoggerrnrJrrr#r$rsrtrurrrrrr<r=rr!r!r!r"s|                d  ]%"%