o wi $@sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z ddlmZddlmZmZddlmZmZmZmZmZmZmZddlZddlZddlmZdd l m!Z!dd l"m#Z#m$Z$dd l%m&Z&dd l'm(Z(m)Z)dd l*m+Z+m,Z,m-Z-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6m7Z7m8Z8ddl9m:Z:ddl;mZ>m?Z?ddl@mAZAmBZBddlCmDZDddlEmFZFmGZGddlHmIZIddlJmKZKddlLmMZMddlNmOZOddlPmQZQddlRmSZSmTZTmUZUmVZVmWZWddlXmYZYddlZm[Z[dd l\m]Z]m^Z^eOd!d"\Z_Z`z dd#lambZbd$ZcWnedeefy+d%ZcYnwz dd&lamfZfd$ZgWnedeefyDd%ZgYnwGd'd(d(eKZhGd)d*d*eKZiGd+d,d,eKZje Gd-d.d.Zke Gd/d0d0Zle Gd1d2d2Zme Gd3d4d4Zne Gd5d6d6Zoe Gd7d8d8Zpe Gd9d:d:Zqe Gd;d<d<ZrGd=d>d>e#ZsGd?d@d@e#ZtddAdBdCeee6efdDeefdEdFZuddAdBdCeee6effdGdHZvdIeeeewfdDeeeewffdJdKZx % % %  ddAdBdLewdMeydNeydOeydPewdQewfdRdSZzdAdBdTeeewfdUewdVewdWewdDeeewewewff dXdYZ{     $ %ddAdBdUewdVewdWewdTewdZeydMeydDeeewewewffd[d\Z|d]d^Z}d_d`Z~dAdBdUeewgdLeewgdVewdWewdaedbeydceddeydeedfeydgedheydiedjeydkedleydmef$dndoZGdpdqdqe4ZdAdBdLedVewdreydsdtdueyf dvdwZdxdyZGdzd{d{e)ZdAejjdDdfd|d}ZGd~dde0ZddeewefdeydeyfddZdS)N) defaultdict) dataclassfield) timedelta)Path)copymove)Any CollectionDictListOptionalTupleUnion) HydraConfig)get_original_cwd)CallbackModelCheckpoint) EarlyStopping)IntervalTimer) MLFlowLogger NeptuneLoggerTensorBoardLogger WandbLogger)_TrainingEpochLoop) DDPStrategy)_CheckpointConnector) DictConfig OmegaConf open_dict)EMA)IPLEpochStopper)NEMO_ENV_VARNAME_TESTINGNEMO_ENV_VARNAME_VERSION)loggingtimers)AppState)NeMoModelCheckpointPreemptionCallback) get_envbool)NeMoBaseException)is_global_rank_zero)safe_import_from)add_filehandlers_to_pl_logger) ClearMLLogger ClearMLParamsDLLoggerDLLoggerParams MLFlowParams)add_handlers_to_mcore_logger)uninject_model_parallel_rank)import_multistorageclientis_multistorageclient_urlz)megatron.core.num_microbatches_calculatorget_current_global_batch_size)StragglerDetectionCallbackTF)FaultToleranceCallbackc@eZdZdZdS) NotFoundErrorz)Raised when a file or folder is not foundN__name__ __module__ __qualname____doc__rBrBS/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/utils/exp_manager.pyr<Or<cs eZdZdZfddZZS)LoggerMisconfigurationErrorzDRaised when a mismatch between trainer.logger and exp_manager occurscs|d}t|dS)Nzf You can disable lighning's trainer from creating a logger by passing logger=False to its constructor.super__init__)selfmessage __class__rBrCrHVs z$LoggerMisconfigurationError.__init__)r>r?r@rArH __classcell__rBrBrKrCrESsrEc@r;)CheckpointMisconfigurationErrorzGRaised when a mismatch between trainer.callbacks and exp_manager occursNr=rBrBrBrCrN^rDrNc@seZdZUdZdZeed<dZeed<dZe ed<dZ e ed <d Z e ed <d Ze ed <d Ze ed <dZee ed<dZee ed<dZee ed<dZe ed<dS)EarlyStoppingParamszEarlyStoppingParams PODval_lossmonitorminmodegMbP? min_delta patienceTverbosestrict check_finiteNstopping_thresholddivergence_thresholdcheck_on_train_epoch_endFlog_rank_zero_only)r>r?r@rArQstr__annotations__rSrTfloatrVintrWboolrXrYrZr r[r\r]rBrBrBrCrObs        rOc@s*eZdZUdZdZeed<dZeed<dS)IPLEpochStopperParamsu Parameters for the IPLEpochStopper callback used in iterative pseudo-label training. This is part of the TopIPL pipeline, a semi-supervised training method for ASR that uses iterative pseudo-labeling (IPL) — periodically stopping training to generate pseudo-labels for unlabeled data and fine-tuning the model on them. For more details, see: 🔗 Top-IPL: Top-N Pseudo-Label Averaging for Iterative ASR Training https://arxiv.org/abs/2506.07659 Attributes: enable_stop (bool): If True, enables the stopping behavior in the callback. stop_every_n_epochs (int): Specifies how many epochs to train before stopping. T enable_stopstop_every_n_epochsN) r>r?r@rArdrbr_rfrarBrBrBrCrcvs  rcc@sbeZdZUdZdZeeed<dZeeed<dZ eeed<dZ eeed<dZ ee ed <dZ ee ed <d Zeeed <d Zee ed<dZeeed<dZe ed<dZeeed<dZeeed<dZeeed<dZeeed<dZeed<d Ze ed<d Ze ed<dZee ed<dZeeed<d Zee ed<d Zee ed<dZeeed <dS)!CallbackParamszCallbackParams PODNfilepathdirpathfilenamerPrQTrW save_last save_top_kFsave_weights_onlyrRrSauto_insert_metric_namereevery_n_epochsevery_n_train_stepstrain_time_intervalprefixz.nemopostfixsave_best_modelalways_save_nemosave_nemo_on_train_endmodel_parallel_sizesave_on_train_epoch_end async_savesave_last_n_optim_states) r>r?r@rArhr r^r_rirjrQrWrbrkrmrarnrSrorprqrrr rsrtrurvrwrxryrzr|rBrBrBrCrgs0     rgc@sBeZdZUdZdZeeed<dZee ed<dZ ee ed<dS) StepTimingParamszStepTimingParams PODmean reductionF sync_cudare buffer_sizeN) r>r?r@rArr r^r_rrbrrarBrBrBrCr}s r}c@s^eZdZUdZdZeeed<dZee ed<dZ eeed<dZ eeed<dZ e ed <d S) EMAParamsz EMAParams PODFenableg+?decay cpu_offloadvalidate_original_weightsre every_n_stepsN)r>r?r@rArr rbr_rr`rrrrarBrBrBrCrs rc@sfeZdZUdZdZeed<dZeed<dZ eed<dZ e ed<d Z eed <d Z eed <d Zeed <dS)StragglerDetectionParamszStragglerDetectionParams PODi,report_time_intervalTcalc_relative_gpu_perfcalc_individual_gpu_perfnum_gpu_perf_scores_to_loggffffff?gpu_relative_perf_thresholdgpu_individual_perf_thresholdFstop_if_detectedN)r>r?r@rArr`r_rrbrrrarrrrBrBrBrCrs       rc@seZdZUdZdZeed<dZeeed<dZ eeed<dZ e ed <dZ eed <e jd kr1ejnejZejed <d Zeed<dZeed<dZeed<dZeed<dZeeed<dS)FaultToleranceParamszFaultToleranceParams PODg@workload_check_intervalg @initial_rank_heartbeat_timeoutg@rank_heartbeat_timeoutTcalculate_timeouts safety_factorntrank_termination_signalINFO log_levelrmax_rank_restartsmax_subsequent_job_failuresadditional_ft_launcher_argsNsimulated_fault)r>r?r@rArr`r_rr rrrbrosnamesignalSIGKILLSIGTERMrSignalsrr^rrarrrr rBrBrBrCrs         rc@s6eZdZUdZdZeeed<dZeeed<dZ eeed<dZ eeed<dZ ee ed<d Z ee ed <d Zee ed <d Zee ed <dZeeed <dZee ed<dZeeeefed<d Zee ed<dZeeeefed<d Zee ed<edddZeeed<d Zee ed<edddZeeed<d Zee ed<edddZee ed<d Z!ee ed<dZ"eeeefed<dZ#ee ed<ed ddZ$ee%ed!<d Z&ee ed"<d Z'ee ed#<ed$ddZ(ee)ed%<ed&ddZ*ee+ed'<dZ,ee ed(<dZ-ee.eed)<dZ/ee ed*<d Z0ee ed+<ed,ddZ1ee2ed-<d Z3ee ed.<d Z4ee ed/<dZ5ee ed0<ed1ddZ6ee7ed2<dZ8eeed3<d4Z9e:ed5<d Z;ee ed6<eeee ed8<ee?dZ@ee?ed9<dZAee ed:<dS);ExpManagerConfigz=Experiment Manager config for validation of passed arguments.Nexplicit_log_direxp_dirrversionTuse_datetime_versionFresume_if_existsresume_past_endresume_ignore_no_checkpointresume_from_checkpointcreate_tensorboard_loggersummary_writer_kwargscreate_wandb_loggerwandb_logger_kwargscreate_mlflow_loggercCtSN)r3rBrBrBrCzExpManagerConfig.)default_factorymlflow_logger_kwargscreate_dllogger_loggercCrr)r2rBrBrBrCrrdllogger_logger_kwargscreate_clearml_loggercCrr)r0rBrBrBrCrrclearml_logger_kwargscreate_neptune_loggerneptune_logger_kwargscreate_checkpoint_callbackcCrr)rgrBrBrBrCrrcheckpoint_callback_paramscreate_early_stopping_callback!create_ipl_epoch_stopper_callbackcCrr)rOrBrBrBrCrrearly_stopping_callback_paramscCrr)rcrBrBrBrCr r!ipl_epoch_stopper_callback_paramscreate_preemption_callback files_to_copylog_step_timinglog_delta_step_timingcCrr)r}rBrBrBrCrrstep_timing_kwargslog_local_rank_0_onlylog_global_rank_0_onlydisable_validation_on_resumecCrr)rrBrBrBrCrremamax_time_per_runrseconds_to_sleep#create_straggler_detection_callbackstraggler_detection_paramscreate_fault_tolerance_callbackfault_tolerancelog_tflops_per_sec_per_gpu)Br>r?r@rArr r^r_rrrrrbrrrrrrr r rrrrrr3rrr2rrr0rrrrrgrrrrOrrcrrr rrrr}rrrrrrrr`rrrrrrrrBrBrBrCrsb  rc@s|eZdZdZdifdefddZddZdd Zd d Zd d Z dddZ dddZ dddZ dddZ ddZddZdS)TimingCallbackz5 Logs execution time of train/val/test steps Flog_tokens_per_seccCs||_tjdi||_dS)zinit for TimitCallback Args: log_tokens_per_sec (bool, optional): _description_. Defaults to False. timer_kwargs (dict, optional): _description_. Defaults to {}. NrB)rr& NamedTimertimer)rIr timer_kwargsrBrBrCrH.szTimingCallback.__init__cCsR|jjdkr |j||j|r!td|d|j||j|dS)zPSetup the timer Args: name (_type_): name of timer rzTimer `zZ` was not correctly stopped, suggesting a possible issue. The timer will be reset for now.N)rrreset is_activer%warningstart)rIrrBrBrC_on_batch_start8s     zTimingCallback._on_batch_startcCs:|j||j|dt|j|ddd|dkddS)zend of the callback log Args: name (_type_): _description_ pl_module (_type_): _description_ z in sTFretrain_step_timingon_stepon_epoch batch_sizeprog_barN)rstoplogtorch as_tensor)rIr pl_modulerBrBrC _on_batch_endKs  zTimingCallback._on_batch_endcC|ddS)zwrapper Args: trainer (_type_): _description_ pl_module (_type_): _description_ batch (_type_): _description_ batch_idx (_type_): _description_ rNr)rItrainerrbatch batch_idxrBrBrCon_train_batch_start]s z#TimingCallback.on_train_batch_startcCsx|d||jr:d|vr|d|d<t|j|djdtj}|jd|t |j ddddddd Sd S) zwrapper Args: trainer (_type_): _description_ pl_module (_type_): _description_ outputs (_type_): _description_ batch (_type_): _description_ batch_idx (_type_): _description_ rtexttokensretokens_per_sec_per_gpuTFrN) rrr8accumulate_grad_batchesshaper distributedget_world_sizerrr)rIrroutputsrrtokens_per_gpurBrBrCon_train_batch_endhs&     z!TimingCallback.on_train_batch_endrcCr)on_validation_batch_startvalidation_step_timingNrrIrrrrdataloader_idxrBrBrCrz(TimingCallback.on_validation_batch_startcC|d|dS)on_validation_batch_endrNrrIrrrrrrrBrBrCrz&TimingCallback.on_validation_batch_endcCr)on_test_batch_starttest_step_timingNrrrBrBrCrrz"TimingCallback.on_test_batch_startcCr)on_test_batch_endrNrrrBrBrCrrz TimingCallback.on_test_batch_endcCr)on_before_backwardtrain_backward_timingNr)rIrrlossrBrBrCr rz!TimingCallback.on_before_backwardcCr)on_after_backwardr NrrIrrrBrBrCr rz TimingCallback.on_after_backwardNr)r>r?r@rArbrHrrrrrrrrr r rBrBrBrCr)s      rc@sLeZdZdZifddZddZddZdd Zd d Zd d Z ddZ dS)DeltaTimingCallbacka Logs execution time of train/val/test steps using nemo logger. Calculates time from previous batch end to current batch end. This ensures accuracy. Note: step time will only be printed in stdout. If you have initialized loggers like TensorBoard, WandB, etc, step time will not be recorded there. Use this callback instead of 'TimingCallback' to avoid logging overhead with lightning logger used in the latter. cCs|dd|_tt|_dS)zfinit Args: timer_kwargs (dict, optional): _description_. Defaults to {}. rFN)get _sync_cudarr&)rIrrBrBrCrHszDeltaTimingCallback.__init__cCs>|jr tjr tjd|j|d<t|j|d<dS)_on_epoch_startrsteprN)rrcudais_initialized synchronizer&time)rIrrrrBrBrCrs z#DeltaTimingCallback._on_epoch_startcCs|jr tjr tjt}||j|d}td|j|dd|d||j|dd7<||j|d<dS)rrzStep rz: z in s=reN) rrrrrrr&r%info)rIrrrenddtrBrBrCrs &z!DeltaTimingCallback._on_batch_endcC|d||dS)on_train_epoch_starttrain_step_timing in sNrr rBrBrCrz(DeltaTimingCallback.on_train_epoch_startcCr)on_validation_epoch_startvalidation_step_timing in sNrr rBrBrCr rz-DeltaTimingCallback.on_validation_epoch_startcCr)rrNrrIrrrrrrBrBrCrrz&DeltaTimingCallback.on_train_batch_endcCr)rr!Nrr"rBrBrCrrz+DeltaTimingCallback.on_validation_batch_endN) r>r?r@rArHrrrr rrrBrBrBrCrs     rrzlightning.pytorch.TrainercfgreturncCsttjdd}|j|j|}|t_|durtddS|j r)t ddSt t }tr;t dt |t|trFt |}nt|tsUtdt|dt t j|d d }t ||}t||t||j|j|j|j|j|jd \}}}}t|||j|j |j!|j"j#|j$|} | dus| d kr|jpd } |j%r|j&dds|j|j&_'t(d|j&j'||_||_t)} || _*|| _|| _|| _| | _+|j,| _,|j"| _"tj-|d dt d|||_.|j/d ur|j0d urtdt1t2d} |d|d|d} |j/d ur| s|dkrt3| n|j0d ur-| s-|dkr,t3| nt3| |j4sJ|j5sJ|j%sJ|j6sJ|j7sJ|j8rnt9||||j|j|j"|j4|j:|j5|j;|j%|j&|j6|j<|j7|j=|j8|j>|j?rt@|jApxid} |jBCd| n|jDrtE|jApid} |jBCd| |jFjGrtH|jFjI|jFjJ|jFjK|jFjLd}|jBM||jNrtOd+i|jP}|jBM||jQrtRd+i|jS}|jBM||j,rtT||| |j|j"|jU|jVrtW||jXdur(d}tY|jBD]\}}t|tZrt(dt[|jX|jB|<d }nq|s(|jX|_\|jBMt[|jX|j]rKt^rGt dt|j_}t`d+i|}|jBM|ntd|jarwtbrst d|jc}|jddk}tetf|jg||jh|jid}|jBM|ntd|jjrt dtr|jkr|jkD] }tltf||qtm|d d!d"d#}|nd$otpjqWdn 1swYtr\}}|rtm|d%d&d"d#}|nd'||ntsWdn 1swYtt|d(tu|d)|d(n|jv|jd*kr twx|jytz|S),a exp_manager is a helper function used to manage folders for experiments. It follows the pytorch lightning paradigm of exp_dir/model_or_experiment_name/version. If the lightning trainer has a logger, exp_manager will get exp_dir, name, and version from the logger. Otherwise it will use the exp_dir and name arguments to create the logging directory. exp_manager also allows for explicit folder creation via explicit_log_dir. The version can be a datetime string or an integer. Datestime version can be disabled if use_datetime_version is set to False. It optionally creates TensorBoardLogger, WandBLogger, DLLogger, MLFlowLogger, ClearMLLogger, ModelCheckpoint objects from pytorch lightning. It copies sys.argv, and git information if available to the logging directory. It creates a log file for each process to log their output into. exp_manager additionally has a resume feature (resume_if_exists) which can be used to continuing training from the constructed log_dir. When you need to continue the training repeatedly (like on a cluster which you need multiple consecutive jobs), you need to avoid creating the version folders. Therefore from v1.0.0, when resume_if_exists is set to True, creating the version folders is ignored. Args: trainer (lightning.pytorch.Trainer): The lightning trainer. cfg (DictConfig, dict): Can have the following keys: - explicit_log_dir (str, Path): Can be used to override exp_dir/name/version folder creation. Defaults to None, which will use exp_dir, name, and version to construct the logging directory. - exp_dir (str, Path): The base directory to create the logging directory. Defaults to None, which logs to ./nemo_experiments. - name (str): The name of the experiment. Defaults to None which turns into "default" via name = name or "default". - version (str): The version of the experiment. Defaults to None which uses either a datetime string or lightning's TensorboardLogger system of using version_{int}. - use_datetime_version (bool): Whether to use a datetime string for version. Defaults to True. - resume_if_exists (bool): Whether this experiment is resuming from a previous run. If True, it sets trainer._checkpoint_connector._ckpt_path so that the trainer should auto-resume. exp_manager will move files under log_dir to log_dir/run_{int}. Defaults to False. From v1.0.0, when resume_if_exists is True, we would not create version folders to make it easier to find the log folder for next runs. - resume_past_end (bool): exp_manager errors out if resume_if_exists is True and a checkpoint matching ``*end.ckpt`` indicating a previous training run fully completed. This behaviour can be disabled, in which case the ``*end.ckpt`` will be loaded by setting resume_past_end to True. Defaults to False. - resume_ignore_no_checkpoint (bool): exp_manager errors out if resume_if_exists is True and no checkpoint could be found. This behaviour can be disabled, in which case exp_manager will print a message and continue without restoring, by setting resume_ignore_no_checkpoint to True. Defaults to False. - resume_from_checkpoint (str): Can be used to specify a path to a specific checkpoint file to load from. This will override any checkpoint found when resume_if_exists is True. Defaults to None. - create_tensorboard_logger (bool): Whether to create a tensorboard logger and attach it to the pytorch lightning trainer. Defaults to True. - summary_writer_kwargs (dict): A dictionary of kwargs that can be passed to lightning's TensorboardLogger class. Note that log_dir is passed by exp_manager and cannot exist in this dict. Defaults to None. - create_wandb_logger (bool): Whether to create a Weights and Baises logger and attach it to the pytorch lightning trainer. Defaults to False. - wandb_logger_kwargs (dict): A dictionary of kwargs that can be passed to lightning's WandBLogger class. Note that name and project are required parameters if create_wandb_logger is True. Defaults to None. - create_mlflow_logger (bool): Whether to create an MLFlow logger and attach it to the pytorch lightning training. Defaults to False - mlflow_logger_kwargs (dict): optional parameters for the MLFlow logger - create_dllogger_logger (bool): Whether to create an DLLogger logger and attach it to the pytorch lightning training. Defaults to False - dllogger_logger_kwargs (dict): optional parameters for the DLLogger logger - create_clearml_logger (bool): Whether to create an ClearML logger and attach it to the pytorch lightning training. Defaults to False - clearml_logger_kwargs (dict): optional parameters for the ClearML logger - create_checkpoint_callback (bool): Whether to create a ModelCheckpoint callback and attach it to the pytorch lightning trainer. The ModelCheckpoint saves the top 3 models with the best "val_loss", the most recent checkpoint under ``*last.ckpt``, and the final checkpoint after training completes under ``*end.ckpt``. Defaults to True. - create_early_stopping_callback (bool): Flag to decide if early stopping should be used to stop training. Default is False. See EarlyStoppingParams dataclass above. - create_preemption_callback (bool): Flag to decide whether to enable preemption callback to save checkpoints and exit training immediately upon preemption. Default is True. - create_straggler_detection_callback (bool): Use straggler detection callback. Default is False. - create_fault_tolerance_callback (bool): Use fault tolerance callback. Default is False. - files_to_copy (list): A list of files to copy to the experiment logging directory. Defaults to None which copies no files. - log_local_rank_0_only (bool): Whether to only create log files for local rank 0. Defaults to False. Set this to True if you are using DDP with many GPUs and do not want many log files in your exp dir. - log_global_rank_0_only (bool): Whether to only create log files for global rank 0. Defaults to False. Set this to True if you are using DDP with many GPUs and do not want many log files in your exp dir. - max_time (str): The maximum wall clock time *per run*. This is intended to be used on clusters where you want a checkpoint to be saved after this specified time and be able to resume from that checkpoint. Defaults to None. - seconds_to_sleep (float): seconds to sleep non rank 0 processes for. Used to give enough time for rank 0 to initialize - train_time_interval (timedelta): pass an object of timedelta to save the model every timedelta. Defaults to None. (use _target_ with hydra to achieve this) returns: log_dir (Path): The final logging directory where logging files are saved. Usually the concatenation of exp_dir, name, and version. LOCAL_RANKrNz@exp_manager did not receive a cfg argument. It will be disabled.zXTrainer was called with fast_dev_run. exp_manager will return without any functionality.zExpManager schemazcfg was type: z(. Expected either a dict or a DictConfigT)resolve)rrrrrrrrdefaultexperiment_namezUmlflow logger specified but no experiment name set. Using the same as Tensorboard: %sexist_okzExperiments will be logged at zjCannot set both log_local_rank_0_only and log_global_rank_0_only to True.Please set either one or neither.Fznemo_log_globalrank-z _localrank-z.txt)r)rrrrzFound a PTL Timer callback, replacing with a StatelessTimer callback. This will happen if you set trainer.max_time as well as exp_manager.max_time_per_run.zEnabling straggler detection...z``create_straggler_detection_callback` is True, but there is no Straggler Det. package installed.zEnabling fault tolerance...)r autoresumersimulated_fault_paramszvFaultToleranceCallback was enabled with create_fault_tolerance_callback, but fault_tolerance package is not installed.zhTFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.z cmd-args.logwutf-8)encoding z git-info.logaz commit hash: znemo_error_log.txtzlightning_logs.txtrerB){rarenvironr node_rank num_devicesr%rankerror fast_dev_runrr structuredrr, isinstancedictcreater ValueErrortype to_containermerge error_checks get_log_dirrrrrrr check_resumerrrrirrrr(rr'log_dircheckpoint_namermakedirs_default_root_dirrrr*r#add_file_handlerrrrrrconfigure_loggersrrrrrrrr callbacksinsertrrrrr!rrrrappendrrrrr"rconfigure_checkpointingrr-configure_no_restart_validation_training_loopr enumeraterStatelessTimermax_timerHAVE_STRAGGLER_DETrr9rHAVE_FTrrr:rparentrrrrropenwritejoinsysargv get_git_hash get_git_diffadd_err_file_handlerr. num_nodesrsleeprr4)rr# local_rank global_rankschemarCrrrrD app_state nemo_testinglog_filetiming_callback ema_callbackearly_stop_callbackipl_epoch_stopper_callbackfound_ptl_timeridxcallbackstraggler_det_args_dictstraggler_det_callback ft_paramsft_use_autoresumefault_tol_callback_filegit_repogit_hashrBrBrC exp_managersl                                  rsc Cstrttkrtd|jdur2|js|js|j r2t d|jd|jd|j d|j d |j dkr@t |s@t d |jdkrRt|jtsTt d dSdSdS) a Checks that the passed trainer is compliant with NeMo and exp_manager's passed configuration. Checks that: - Throws error when hydra has changed the working directory. This causes issues with lightning's DDP - Throws error when trainer has loggers defined but create_tensorboard_logger or create_wandB_logger or create_mlflow_logger or create_dllogger_logger is True - Prints error messages when 1) run on multi-node and not Slurm, and 2) run on multi-gpu without DDP zHydra changed the working directory. This interferes with ExpManger's functionality. Please pass hydra.run.dir=. to your python script.NzwThe pytorch lightning trainer that was passed to exp_manager contained a logger, and either create_tensorboard_logger: z or create_wandb_logger: z or create_mlflow_logger: zor create_dllogger_logger: zS was set to True. These can only be used if trainer does not already have a logger.rezYou are running multi-node training without SLURM handling the processes. Please note that this is not tested in NeMo and could result in errors.zmYou are running multi-gpu without ddp.Please note that this is not tested in NeMo and could result in errors.)r initializedrrgetcwdr<loggerrrrrEr\ check_slurmr%r6r4r9strategyr)rr#rBrBrCr@Js>  r@checkpoint_pathscCs:g}|D]}t|rtd|dq||q|S)"_filter_out_unfinished_checkpointsz Checkpoint zH has the unfinished marker set - skipped while looking for the last one.)r(is_checkpoint_unfinishedr%rrK)ryres chkpt_pathrBrBrCrzqs   rzrCrrrrirc s|s td|dddlm}z|dur||rddlm}Wnty=} zdd| jd fWYd} ~ Sd} ~ wwd} |rD|} |r t sT||rTt |sd} ||r|} |j | d d } | r||j | ddd } d d| D}dd| Dng}gnt |rt }|} ||d} | rd nd} | rtdd| Dd d}tdd| Dd dn|g}gnw|rt|ntt|d} | } ddt| dD}dd|D}dd|D}|r|nt| d}t|}t|}t|}|dkr|dkrtd|r |nt| dt}tt}|dkr-|dkr-td| r>t|dksltdksl|rdd| d}| durQ|d7}n | |kr^|d |d!7}t|nbtd| d"t|dkr|rt|d#krd$t|dvr|d} n;td%|d&n2td'|dd(td#krtfd)dd*Drd} t| } n td%d+d} | durt| |_td,|jt r>g}t|rt|D]}|r|j !d-s|"|qt|dkr@t|d.}d}|D] }|#r|d#7}qtt|d/|}|$|D]}t%t|t|q1dSdSdS)0aQChecks that resume=True was used correctly with the arguments pass to exp_manager. Sets trainer._checkpoint_connector._ckpt_path as necessary. Returns: log_dir (Path): The log_dir exp_dir (str): The base exp_dir without name nor version name (str): The name of the experiment version (str): The version of the experiment Raises: NotFoundError: If resume is True, resume_ignore_no_checkpoint is False, and checkpoints could not be found. ValueError: If resume is True, and there were more than 1 checkpoint could found. zResuming requires the log_dir z to be passed to exp_managerr) is_s3_urlN)S3UtilsFzz check_resume..cSrz last.ckptrrrBrBrCrrz **/*.ckptcSrrrrrBrBrCrr)reversecSrrrrrBrBrCrr checkpointscSsg|]}|r|qSrB)is_dirrdrBrBrCrs*cSr)z*endmatchrrBrBrCrrcSr)z*lastrrrBrBrCrrz *end.ckptzEnd checkpoint is unfinished and cannot be used to resume the training. Please remove the checkpoint manually to avoid unexpected cosequences, such as restarting from scratch.z *last.ckptaLLast checkpoint is unfinished and cannot be used to resume the training. Please remove the checkpoint manually to avoid unexpected cosequences, such as restarting from scratch. Hint: Iteration number can be added to the checkpoint name pattern to maximize chance that there is at least one finished last checkpoint to resume from.z]There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :z. zTraining from scratch.zTraining from .z. Cannot resume.remp_rankzMultiple checkpoints z that matches *end.ckpt.zFound z= indicating that the last training run has already completed.cs g|] }|tdvr|qSr)r^)rslast_checkpointsrBrCrs )rtp_rank fsdp_shardz that matches *last.ckpt.z#Resuming training from checkpoint: zevents.out.tfeventszrun_*run_)&r<nemo.utils.s3_dirpath_utilsr~nemo.utils.s3_utilsr ImportErrorformatoutputdecoder,r7s3_path_existsfind_files_with_suffixr6globsortedrexistslistrgloblenrzr%rr<r^anyr5 ckpt_pathriterdiris_filer startswithrKrmkdirr)rrCrrrrirr~rerr checkpointcheckpoint_dir_existscheckpoint_dirall_keysend_checkpointsmscdist_checkpointsend_dist_checkpointslast_dist_checkpoints end_chkpt_cntfinished_end_chkpt_cntlast_chkpt_cntfinished_last_chkpt_cntwarn files_to_movechildother_run_dirs run_countfold new_run_dirrprBrrCrB~s     "            rBrrrrc Csx|jdur td|d|s|r td|d|d|dtr2t|r2td|d t|t|d d fS) aRChecks that the passed arguments are compatible with explicit_log_dir. Returns: log_dir (Path): the log_dir exp_dir (str): the base exp_dir without name nor version name (str): The name of the experiment version (str): The version of the experiment Raise: LoggerMisconfigurationError NzfThe pytorch lightning trainer that was passed to exp_manager contained a logger and explicit_log_dir: zN was pass to exp_manager. Please remove the logger from the lightning trainer.z'exp_manager received explicit_log_dir: z and at least one of exp_dir: z, or version: z>. Please note that exp_dir, name, and version will be ignored.zExp_manager is logging to z, but it already exists.r) rvrEr%r6r,rrrr^)rrrrrrBrBrCcheck_explicit_log_dir2s" rrc CsB|r t|||||S|}|durttd}|jdurE|jjr/|r+td|d|jj}|r9td|d|jj}d|jj}n>|pHd}|pQt j t d}|s|r^t d d}n%tr|ritd }ntt|||d }d|j}|dur~d n|t j t <t|tt|t|durd nt|} | t|||fS) a Obtains the log_dir used for exp_manager. Returns: log_dir (Path): the log_dir exp_dir (str): the base exp_dir without name nor version name (str): The name of the experiment version (str): The version of the experiment explicit_log_dir (str): The explicit path to the log folder. Defaults to False. use_datetime_version (bool): Uses date and time as the version of the log folder. Defaults to True. resume_if_exists (bool): if resume_if_exists of the exp_manager's config is enabled or not. When enabled, the version folders would not get created. Raise: LoggerMisconfigurationError: If trainer is incompatible with arguments NotFoundError: If resume is True, resume_ignore_no_checkpoint is False, and checkpoints could not be found. ValueError: If resume is True, and there were more than 1 checkpoint could found. Nnemo_experimentszThe pytorch lightning trainer that was passed to exp_manager contained a logger, the logger's save_dir was not None, and exp_dir (z) was not None. If trainer.logger.save_dir exists, exp_manager will use trainer.logger.save_dir as the logging directory and exp_dir must be None.z[The pytorch lightning trainer that was passed to exp_manager contained a logger, and name: z was also passed to exp_manager. If the trainer contains a logger, exp_manager will use trainer.logger.name, and name passed to exp_manager must be None.version_r'zZNo version folders would be created under the log folder as 'resume_if_exists' is enabled.z%Y-%m-%d_%H-%M-%Ssave_dirrrr)rr^rcwdrvrrErrrr2rr$r%rr,rstrftimer) rrrrrrr_exp_dirtensorboard_loggerrCrBrBrCrATsL    ,rAc CsXzdtjgdtjdfWStjtfy+}z dd|fWYd}~Sd}~ww)z Helper function that tries to get the commit hash if running inside a git folder returns: Bool: Whether the git subprocess ran without error str: git subprocess output or error message T)gitz rev-parseHEADstderrF{} N) subprocess check_outputSTDOUTrCalledProcessErrorFileNotFoundErrorrrrBrBrCrYsrYc CsTz tjddgtjdWStjy)}zd|jdWYd}~Sd}~ww)z Helper function that tries to get the git diff if running inside a git folder returns: Bool: Whether the git subprocess ran without error str: git subprocess output or error message rdiffrrr.N)rrrrrrrrrBrBrCrZs rZrrrr wandb_kwargsr mlflow_kwargsrdllogger_kwargsrclearml_kwargsrneptune_kwargscCsg}|r(|dur i}nd|vrtdtd|||d|}||td|rd| dur0i} d| vrr?r@rArrMrBrBrKrCr(srresumeparamsrrc Cs|jD] }t|trtdqd|vrL|jdur5td|jdur)t|jj |_|j dur5t|jj |_ t | |d=Wdn1sGwY|jdurXt|d|_|j durg|d|j d|_ |jduro||_|jrt}|jdur|jdks|jdur|jdks|jdur|jdkrtd |jd |jd |jd |j d t_t|jt|j t|jd|j vr|jdur|jdkr|j|jkrtd|jd|jd|j dn|jdur|jdkrtd|jd|jdtdd|i|}|jp d|_d|jvsd|jvrt|j|_|j||r@t j!"r9t#|} |j| dSt$ddSdS)zAdds ModelCheckpoint to trainer. Raises CheckpointMisconfigurationError if trainer already has a ModelCheckpoint callback zThe pytorch lightning trainer that was passed to exp_manager contained a ModelCheckpoint and create_checkpoint_callback was set to True. Please either set create_checkpoint_callback to False, or remove ModelCheckpoint from the lightning trainerrhNzEfilepath is deprecated. Please switch to dirpath and filename insteadrz--{z :.4f}-{epoch}rezkalways_save_nemo is set to True, please ensure that model parallel is not used.tensor_model_parallel_size: z,pipeline_model_parallel_size: z,context_parallel_size: ,z-lastvalr{zVThe checkpoint callback was told to monitor a validation value but trainer.max_epochs(z0) was less than trainer.check_val_every_n_epoch(zF). It is very likely this run will fail with ModelCheckpoint(monitor='ze') not found in the returned metrics. Please ensure that validation is run within trainer.max_epochs.zbThe checkpoint callback was told to monitor a validation value and trainer's max_steps was set to z5. Please ensure that max_steps will run for at least z8 epochs to ensure that checkpointing will not error out.n_resumerrrz:Preemption is supported only on GPUs, disabling preemptionrB)%rIr9rrNrhr%rrirrSrjrr rQrsrvr'tensor_model_parallel_sizepipeline_model_parallel_sizecontext_parallel_sizerEr(CHECKPOINT_NAME_LASTdebug max_epochscheck_val_every_n_epochr6 max_stepsrlast_model_pathr5rKrr is_availabler)r) rrCrrrrrjracheckpoint_callbackpreemption_callbackrBrBrCrL=s                     rLcCs z|jjWStyYdSw)rwF)accelerator_connectoris_slurm_managing_tasksAttributeError)rrBrBrCrws   rwc seZdZdZdejdfdedededdffdd Z de ee ffd d Z d e ee fddfd dZ dejjddffdd ZZS)rOz&Extension of PTL timers to be per run.NTdurationintervalrWr$cst|||dS)a stateless timer Args: duration (timedelta, optional): _description_. Defaults to None. interval (str, optional): _description_. Defaults to Interval.step. verbose (bool, optional): _description_. Defaults to True. NrF)rIrr rWrKrBrCrHs zStatelessTimer.__init__cCsiS) state_dictrB)rIrBrBrCr zStatelessTimer.state_dictr cCsdS)load_state_dictNrB)rIr rBrBrCr r zStatelessTimer.load_state_dictrcsHt||jr"|j}|r||}|||ddlm}|dS)_check_time_remainingr)_TunerExitExceptionN)rGr  should_stopr_monitor_candidates_save_last_checkpoint&lightning.pytorch.utilities.exceptionsr)rIrrmonitor_candidatesrrKrBrCr s    z$StatelessTimer._check_time_remaining)r>r?r@rArrrr^rbrHr r r r  lightningpytorchTrainerr rMrBrBrKrCrOs""rOcCs<t|jjtkrtdtdSt||j|j }||j_dS)rMzFDetected custom epoch loop. Skipping no validation on restart support.N) r=fit_loop epoch_looprwarningsr UserWarning SkipResumeTrainingValidationLoop min_stepsr)rlooprBrBrCrMs   rMcs&eZdZdZdeffdd ZZS)rz Extend the PTL Epoch loop to skip validating when resuming. This happens when resuming a checkpoint that has already run validation, but loading restores the training state before validation has run. r$cs|jrdSt|S)_should_check_val_fxF) restartingrGr)rI data_fetcherrKrBrCrs z5SkipResumeTrainingValidationLoop._should_check_val_fx)r>r?r@rArbrrMrBrBrKrCrsr exp_log_dir remove_ckpt remove_nemocCst|}|r(tdttj|dd}|D]}t|td|q|rLtdttj|dd}|D]}t|td|qs   $                   "   At(: u*' 5  $ U      ` a* $