o }oiD7@sddlZddlZddlZddlmZmZddlmZddlm Z m Z m Z ddl m ZddlmZddlmZddlmZmZmZddlmZddlmZddlmZdd lmZdd l m!Z!eGd d d eZ"dS) N) dataclassfield)Path)ListOptionalUnion)ModelCheckpoint)LoggerTensorBoardLogger WandbLogger)LocalCheckpointCallback)IOMixin)logging)AppStatec@s*eZdZUdZdZeed<dZeeed<dZ eeed<dZ eeed<dZ e ed <d Z e ed <d Ze ed <dZeeeed <dZe ed<dZeeed<dZeeed<dZeeed<eedZeeed<ddZd$deeje j!fde fddZ"ddZ#d%ddZ$ddZ%d d!Z&d"d#Z'dS)& NeMoLoggeraLogger for NeMo runs. Args: name (str): Name of the experiment. log_dir (Optional[str]): Directory to save logs. explicit_log_dir (Optional[str]): Explicit log directory. version (Optional[str]): Version of the experiment. use_datetime_version (bool): Whether to use datetime as version. log_local_rank_0_only (bool): Log only on local rank 0. log_global_rank_0_only (bool): Log only on global rank 0. files_to_copy (Optional[List[str]]): List of files to copy to log directory. update_logger_directory (bool): Whether to update logger directory to write to `exp_dir`. If True, the `save_dir` passed to the logger will be reconfigured to write to `exp_dir / save_dir`. This ensures that all output from an experiment is written to a common directory. If False, the logger's save_dir will not be overwritten. This argument applies only to TensorBoardLogger and WandbLogger instances. ckpt (Optional[ModelCheckpoint]): Model checkpoint callback. tensorboard: (Optional[TensorBoardLogger]): A PyTorch Lightning TensorBoardLogger instance to add to the trainer. wandb (Optional[WandbLogger]): A PyTorch Lightning WandBLogger instance to add to the trainer. extra_loggers(Optional[List[Logger]]): Any additional loggers to add to the trainer. defaultnameNlog_direxplicit_log_dirversionTuse_datetime_versionFlog_local_rank_0_onlylog_global_rank_0_only files_to_copyupdate_logger_directoryckpt tensorboardwandb)default_factory extra_loggerscCs$|jdur|jdurtddSdS)NTzkCannot set both log_local_rank_0_only and log_global_rank_0_only to True. Please set either one or neither.)rr ValueError)selfr"N/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/nemo_logger.py __post_init__Is zNeMoLogger.__post_init__trainerresume_if_existsc Cs(ddlm}ddlm}|j|_|j|_|jt_|jrjt |t j rj|j dur2|j s2td|j|js8|jrEtd|j|j|j|rXt|jrXtd|jt|jt|jddf\}}|_}nX|j}|jdurzttd }|jsd |_|jptj|d}|s|rtd d}n |r|jrtd }|r|r|tj|<t|tt|jt|durdnt|}t} || _|| _|j| _|| _t j!| _"tj#|d dt$d||r|r|%||t |t j r|&||||j'|||j(d|)|| |*|| S)aISetup the logger for the experiment. Args: trainer (Union[pl.Trainer, fl.Fabric]): Trainer or Fabric instance. resume_if_exists (bool): Whether to resume if log directory exists. Returns: AppState: The application state with updated log directory and other settings. r)NEMO_ENV_VARNAME_VERSION)is_global_rank_zeroNznemo logger received explicit_log_dir: {} and the pytorch lightning trainer that was passed to nemo_logger container a logger, but update_logger_directory is False. This means that the trainer's logger directory may not match with the explicit_log_dir.znemo logger received explicit_log_dir: {} and at least one of dir: {}or version: {}. Please note that dir, name, and version will be ignored.z3NeMoLogger is logging to {}, but it already exists.nemo_experimentsrzZNo version folders would be created under the log folder as 'resume_if_exists' is enabled.z%Y-%m-%d_%H-%M-%ST)exist_okz Experiments will be logged at {})rr)+nemo.constantsr'nemo.utils.get_rankr( local_rank global_rankrrankr isinstanceplTrainerloggerrwarningformatrrerrorrexistsstrrcwdosenvirongetrtimestrftimerexp_dirsysargvcmd_argsmakedirsinfo_handle_task_config_setup_trainer_loggers_setup_trainer_model_checkpointr_setup_files_to_move_setup_file_logging) r!r%r& task_configr'r(r_dirr app_stater"r"r#setupPsx   $   .    zNeMoLogger.setupcCs|j|jg|j}dd|D}|r&|jdur |js |jg|}|j||jrt|jD]I}t|t rN|p6d|_ t |t j |j|_td|jq,t|trs|pVd|_t ||j|_t ||j|jd<td|jq,dSdS)NcSsg|]}|dur|qSNr").0r4r"r"r# sz5NeMoLogger._setup_trainer_loggers..r)zR"update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to {}dirzL"update_logger_directory" is True. Overwriting wandb logger "save_dir" to {})rrrr4_logger_connectorconfigure_loggerrloggersr1r _versionrr;pathrelpathsave_dir _root_dirrr5r6r _id _save_dir _wandb_init)r!r%rRrrUr4r"r"r#rGs6       z!NeMoLogger._setup_trainer_loggerscCsp|rld}t|jD]\}}t|tr t|ts td|}nq |dur+||j|<n|j||jrld|jvrl|j durW|j dkrW|j |j krWt d |j |j |jn|j durl|j dkrltd |j |j ddlm}|jD]@}t|trt|ts|jdurt|d|_|jdurt|j|r|jd |jd |_n |jd |jd |_|jd t_qudS) NzRThe Trainer already contains a ModelCheckpoint callback. This will be overwritten.vala6The checkpoint callback was told to monitor a validation value but trainer.max_epochs({}) was less than trainer.check_val_every_n_epoch({}).It is very likely this run will fail with ModelCheckpoint(monitor='{}') not found in the returned metrics. Please ensure that validation is run within trainer.max_epochs.zThe checkpoint callback was told to monitor a validation value and trainer's max_steps was set to {}. Please ensure that max_steps will run for at least {} epochs to ensure that checkpointing will not error out.r)MegatronStrategy checkpointsz--{z :.4f}-{epoch}-{consumed_samples}z:.4f}-{epoch}-{step}z-last) enumerate callbacksr1PTLModelCheckpointr rr5appendmonitor max_epochscheck_val_every_n_epochr7r6 max_stepsnemo.lightningr`dirpathrfilenamestrategyrrCHECKPOINT_NAME_LAST)r!r%rr _overwrite_iicallbackr`r"r"r#rHsZ           z*NeMoLogger._setup_trainer_model_checkpointc Csz3ddlm}||d||}t|dd}||WdWdS1s,wYWdStyN}ztd |WYd}~dSd}~ww)Nr) serializationztask.pngz task.jsonwz.Saving task config failed: {}. Skipping saving) fiddle._src.experimentalrrsave_config_img dump_jsonopenwrite Exceptionrr5r6)r!rKrrr task_jsonfer"r"r#rFs   &zNeMoLogger._handle_task_configcCsddlm}ddlm}ddlm}||d}|d|jd|jd}|jr4|s4|jdkr4t |n|j rD|sD|jdkrDt |n |jsO|j sOt ||d S) z+Set up file logging based on rank settings.r)NEMO_ENV_VARNAME_TESTING) get_envbool)add_handlers_to_mcore_loggerFznemo_log_globalrank-z _localrank-z.txtN) r,r}nemo.utils.env_var_parsingr~nemo.utils.mcore_loggerrr/r.rradd_file_handlerr)r!rr}r~r nemo_testinglog_filer"r"r#rJs         zNeMoLogger._setup_file_loggingcCsFg}t|rt|D] }|r||q||_|j|_dSrO)rr8iterdiris_filere files_to_mover)r!rrMrchildr"r"r#rI$s   zNeMoLogger._setup_files_to_move)FNrO)(__name__ __module__ __qualname____doc__rr9__annotations__rrrrrboolrrrrrrrrr rr rlistrr r$rr2r3flFabricrNrGrHrFrJrIr"r"r"r#r!s,       ^ < r)#r;rAr> dataclassesrrpathlibrtypingrrrlightning.fabricfabricrlightning.pytorchpytorchr2,lightning.pytorch.callbacks.model_checkpointrrdlightning.pytorch.loggersr r r >nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callbackr nemo.lightning.io.mixinr nemo.lightning.pytorch.callbacks nemo.utilsrnemo.utils.app_staterrr"r"r"r#s"