o Xεi_@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z m Z ddlmZddlmZmZmZmZmZddlZddlmZddlmZddlmZddlmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8ddl9m:Z:m;Z;mZ>ddl?m@Z@mAZAddlBmCZCmDZDmEZEmFZFeGdZHe<rddlImJZJe GdddeZKe GdddeZLGdddZMdS)N) nullcontext) dataclassfield) signature)CallableDictListTupleUnion)Coqpit)nn)DistributedDataParallel) DataLoader)ping_training_run)TrainerCallback) KeepAveragecount_parametersget_experiment_folder_pathget_git_branch isimplementedremove_experiment_folderset_partial_state_dictto_cuda)copy_model_filesget_last_checkpoint load_fsspecsave_best_modelsave_checkpoint) ConsoleLogger DummyLoggerlogger_factory) get_optimizer get_scheduleris_apex_availableprint_training_envsetup_torch_training_env) cuda_meminfoshould_reduce_batch_size)get_rankinit_distributedrank_zero_logger_inforank_zero_onlytrainer)ampc@s,eZdZUdZeddZeed<edddidZeed <ed dd idZ eed <eddd idZ eed<edddidZ eed<edddidZ e ed<edddidZe ed<edddidZeed<edddidZeed<edddidZeed<ed dd!idZeed"<eddd#idZe ed$<ed%dd&idZe ed'<ed(dd)idZe ed*<ed dd+idZeed,<eddd-idZeed.<ed/dd0idZe ed1<eddd2idZeed3<eddd4idZeed5<ed/dd6idZe ed7<ed dd8idZeed9<eddd:idZe ed;<ed<ed?dd@idZ eedA<edddBidZ!eedC<edDddEidZ"eedF<edGddHidZ#e edI<edJddKidZ$e edL<edMddNidZ%e edO<edPddQidZ&e'edR<ed ddSidZ(eedT<edUddVidZ)e*e'e+e'fedW<edddXidZ,e*ee+efedY<ee-ddZid[Z.e*e/e+e/fed\<eddd]idZ0e*ee+efed^<ee-dd_id[Z1e/ed`<edddaidZ2eedb<edddcidZ3eedd<ed ddeidZ4eedf<edddgidZ5eedh<edddiidZ6eedj<edkddlidZ7e edm<dS)n TrainerConfiguConfig fields tweaking the Trainer for a model. A ````ModelConfig```, by inheriting ```TrainerConfig``` must be defined for using 👟. Inherit this by a new model config and override the fields as needed. All the fields can be overridden from comman-line as ```--coqpit.arg_name=value```. Example:: Run the training code by overriding the ```lr``` and ```plot_step``` fields. >>> python train.py --coqpit.plot_step=22 --coqpit.lr=0.001 Defining a model using ```TrainerConfig```. >>> from trainer import TrainerConfig >>> class MyModelConfig(TrainerConfig): ... optimizer: str = "Adam" ... lr: float = 0.001 ... epochs: int = 1 ... ... >>> class MyModel(nn.module): ... def __init__(self, config): ... ... >>> model = MyModel(MyModelConfig()) output)default output_pathNhelpzqURI to save training artifacts by the logger. If not set, logs will be saved in the output_path. Defaults to Noner0metadata logger_urirunz"Name of the run. Defaults to 'run'run_namez%Name of the project. Defaults to None project_nameu🐸Coqui trainer run.uINotes and description about the run. Defaults to '🐸Coqui trainer run.'run_descriptionzKPrint training stats on the terminal every print_step steps. Defaults to 25 print_stepdzHPlot training stats on the logger every plot_step steps. Defaults to 100 plot_stepFzELog model parameters stats on the logger dashboard. Defaults to Falsemodel_param_statsz-Wandb entity to log the run. Defaults to None wandb_entity tensorboardzCLogger to use for the tracking dashboard. Defaults to 'tensorboard'dashboard_loggerTz7Save checkpoint on interrupt (Ctrl+C). Defaults to Truesave_on_interruptzgSave checkpoint to the logger every log_model_step steps. If not defined `save_step == log_model_step`.log_model_stepi'z>Save local checkpoint every save_step steps. Defaults to 10000 save_stepz'Keep n local checkpoints. Defaults to 5save_n_checkpointsz*Save checkpoints locally. Defaults to Truesave_checkpointszDSave all best checkpoints and keep the older ones. Defaults to False save_all_bestrz4Wait N steps to save best checkpoints. Defaults to 0save_best_afterz;Target loss name to select the best model. Defaults to None target_lossz3Print eval steps on the terminal. Defaults to False print_evalz4Wait N epochs before running the test. Defaults to 0test_delay_epochszboolr?rArBrCrDrFrGrHrIrJrKrLrMrNrPrQrRrTrUrWrYr[floatr\r]r rr^dictr`rrarbrcrdrerfrgrhrsrsC/home/ubuntu/.local/lib/python3.10/site-packages/trainer/trainer.pyr.@s $r.c@sJeZdZUdZedddidZeed<edddidZeed<eddd idZ eed <ed dd idZ e ed <ed ddidZ e ed<edddidZ eed<ed ddidZe ed<ed ddidZe ed<ed ddidZe ed<edddidZeed<edddidZeed<edddidZeed <eddd!idZeed"<dS)# TrainerArgszTrainer arguments that can be accessed from the command line. Examples:: >>> python train.py --restore_path /path/to/checkpoint.pth r2zPath to a training folder to continue training. Restore the model from the last checkpoint and continue training under the same folder.r3 continue_pathz`Path to a model checkpoit. Restore the model with the given checkpoint and start a new training. restore_pathzyBest model file to be used for extracting the best loss. If not specified, the latest best model in continue path is used best_pathFzVUse DDP in distributed training. It is to set in `distribute.py`. Do not set manually.use_ddpz/Use HF Accelerate as the back end for training.use_acceleratez`Number of gradient accumulation steps. It is used to accumulate gradients over multiple batches.grad_accum_stepsz%Overfit a single batch for debugging. overfit_batchz/Skip training and only run evaluation and test.skip_train_epochzStart with evaluation and test.start_with_evalNzdOnly use a subset of the samples for debugging. Set the number of samples to use. Defaults to None. small_runzIGPU ID to use if ```CUDA_VISIBLE_DEVICES``` is not set. Defaults to None.gpurz;Process rank in a distributed training. Don't set manually.rankz?Process group id in a distributed training. Don't set manually.group_id)rirjrkrlrrwrmrnrxryrzrpr{r}ror~rrrrrrrsrsrsrtrus` ruc%@seZdZddddddddddididfdededededdd ejd e d e d e d e de de de de de de ee fdeddf$ddZeddZeddZeddZddZd d!Zed"d#Zd$d%Zedeee ffd&d'Zeddd(defd)d*Zdd+efd,d-Zedded.e defd/d0Zed1d2Zeded e dejfd3d4Zeded e dejfd5d6Z dded7ed ejd8e!j"j#d9e!j$j%j&de'eje!j"j#e!j$j%j&eff d:d;Z(de d?ed@e dAe dBede fdCdDZ*de d@e dAe de fdEdFZ+de d@e dAe de fdGdHZ,de d@e dAe de fdIdJZ-dKe de fdLdMZ.ed8e!j"j#fdNdOZ/e ddKe d ejdPejdQede'e e ff dRdSZ0dTe dUefdVdWZ1 ddXe dYe dQedZe2fd[d\Z3dKe d ejdPejdedQef d]d^Z4ededQefd_d`Z5d8e!j"j#fdadbZ6dce2d8e!j"j#d9ddfdedfZ7   gddKe d ejd8e!j"j#d9dddPejdhee!j"j8j9e e fdedQedYe diede'e e effdjdkZ:dKe dledmedne2de'e e ff dodpZ;ddqdrZ< ddKe d ejdPejdQede'e e ff dsdtZ=dKe dmede'e e ffdudvZ>ddwdxZ?ddydzZ@d{d|ZAddd}d~ZBdddZCddddZDdddZEdddZFeGdddZHeGdddZIeGdddZJed ejdedee!j"j#e ffddZKed ejdedee2e e2ffddZLed ejded8ee!j"j#e e fdee!j"j8j9e ffddZMedhede e fdededededede ff ddZNed ejdejfddZOedXe de fddZPdeQde fddZRdeddfddZSede fddZTdS)TrainerNTargsconfigr1c_loggerrALoggermodel get_modelget_data_samples train_samples eval_samples test_samples train_loader eval_loadertraining_assetsparse_command_line_args callbacksrreturncCs|r||\}}||||\}}n|js|jr#||i|\}}ni}|jr,|j}n|dur3|jn|}t|j|j}tj|ddt |||||_ ||_ ||_||_ |j |_ |j|_|j|_|j|_|j dksnJdtj|jd|jd}|||j|||d\|_|_||j |||\|_|_|j js|j j|j _|j js|jrd |_d|_d|_d|_ d|_!t"d |j jrt"d ndd |_#d|_$d|_%d|_&d|_'d|_(|j j)r|j j*d kr|jn|j j+|_,| dur| |_-| |_.| |_/n|dur |0||\|_-|_.|_/n d|_-d|_.d|_/| |_$| |_&|1|j2|dur-|dur-t3d |dur6||_4n|5|j |t6|j4drI|j47|8|j4|_9|j:rht;dt<t=|j|j|j>|j j?|j j@|jr|j4AtB|j9tCr|j9D]}tB|tDjEjFr|Aq{ntB|j9tDjEjFr|j9A|G|j4|j |_H|j dkrtB|jHtCrt6|j4dst3dtI|_J|jJK||jJL||j,r|jMrd|_NtOjP|j4|jHdd\|_4|_HtDjAjOQ|_Nnd|_N|j jr|R|j |j|j4|jH|jN\|_4|_H|_N|_ |_!tDjAjOQ|_N|S|j4|j |jH|_T|U|jT|j |j |j!|j |_T|j:r=tV|j4|jg|jd|_4|WtX|j4}t;d|dt<|jJY||jZ||[t\dS)uiSimple yet powerful 🐸💬 TTS trainer for PyTorch. It can train all the available `tts` and `vocoder` models or easily be customized. Notes: Supports Automatic Mixed Precision training. If `Apex` is availabe, it automatically picks that, else it uses PyTorch's native `amp` module. `Apex` may provide more stable training in some cases. Args: args (Union[Coqpit, Namespace]): Training arguments parsed either from console by `argparse` or `TrainerArgs` config object. config (Coqpit): Model config object. It includes all the values necessary for initializing, training, evaluating and testing the model. output_path (str): Path to the output training folder. All the files are saved under thi path. c_logger (ConsoleLogger, optional): Console logger for printing training status. If not provided, the default console logger is used. Defaults to None. dashboard_logger Union[TensorboardLogger, WandbLogger]: Dashboard logger. If not provided, the tensorboard logger is used. Defaults to None. model (nn.Module, optional): Initialized and ready-to-train model. If it is not defined, `Trainer` initializes a model from the provided config. Defaults to None. get_model (Callable): A function that returns a model. It is used to initialize the model when `model` is not provided. It either takes the config as the only argument or does not take any argument. Defaults to None get_data_samples (Callable): A function that returns a list of training and evaluation samples. Used if `train_samples` and `eval_samples` are None. Defaults to None. train_samples (List): A list of training samples used by the model's `get_train_data_loader` to init the `dataset` and the `data_loader`. Defaults to None. eval_samples (List): A list of evaluation samples used by the model's `get_eval_data_loader` to init the `dataset` and the `data_loader`. Defaults to None. train_loader (DataLoader): A pytorch data loader object for training epochs. Leave as None if you want it to be made during training. Defaults to None. eval_loader (DataLoader): A pytorch data loader object for evaluation epochs. Leave as None to be generated during training. Defaults to None. test_samples (List): A list of test samples used by the model's `get_test_data_loader` to init the `dataset` and the `data_loader`. If None, the ```model.test_run()``` is expected to load the data. Defaults to None. training_assets (Dict): A dictionary of assets to be used at training and passed to the model's ```train_log(), eval_log(), get_data_loader()``` during training. It can include `AudioProcessor` or/and `Tokenizer`. Defaults to {}. parse_command_line_args (bool): If true, parse command-line arguments and update `TrainerArgs` and model `config` values. Set it to false if you parse the arguments yourself. Defaults to True. callbacks (Dict[str, Callable]): A dictionary of callbacks to be used during training. The keys are the callback names and the values gpu (int): GPU ID to use for training If "CUDA_VISIBLE_DEVICES" is not set. Defaults to None. Example:: Running trainer with a model. >>> args = TrainerArgs(...) >>> config = ModelConfig(...) >>> model = Model(config) >>> trainer = Trainer(args, config, output_path, model=model) >>> trainer.fit() TODO: - Wrap model for not calling .module in DDP. - Deepspeed integration - Profiler integration. - Overfitting to a batch. - TPU training NT)exist_okrz- [!] grad_accum_steps must be greater than 0.trainer_z_log.txt)rrrFinf train_loss eval_lossrSz0[!] `model` and `get_model` cannot both be None.init_for_trainingz > Using PyTorch DDPr|optimizeu [!] Coqui Trainer does not support grad_accum_steps for multiple-optimizer setup, please set grad_accum_steps to 1 or implement in your model a custom method called ´optimize` that need to deal with dangling gradients in multiple-optimizer setup!O1) opt_level) device_ids output_devicez > Model has z parameters)] parse_argv init_trainingrwrxr1rr7osmakedirsrrrrr}r~rrpathjoinr_setup_logger_configsetup_training_environmentuse_cudanum_gpus init_loggersrArrCrDrMtotal_steps_done epochs_done restore_step restore_epochrq best_lossr test_loaderrkeep_avg_train keep_avg_evalrRrTrcuse_amp_scalerrrrrun_get_data_samplessetup_small_runr ValueErrorr run_get_modelrr get_criterion criterion use_pt_ddpr*loggerr)rrPrQcuda isinstancelisttorchr Moduler!r^rrparse_callbacks_dict on_init_startuse_apexscalerr- initialize GradScaler restore_modelr" schedulerrestore_schedulerDDP_thsetup_accelerater on_init_end add_configsave_training_scriptr)selfrrr1rrArrrrrrrrrrrrcoqpit_overrides new_fieldslog_filer num_paramsrsrsrt__init__"si                         zTrainer.__init__cCs|jj o|S)zReturn True if using APEX.)rr{_is_apex_availablerrsrsrtrIzTrainer.use_apexcCs|jdko|j S)z!Return True if using PyTorch DDP.r|)rr{rrsrsrtrNrzTrainer.use_pt_ddpcCs|jjS)z#Return True if using HF Accelerate.)rr{rrsrsrtr{SszTrainer.use_acceleratec CsL|jr$|j|j|j|j|j|j|jj|jj d\|_|_|_|_|_ dSdS)N)rr^training_dataloaderrr}rRrT) r{init_acceleraterr^rrr}rrRrT acceleratorrrsrsrtrXszTrainer.setup_acceleratecCs|jr |j|S|S)z)Prepare the accelerator for the training.)r{rprepare_data_loader)r data_loaderrsrsrtprepare_accelerate_loaderds z!Trainer.prepare_accelerate_loaderc Cszddlm}Wnty}ztd|d}~ww|dur |n|r$dnd} | dkr-d} n | dkr4d} n| d kr:d } ||| d } t|tjjrL| |}t|trb| D] \} } | | || <qUnt|t rxt |D] \} } | | || <qkn |dur| |}t|tj jjr| |}t|tr| D] \} }| ||| <qnt|t rt |D] \} }| ||| <qn |dur| |}||||| fS) z%Setup HF Accelerate for the training.r) Acceleratorz.Please install accelerate to use this feature.Nf16float16float8f8bfloat16bf16)gradient_accumulation_stepsrR) accelerater ImportErrorrrr r prepare_modelrritemsprepare_optimizerr enumerateutilsdatarrprepare_scheduler)rr^rrr}rRrTre _precisionrkeyoptimischedrsrsrtrjsN          zTrainer.init_acceleratecCstjd}tj|rJtj|}|jj||ddt|ddd}|j d| dWdn1s7wYt |tj |j|dSdS) z?Save the training script to tracking dashboard and output path.rfile) file_or_dirname artifact_typerutf8)encodingztraining-scriptN)sysargvrrisfilebasenamerA add_artifactopenadd_textreadshutilcopyfilerr1)r file_path file_namefrsrsrtrs   zTrainer.save_training_scriptcCsHt|tr |jdd}n t}|jdd}|\}}||||fS)zAParse command line arguments to init or override `TrainerArgs()`.rv) arg_prefix)rr init_argparseruparse_known_args parse_args)rparser train_config training_argsrrsrsrtrs    zTrainer.parse_argvr cCs@|durtn|}tdkrt|fS|durt||}||fS)aInit console and dashboard loggers. Use the given logger if passed externally else use config values to pick the right logger. Return a dashboard logger only for the rank 0 process in DDP Define a console logger for each process in DDP Args: config (Coqpit): Model config. output_path (str): Output path to save the training artifacts. dashboard_logger (DashboardLogger): Object passed to the trainer from outside. c_logger (ConsoleLogger): Object passed to the trained from outside. Returns: Initialized dashboard_logger and console_logger objects. Nr)rr(rr )rr1rArrsrsrtrs    zTrainer.init_loggersrcCsv|dur9td||jdurdn|jd||_|jdur dn|jd||_|jdur/dn|jd||_dSdS)z=Use a subset of samples for training, evaluation and testing.Nz%[!] Small Run, only using %i samples.)rinforrr)rrrsrsrtrs  "zTrainer.setup_small_runrcCs|jr,tj|jd|_t|j\|_}|js||_|r#||jn t }||jt |dkr9|j |ddi}|j dkrM|jrH|j|d<t |d<||fS)aInitialize training and update model configs from command line arguments. Args: args (argparse.Namespace or dict like): Parsed trainer arguments. config_overrides (argparse.Namespace or dict like): Parsed config overriding arguments. config (Coqpit): Model config. If none, it is generated from `args`. Defaults to None. Returns: config (Coqpit): Config paramaters. z config.jsonrT)relaxed_parserrx github_branch)rwrrr config_pathrrxry load_jsonr lenr rr)rrr best_modelcoqpitrrsrsrtrs"      zTrainer.init_trainingc Cstdkrddl}||j}||jd|dft||j|j|j |j |j |j |j dur0|n|j d\}}t||||fS)NWindowsrir|)rrerfrgrzrhrdr)platformsystemresource getrlimit RLIMIT_NOFILE setrlimitr%rerfrgrzrhrdrr$)rrrrrlimitrrrsrsrtrs   z"Trainer.setup_training_environmentcCs*tt|jjdkr||}|S|}|S)zRun the `get_model` function and return the model. Args: config (Coqpit): Model config. Returns: nn.Module: initialized model. r|)rrsig parameters)rrrrsrsrtr s  zTrainer.run_get_modelcCsFt|r!tt|jjdkr||\}}||fS|\}}||fSdS)Nr|NN)callablerrr"r#)rrrrrsrsrtrs  zTrainer.run_get_data_samplesrxr^rc CsHdd}tdtj|t|dd}zEtd||dtdz ||d |}Wnttt fy@td Ynwd |vrX|j rX|d rXtd ||d |}Wn#tt t fy|td | }t ||d|}||~Ynw|||j||}td|d|dd} |d} tj|||| | fS)a3Restore training from an old run. It restores model, optimizer, AMP scaler and training stats. Args: config (Coqpit): Model config. restore_path (str): Path to the restored training run. model (nn.Module): Model to restored. optimizer (torch.optim.Optimizer): Optimizer to restore. scaler (torch.cuda.amp.GradScaler, optional): AMP scaler to restore. Defaults to None. Returns: Tuple[nn.Module, torch.optim.Optimizer, torch.cuda.amp.GradScaler, int]: [description] cSsjt|trt|D] \}}|||q |St|tr.|D] \}}|||q |S|||SN)rrrload_state_dictrrr)statesobjidxstaterrsrsrt_restore_list_objs:s   z1Trainer.restore_model.._restore_list_objsz > Restoring from %s ...cpu map_locationz > Restoring Model...rz > Restoring Optimizer...r^z7 > Optimizer is not compatible with the restored model.rz > Restoring Scaler...z" > Partial model initialization...z > Model restored from step %istepr|epoch)rrrrrrr'KeyError TypeError RuntimeErrorrr state_dictr restore_lrrrr empty_cache) rrrxrr^rr, checkpoint model_dictrrrsrsrtr%s:        zTrainer.restore_modelc Cs|jsQt|tr#t|D]\}}|jD] }|||||d<qq |St|trC|D]\}}|jD] }|||||d<q3q,|S|jD] }||||d<qF|S)Nr])rwrrr param_groupsget_lrrrr) rrrrr^r*rgroup optim_namersrsrtr6bs      zTrainer.restore_lrassetsis_evalsamplesverboserc Csj|dkrt|jdr|j|||||||jj}nt|dr)|j||||||d}t|dks3Jd|S)Nr|get_data_loader)rr>r?r@rArruY ❗ len(DataLoader) returns 0. Make sure your dataset is not empty or len(dataset) > 0. )rmodulerBrrr) rrrr>r?r@rArloaderrsrsrt _get_loadervs(   zTrainer._get_loaderc C|jdkr t|jjdr|jj|j|j|||j|jj}|Snt|jdr5|j|j|j|||j}|S| |j|j|d|||jS)aInitialize and return a training data loader. Call ```model.get_train_data_loader``` if it is implemented, else call ```model.get_data_loader``` and set ```is_eval=False```. Args: ap (AudioProcessor): Audio processor. samples (List): Data samples used for training. verbose (bool): enable/disable printing loader stats at initialization. Returns: DataLoader: Initialized training data loader. r|get_train_data_loaderF) rrrrCrGrrrrrErrr@rArDrsrsrtget_train_dataloader4  zTrainer.get_train_dataloaderc CrF)aInitialize and return a evaluation data loader. Call ```model.get_eval_data_loader``` if it is implemented, else call ```model.get_data_loader``` and set ```is_eval=True```. Args: ap (AudioProcessor): Audio processor. samples (List): Data samples used for training. verbose (bool): enable/disable printing loader stats at initialization. Returns: DataLoader: Initialized training data loader. r|get_eval_data_loaderT) rrrrCrKrrrrrErHrsrsrtget_eval_dataloaderrJzTrainer.get_eval_dataloaderc CrF)aInitialize and return a evaluation data loader. Call ```model.get_test_data_loader``` if it is implemented, else call ```model.get_data_loader``` and set ```is_eval=True```. Args: ap (AudioProcessor): Audio processor. samples (List): Data samples used for training. verbose (bool): enable/disable printing loader stats at initialization. Returns: DataLoader: Initialized training data loader. r|get_test_data_loaderT) rrrrCrMrrrrrErHrsrsrtget_test_dataloaderrJzTrainer.get_test_dataloaderbatchcCsz|jdkr|jj|}n|j|}Wn tyYnwt|tr4|D] \}}t|||<q(n t|t r@dd|D}z|jdkrP|jj |}W|S|j |}W|StybY|Sw)aAFormat the dataloader output and return a batch. 1. Call ```model.format_batch```. 2. Pass the batch to the Device. 3. Call ```model.format_batch_on_device```. Args: batch (List): Batch returned by the dataloader. Returns: Dict: Formatted batch. r|cSsg|]}t|qSrs)r).0vrsrsrt *sz(Trainer.format_batch..) rrrC format_batchNotImplementedErrorrrrrrrformat_batch_on_device)rrOkrQrsrsrtrSs0       zTrainer.format_batchccs&|jD] }|dD]}|Vq qdS)zGenerator over parameters owned by the optimizer. Used to select parameters used by the optimizer for gradient clipping. Args: optimizer: Target optimizer. paramsN)r:)r^r<prsrsrt master_params9s  zTrainer.master_paramsr optimizer_idxcCs:||g}|dur ||t|dr|jj|S|j|S)as Perform a trainig forward step. Compute model outputs and losses. Args: batch (Dict): [description] model (nn.Module): [description] criterion (nn.Module): [description] optimizer_idx (int, optional): [description]. Defaults to None. Returns: Tuple[Dict, Dict]: [description] NrC)appendhasattrrC train_step)rOrrrZ input_argsrsrsrt_model_train_stepFs     zTrainer._model_train_steprRrTcCstd}t}|jr1d}tj}|r-|dkrtj}||fS|dkr&tj}||fStd|||fS|r6tj}||fS)Nr-rrSru ❗ Unknown precision )rget_autocast_cpu_dtyperfloat32rrr)rrRrTdevicedtypersrsrt_get_autocast_args^s"zTrainer._get_autocast_args loss_dictstep_optimizer grad_normcCs\||}|dur"|d|d|<|r |dur ||d|<|S|r,|dur,||d<|S)Nlossloss_ grad_norm_rg)_detach_loss_dictpop)rrerfrZrgloss_dict_detachedrsrsrtdetach_loss_dictos   zTrainer.detach_loss_dictc Cs||j|j\}}tj|||jd.|dur$|j||||d\}} n||||\}} Wd|| fSWd|| fS1sDwY|| fS)N device_typercenabled)rZ)rdrRrTrautocastr_) rrOrrrrZrbrcoutputsrersrsrt _compute_losss  zTrainer._compute_losscCsXd}d|vr*|jdur*|dur'z|j|}W|Sty&tdY|Sw|j}|S)NrZr[zE [!] You are using multiple optimizers but `grad_clip` is not a list.)r[r3rr)rrZr[rsrsrt_set_grad_clip_per_optimizers  z$Trainer._set_grad_clip_per_optimizercCs(tjtjdd||DddddS)NcSsg|]}|jdqS))gradview)rPparamrsrsrtrRsz.Trainer._compute_grad_norm..r)dim)rX)rnormcatrY)rr^rsrsrt_compute_grad_norms(zTrainer._compute_grad_normr[ AMPScalercCsR|dur"|dkr"|r|||j|tjj|||}|S||}|S)zPerform gradient clippingNr) unscale_rbefore_gradient_clippingrr rclip_grad_norm_rYr~)rr[r^rrgrsrsrt_grad_clippings   zTrainer._grad_clippingr|rnum_optimizersc  Cst} |j|||||d\} } | st| }| i|fS|j||d}d}d}|j|| | dt|j| d<|jr|j |\|j rK|jj nt }|@|j | d||}|jjrs|durs|dkrs|j||||jjs|jjs||jddWdn1swYWdn1swYn|jr|jrt| d| }| Wdn1swY| r|j||dd}n\|| d | r|j|||d}|}|||dus|d | kr||| d <||k}n#| d | r2|j||dkr.t j!j"|#||}||durF|rF|jjsF| rF|| rO|jddt$|t j%rdt &|sbt '|rdd}t| }|(| | ||}| ||fS) alPerform a forward - backward pass and run the optimizer. Args: batch (Dict): Input batch. If model (nn.Module): Model for training. Defaults to None. optimizer (Union[nn.optim.Optimizer, List]): Model's optimizer. If it is a list then, `optimizer_idx` must be defined to indicate the optimizer in use. scaler (AMPScaler): AMP scaler. criterion (nn.Module): Model's criterion. scheduler (torch.optim.lr_scheduler._LRScheduler): LR scheduler used by the optimizer. config (Coqpit): Model config. optimizer_idx (int, optional): Target optimizer being used. Defaults to None. step_optimizer (bool, optional): Whether step the optimizer. If False, gradients are accumulated and model parameters are not updated. Defaults to True. num_optimizers (int, optional): Number of optimizers. Defaults to 1. Raises: RuntimeError: When the loss is NaN. Returns: Tuple[Dict, Dict, int, torch.Tensor]: model outputs, losses, step time and gradient norm. )rOrrrrZ)rrZrTrhN set_to_none)r[r^rr| amp_scaler))timertrurbefore_backward_passrqr}r{r accumulaterRrrrbackwardr~sync_gradientsrr#r0rr\optimizer_step_was_skipped zero_gradrrr- scale_lossrscale get_scaleupdaterrr rrYrTensorisnanisinfrn)rrOrr^rrrrrZrfrstep_start_timersre step_timer[rgupdate_lr_schedulerctx_mgr scaled_loss scale_prevrmrsrsrtrs#               &  zTrainer.optimize batch_n_stepsr0loader_start_timec Cs0|j|||}t|}d}i}t|jdrht}||jj|jj \} } t j | | |jjd|j ||\} } Wdn1sIwYt|}| durZdS| | ddd} || nd} |d|jdkr{|d|kr{d} t|jts|j ||j|j|j|j|j|j| dd \} } }|| nsdgt|j}d}t|jD]W\}}|j}|j}d}|jdur|j|}|j ||j|||||j|| t|jd \} } }||7}| ||<| dur| D]\}}||vr|||d |<q|||<q|}q|} | r|jjdd i}||d <||d <|j|i}|D] \}}||d|<q'|j||j|jjdkri}t|jtrlt|jD]\}}|j|jdd}|d||iqQn3t|jtr|jD]\}}|j|jdd}|d||iqxn |jjdd}d|i}|||t |dt |dd|j!"|||j||jj#|j$j%dkr|j|jj&dkr|j'(|j||j|jj)dkr|jdkr|jj*r|+|j|jj,dkr|j-|| d|j'.|jd7_|j/|| |fS)aPerform a training step on a batch of inputs and log the process. Args: batch (Dict): Input batch. batch_n_steps (int): Number of steps needed to complete an epoch. Needed for logging. step (int): Current step number in this epoch. loader_start_time (float): The time when the data loading is started. Needed for logging. Returns: Tuple[Dict, Dict]: Model outputs and losses. Nrror$Tr|rF)rfr-ravg_loader_time avg_step_timeavg_r] current_lr_ current_lr)r loader_time)rOrs)0ron_train_step_startrSrrrrdrrRrTrrrrrnrr}rr^rrrrrrrrr update_valuesrr;r:rrroundrprint_train_step avg_valuesrrr=rAtrain_step_statsrDrGrrC update_training_dashboard_loggerflushon_train_step_end)rrOrr0rroutputs_per_optimizerrerrbrcrs loss_dict_newrftotal_step_timer*r^rrrrVrQkeep_avg_updateupdate_eval_valuesrvaluelrsrrsrsrtr]!s                     zTrainer.train_stepc Cs|jdur|j|j|jdd|_||j|_td|jdkr(|jj n|j t }|j ||jt }t|j}t|jD]F\}}|||||\}}|durbtdqJ~t }|jjdur|j|jjdkr||jdkr|jj n|j tdqJt |}|j ||jdur|jjrt|jtr|jD] } | dur| qnt|jtr|j D] } | dur| qn|j|j!j"dkrd|i} | #|j$j%|j&'|j| |jj(r|j&)|j|jtj*+dS)zQMain entry point for the training loop. Run training on the all training samples.NTrAr|zC [!] `train_step()` retuned `None` outputs. Skipping training step.r epoch_time),rrIrrrrset_grad_enabledrrrCtrainrron_train_epoch_startrprint_train_startrrr]rrrrNr eval_epochon_train_epoch_endrr\rrr0rrvaluesrrrrrrAtrain_epoch_statsr> model_weightsrr7) repoch_start_timerbatch_num_stepscur_steprOrs_rr epoch_statsrsrsrt train_epochsj                  zTrainer.train_epochcCsh||g}t|drt|dr|j||S|||S|dur$||t|dr/|jj|S|j|S)a Perform a evaluation forward pass. Compute model outputs and losses with no gradients. Args: batch (Dict): IBatch of inputs. model (nn.Module): Model to call evaluation. criterion (nn.Module): Model criterion. optimizer_idx (int, optional): Optimizer ID to define the closure in multi-optimizer training. Defaults to None. Returns: Tuple[Dict, Dict]: model outputs and losses. rrCN)rr\rC eval_stepr[)rrOrrrZr^rsrsrt_model_eval_step s       zTrainer._model_eval_stepc Csttg}i}t|jtrt|jdr.|||j|j\}}|dur- WddSnBdgt |j}t |jD]4\}}|j}|||j||\}} |durZWddS|||<| ro| d| d|<| | q;| |}i} |D] \} } | | d| <q{|j| |jjr|j|||jjWd||fSWd||fS1swY||fS)aPerform a evaluation step on a batch of inputs and log the process. Args: batch (Dict): Input batch. step (int): Current step number in this epoch. Returns: Tuple[Dict, Dict]: Model outputs and losses. rNr$rhrir)rno_gradrr^rrrrrrrrlrrkrrrrrKrprint_eval_stepr) rrOr0rsrer*rroutputs_rrrrrsrsrtr)sF      zTrainer.eval_stepcCsp|jdurtn|j|_|jdur"|jjr|j|j|jddnd|_t d|j |j t}d}d}t|jD]/\}}||}t|}|jd|i|||\}}|durgtdq>|}t}q>|jjdkr|durt|j drt|j jd r|j j|||j|j|jnt|j d r|j |||j|j|j|j|j|jjtj dS) zWMain entry point for the evaluation loop. Run evaluation on the all validation samples.NTrFrzD [!] `eval_step()` retuned `None` outputs. Skipping evaluation step.rrCeval_log)!rrrrrMrLrrrrrevalrprint_eval_startrrrSrrrrrrr\rrCrrAr eval_statsrrr7)rrrOrsrrrrrsrsrtrTsZ        zTrainer.eval_epochcCsJ|jd}t|jds|jdkr/t|jjdr/|jdkr'|jj|j}nF|j|j}n>t|jdsA|jdkrmt|jjdrm|j|j|jrK|jn|j dd|_ |jdkrc|jj |j|j d}n |j |j|j d}t|jds|jdkrt|jjdr|jdkr|jj ||j |j|jdS|j ||j |j|jdSdSdS)aRun model test. Test run is expected to pass over test samples and produce logging artifacts. If ```model.test_run()``` is defined, it will be called and it is expected to set and execute everything in the model. Else if ```mode.test()``` is defined, it will be called and it takes an test data loader as an argument and iterate over it. Ntest_runr|testTrtest_log)rrrrrCrrrNrrrrrrAr)r test_outputsrsrsrtrs0   $   zTrainer.test_runcCs|jjrW|jdks |jjrYtdtj|jjt |jj dd}d|vrNt |dt r2|d|_ nt |dtrN|jjrFd|dd|_ n|ddd|_ td|j dSdSdS) zRestore the best loss from the args.best_path if provided else from the model (`args.continue_path`) used for resuming the trainingrz" > Restoring best loss from %s ...r-r. model_lossNrz) > Starting with loaded last best loss %s)rrwrryrrrrrrrxrrrrrqrrM)rchrsrsrt_restore_best_losss zTrainer._restore_best_losscCsdtdt|_|dur||_|j}|dur||_n|j|_||j |j |jj ||_dS)aRun evaluation steps on the test data split. You can either provide the model and the test samples explicitly or the trainer use values from the initialization. Args: model (nn.Module, optional): Model to use for testing. If None, use the model given in the initialization. Defaults to None. test_samples (List[str], optional): List of test samples to use for testing. If None, use the test samples given in the initialization. Defaults to None. z > USING TEST SET...N) rrrrrrrrrprint_epoch_endrr)rrreval_samples_cachersrsrtrs  z Trainer.testcCs ||j|_td|jjD]s}|jdkrt|j |t |_ |jj r+t nd|_||_|j||jj|j|jsF|jsF||jj rN|||jjkr^|jjdkr^||j||jj rj|jjn|j j|jjdvry||j |d|_qdS)u8🏃 train -> evaluate -> test for the number of epochs.rr|N)NrF)rrrrangerrUrdistbarrierron_epoch_startrrrMrrrprint_epoch_startr1rrrrrLrrrrrr on_epoch_end)rr1rsrsrt_fits2     z Trainer._fitc Cst|} ttjzttj||j_t d|jj| WdSt yT}z|dkrIt |rI|d}ttjnWYd}~n-d}~wt y|}z|dkrqt |rq|d}ttjnWYd}~nd}~wwq)NTz > current batch size: %ir|r{)r&gccollectrrr7rrWrrrr4r' Exception)rstarting_batch_sizebs exceptionrsrsrtfit_with_largest_batch_sizes<       z#Trainer.fit_with_largest_batch_sizec Csz||jjdkr|jWdSWdStyktd|jj r2td| | |j |t|j|jdkrFt|jjdkrQ|jz tdWYdStyjtdYYdSwtyt|jttdYdSw)u&Where the ✨️magic✨️ happens...rz > Keyboard interrupt detected.z! > Saving model before exiting...r|N)rrrrAfinishKeyboardInterruptrrrrBrrron_keyboard_interruptrr1rrdestroy_process_grouprexit SystemExitr_exit BaseException traceback print_excrrsrsrtfit&s6            z Trainer.fitcCsnt|_|r ||j_|r||d|j_d|j_||j_ddg|_||_ |j | |j |j S)a4Run training under the torch profiler. Example:: Run torch profiler to profile CPU, GPU and memory usage with Tensorboard logging. >>> import torch >>> profiler = torch.profiler.profile( >>> activities=[ >>> torch.profiler.ProfilerActivity.CPU, >>> torch.profiler.ProfilerActivity.CUDA, >>> ], >>> schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2), >>> on_trace_ready=torch.profiler.tensorboard_trace_handler("./profiler/"), >>> record_shapes=True, >>> profile_memory=True, >>> with_stack=True, >>> ) >>> prof = trainer.profile_fit(profiler, epochs=1, small_run=64) FicSs |jSr&)torch_profilerr0)r,rsrsrtis z%Trainer.profile_fit..) rrArepocshsrrMrLrUcallbacks_on_train_step_endrstartrstop)rrrUrrsrsrt profile_fitHs   zTrainer.profile_fitcCsj||j}||j}t||d|j|j|j|j|jr|j nd|j |j |j |jj |jj|jjd |_dS)z[Save the best model. It only saves if the current target loss is smaller then the previous.rN) keep_all_best keep_after save_func)_pick_target_avg_lossrrrrrrr^rrrrr1rHrIrA save_modelrrrrsrsrtrts   zTrainer.save_best_modelc Cs^||j}||j}t|j|j|j|jr|jnd|j |j |j ||d|jj |j jd dS)z"Save the current model checkpoint.Nr)rrFr)rrrrrrr^rrrrr1rFrArrrsrsrtrs   zTrainer.save_checkpointcCsd|jd|jg}|jj|jdd|d|durQ|durSt|jdr.N) r getLogger FileHandler Formatter setFormatter addHandlerrrhandlers)rr logger_newhandlerfmtrsrsrtrhs     zTrainer._setup_logger_configcCstjdduS)z$Check if Nvidia's APEX is available.apexN) importlibutil find_specrsrsrsrtruszTrainer._is_apex_availabler$r&)NTr|)rN)r)Urirjrkrur rmrr rrrrrrprorpropertyrrr{rr staticmethodrrr rrrrrrrrr Optimizerrr-rr rr6rErIrLrNrSrYr_rdrqrnrtrur~rra _LRSchedulerrr]rrrrrrrrrrrr+rrrr!r;r"rrrkrrrrrsrsrsrtr!s       )     +  %  =  )))(    "      &} 'F   + :*    ",    &&     r)Nrrrrrrrrr contextlibr dataclassesrrinspectrtypingrrrr r rtorch.distributed distributedrrr r torch.nn.parallelr rtorch.utils.datartrainer.analyticsrtrainer.callbacksrtrainer.generic_utilsrrrrrrrr trainer.iorrrrrtrainer.loggingrrr trainer.trainer_utilsr!r"r#r$r%trainer.utils.cuda_memoryr&r'trainer.utils.distributedr(r)r*r+rrrr-r.rurrsrsrsrtsH         (   #=