o oi@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZddlmZddlmZmZddlmZddlZddlZddlmZdd lmZdd lmZdd lmZ dd l!m"Z"m#Z#m$Z$dd l%m&Z&ddl'Z(ddl)m*Z*m+Z+ddl,m-Z-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6e(j7j89e2e:Z;ej<=ej<>e?Z@ejd?d@dAdBddCddDdEdFdEZEeGdGdHdHZFdie@dIfdJdKZGdLdMZHdVdNdOZIdPdQZJGdRdSdSeZKe(j7jLjMGdTdUdUZNdS)WzCore SpeechBrain code for running experiments. Authors * Peter Plantinga 2020, 2023 * Abdel Heba 2020 * Mirco Ravanelli 2020 * Aku Rouhe 2021 * Andreas Nautsch 2022 * Sylvain de Langen 2023 * Adel Moumen 2023, 2024 N)contextmanager) dataclass)date)Enumauto)SimpleNamespace)resolve_references) DataParallel) SyncBatchNorm)DistributedDataParallel) DataLoaderDistributedSamplerIterableDataset)tqdm) LoopedLoaderSaveableDataLoader)DistributedSamplerWrapperReproducibleRandomSampler)is_distributed_initialized) get_logger)rm_vector_weight_decay)prepare_profilerzlog-config.yamlbrain_intra_epoch_ckpt test_onlyFdebug debug_batches debug_epochsdebug_persistentlydevicecpudata_parallel_backenddistributed_backendncclfind_unused_parametersjitjit_module_keyscompilecompile_module_keys compile_modezreduce-overheadcompile_using_fullgraph#compile_using_dynamic_shape_tracing precisionfp32eval_precision auto_mix_precbfloat16_mix_prec max_grad_normg@skip_nonfinite_gradsnonfinite_patience noprogressbarckpt_interval_minutesckpt_interval_stepsgrad_accumulation_factoroptimizer_step_limittqdm_colored_bar tqdm_barcolorGREENMAGENTACYAN)trainvalidtestremove_vector_weight_decayprofile_trainingprofile_warmup profile_stepsc@s(eZdZUdZejed<eddZdS) AMPConfigzConfiguration for automatic mixed precision (AMP). Arguments --------- dtype : torch.dtype The dtype to use for AMP. dtypecCsN|dus|dkr ttjS|dkrttjS|dkrttjStd|d)a8Create an AMPConfig from a string name. Arguments --------- name : str The name of the AMPConfig to create. Must be one of `fp32`, `fp16`, or `bf16`. Returns ------- AMPConfig The AMPConfig corresponding to the name. Nr/fp16bf16zSpecified autocast mode (z4) incorrect, expected one of `fp32`, `fp16`, `bf16`.)rItorchfloat32float16bfloat16 ValueError)selfnamerTN/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/speechbrain/core.py from_namems    zAMPConfig.from_nameN) __name__ __module__ __qualname____doc__rMrJ__annotations__ classmethodrVrTrTrTrUrIas  rITcCs ztjjrtj|st||durmtj|d}t | }t ||}Wdn1s1wYt |d*}t dt |dt dtj||dt d|dt||Wdn1shwYttj} | durtj| j} t| |tj|d} d d d | iii} tjj|| tt_tjjtd td ||rtjj } t tj|dd }|!| Wdn#1swYWtjj"dSWtjj"dSWtjj"dSWtjj"dStjj"w)a*Create the output folder and relevant experimental files. Arguments --------- experiment_directory : str The place where the experiment directory should be created. hyperparams_to_save : str A filename of a yaml file representing the parameters for this experiment. If passed, references are resolved, and the result is written to a file in the experiment directory called "hyperparams.yaml". overrides : dict A mapping of replacements made in the yaml file, to save in yaml. log_config : str A yaml filename containing configuration options for the logger. save_env_desc : bool If True, an environment state description is saved to the experiment directory, in a file called env.log in the experiment directory. Nzhyperparams.yamlwz# Generated %s from:)filez# %sz# yamllint disablezlog.txthandlers file_handlerfilenamezBeginning experiment!zExperiment folder: zenv.log)#sbutils distributedif_main_processospathisdirmakedirsjoinopenrprintrtodayabspathshutil copyfileobjinspect getmodule currentframef_backrealpath__file__copylogger setup_logging_logging_excepthooksys excepthookquirkslog_applied_quirksinfoget_environment_descriptionwrite ddp_barrier)experiment_directoryhyperparams_to_save overrides log_config save_env_deschyperparams_filenamef resolved_yamlr]module callingfilelog_filelogger_overridesdescription_strforTrTrUcreate_experiment_directorysX             1rcCstjd|||fddS)z-Interrupt exception raising to log the error.z Exception:)exc_infoN)rxerror)exc_type exc_value exc_tracebackrTrTrUrzsrzcCs*|dur tjdd}tjdd}|jdtdd|jdd d d d |jd d d dd |jdtddd|jdtddd|jdd d dd |jdtdd|jdtddd|jdd d dd |jdtdd d|jd!d d d"d |jd#d d d$d |jd%td&d'd(|jd)d d d*d |jd+td&d,d(|jd-td&d.d(|jd/td&d0d(|jd1td&d2d(|jd3td4d|jd5td6d|jd7dd d8d |jd9dd d:d |jd;tdd |jd?td@d|jdAdd dBd |jdCtdDd|jdEtdFd|jdGtdHd|jdItdJd|jdKd d dLd |jdMd d dNd |jdOd d dPd |jdQdRtdSdT|jdUdRtdVdT| |\}}dWdXt | D}|d}|d=t |}|dYrpt jdZkrptd[tjd\}|durd]|d^vr|d^dd_t||d^<|||fS)`aParse command-line arguments to the experiment. Arguments --------- arg_list : list, None A list of arguments to parse. If not given, this is read from `sys.argv[1:]` Returns ------- param_file : str The location of the parameters file. run_opts : dict Run options, such as distributed, device, etc. overrides : dict The overrides to pass to ``load_hyperpyyaml``. Example ------- >>> argv = ['hyperparams.yaml', '--device', 'cuda:1', '--seed', '10'] >>> filename, run_opts, overrides = parse_arguments(argv) >>> filename 'hyperparams.yaml' >>> run_opts["device"] 'cuda:1' >>> overrides 'seed: 10' Nr:zRun a SpeechBrain experiment) description param_filezMA yaml-formatted file using the extended YAML syntax. defined by SpeechBrain.)typehelpz --test_onlyF store_truezRun the experiment in evaluate only mode.It skips the training and goes directly to the evaluation.The model is expected to be already trained.)defaultactionrz--debugzbRun the experiment with only a few batches for all datasets, to ensure code runs without crashing.z--debug_batchesrz'Number of batches to run in debug mode.)rrrz--debug_epochsz^Number of epochs to run in debug mode. If a non-positive number is passed, all epochs are run.z--debug_persistentlyz4Keep data stored during debug mode (not using /tmp).z --log_configz4A file storing the configuration options for loggingz--devicezcuda:0z3The device to run the experiment on (e.g. 'cuda:0')z--data_parallel_backendz.This flag enables training with data_parallel.z--distributed_backendr%zOne of {nccl, gloo, mpi}z--find_unused_parametersz-This flag disable unused parameters detectionz--jitzEnables jit compilation for all modules. Compilation may fail depending on the modules. Use --jit_module_keys to compile a subset of modules.z--jit_module_keys*z.A list of keys in the 'modules' dict to jitify)rnargsrz --compilezEnabling this flag compiles all modules using torch.compile (if available). Beta feature. Use --compile_module_keys to compile a subset of modules. Set the compilation flags below properly. Compilation can be time-consuming and might fail.z--compile_module_keyszA list of keys in the 'modules' dict to compile using TorchInductor. If a module also has a JIT key specified, TorchInductor will take precedence when available.z--compile_modez/One of {default, reduce-overhead, max-autotune}z--compile_using_fullgraphz6Whether it is ok to break model into several subgraphsz%--compile_using_dynamic_shape_tracingz)Use dynamic shape tracing for compilationz --precisionzeThis flag enables training with automatic mixed-precision.It can be set to `fp32`, `fp16`, or `bf16`.z--eval_precisionzfThis flag enables inference with automatic mixed-precision.It can be set to `fp32`, `fp16`, or `bf16`.z--auto_mix_precz:This flag enables training with automatic mixed-precision.z--bfloat16_mix_precz9This flag enables training with bfloat16 mixed-precision.z--max_grad_normzMGradient norm will be clipped to this value, enter negative value to disable.z--skip_nonfinite_gradsz=Set the gradients to None if they are nonfinite (inf or nan).z--nonfinite_patiencez=Max number of batches per epoch to skip if loss is nonfinite.z--noprogressbarz.This flag disables the data loop progressbars.z--ckpt_interval_minuteszyAmount of time between saving intra-epoch checkpoints in minutes. If non-positive, intra-epoch checkpoints are not saved.z--ckpt_interval_stepszlSave an intra-epoch checkpoint after this many steps.If non-positive, intra-epoch checkpoints are not saved.z--grad_accumulation_factorz?Number of batches to accumulate gradients before optimizer stepz--optimizer_step_limitzDNumber of optimizer steps to run. If not passed, all epochs are run.z--tqdm_colored_barzUEnable colored progress-bar in tqdm. If this is false, tqdm shall use default colors.z--remove_vector_weight_decayzUMake vectors (e.g. norms and biases) a separate parameter group without weight_decay.z--profile_trainingzIf set to True, a profiler will be initiated and tensorboard logs will be generated. Please ensure you have installed the torch.TensorBoard profiler with 'pip install torch_tb_profiler'.z--profile_warmuprGz7Number of warmup steps before logging for the profiler.)rrrz--profile_stepsz+Number of steps of logging for the profilercSsi|] \}}|dur||qSNrT).0kvrTrTrU sz#parse_arguments..r#rzYou must have at least 1 GPU. LOCAL_RANKcudar!)r{argvargparseArgumentParser add_argumentstrintboolfloatparse_known_argsvarsitems_convert_to_yamlrMr device_countrQrfenvironget)arg_listparserrun_optsrr local_rankrTrTrUparse_argumentss      rcCs^d}d|}|d}|D]}|dr$|d|tddd7}q|d|7}q|S)z"Convert args to yaml for overrides=z-- N: )rjsplit startswithlenstrip)r yaml_string joined_args split_argsargrTrTrUrs   rc@s"eZdZdZeZeZeZdS)Stagez*Simple enum to track stage of experiments.N)rWrXrYrZrTRAINVALIDTESTrTrTrTrUrs  rc@sjeZdZdZ     dHddZddZddZd d ZdId d ZdId dZ dJddZ ddZ ddZ ddZ dKddZdLddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Zed+d,Zd-d.Zd/d0Zd1d2Zddiifd3d4Zed5d6Zd7d8Z d9d:Z!d;d<Z"dddifd=d>Z#d?d@Z$e%dMdBdCZ&e'j(j)j*dDdEZ+e'j(j)j,dFdGZ-dS)NBrainaTBrain class abstracts away the details of data loops. The primary purpose of the `Brain` class is the implementation of the ``fit()`` method, which iterates epochs and datasets for the purpose of "fitting" a set of modules to a set of data. In order to use the ``fit()`` method, one should sub-class the ``Brain`` class and override any methods for which the default behavior does not match the use case. For a simple use case (e.g., training a single model with a single dataset) the only methods that need to be overridden are: * ``compute_forward()`` * ``compute_objectives()`` The example below illustrates how overriding these two methods is done. For more complicated use cases, such as multiple modules that need to be updated, the following methods can be overridden: * ``fit_batch()`` * ``evaluate_batch()`` Arguments --------- modules : dict of str:torch.nn.Module pairs These modules are passed to the optimizer by default if they have trainable parameters, and will have ``train()``/``eval()`` called on them. opt_class : torch.optim class A torch optimizer constructor that takes only the list of parameters (e.g. a lambda or partial function definition). By default, this will be passed all modules in ``modules`` at the beginning of the ``fit()`` method. This behavior can be changed by overriding the ``configure_optimizers()`` method. hparams : dict Each key:value pair should consist of a string key and a hyperparameter that is used within the overridden methods. These will be accessible via an ``hparams`` attribute, using "dot" notation: e.g., self.hparams.model(x). run_opts : dict A set of options to change the runtime environment, including debug (bool) If ``True``, this will only iterate a few batches for all datasets, to ensure code runs without crashing. debug_batches (int) Number of batches to run in debug mode, Default ``2``. debug_epochs (int) Number of epochs to run in debug mode, Default ``2``. If a non-positive number is passed, all epochs are run. debug_persistently (bool) Keep data stored during debug mode (not using /tmp), Default ``False``. jit (bool) Enable to compile all modules using jit, Default ``False``. jit_module_keys (list of str) List of keys in ``modules`` that should be jit compiled. compile (bool) Enable to compile all modules using torch.compile, Default ``False``. compile_module_keys (list of str) List of keys in ``modules`` that should be compiled using ``torch.compile``. If ``torch.compile`` is unavailable, an error is raised. compile_mode (str) One of ``default``, ``reduce-overhead``, ``max-autotune``, Default ``reduce-overhead``. compile_using_fullgraph (bool) Whether it is ok to break model into several subgraphs, Default ``False``. compile_using_dynamic_shape_tracing (bool) Use dynamic shape tracing for compilation, Default ``False``. distributed_backend (str) One of ``nccl``, ``gloo``, ``mpi``. device (str) The location for performing computations. precision (str) One of ``fp32``, ``fp16``, ``bf16``. eval_precision (str) One of ``fp32``, ``fp16``, ``bf16``. auto_mix_prec (bool) If ``True``, automatic mixed-precision (fp16) is used. Activate it only with cuda. Note: this is a deprecated feature, and will be removed in the future. bfloat16_mix_prec (bool) If ``True``, automatic mixed-precision (bf16) is used. Activate it only with cuda. Note: this is a deprecated feature, and will be removed in the future. max_grad_norm (float) Default implementation of ``fit_batch()`` uses ``clip_grad_norm_`` with this value. Default: ``5``. skip_nonfinite_grads (bool) If ``True``, sets gradients to zero if they are non-finite (e.g., NaN, Inf). Default: ``False``. nonfinite_patience (int) Number of times to ignore non-finite losses before stopping. Default: ``3``. noprogressbar (bool) Whether to turn off progressbar when training. Default: ``False``. ckpt_interval_minutes (float) Amount of time between saving intra-epoch checkpoints, in minutes, default: ``15.0``. If non-positive, these are not saved. ckpt_interval_steps (int) Number of steps between saving intra-epoch checkpoints. If non-positive, these are not saved. Default: ``0``. Typically in a script this comes from ``speechbrain.parse_args``, which has different defaults than Brain. If an option is not defined here (keep in mind that parse_args will inject some options by default), then the option is also searched for in hparams (by key). checkpointer : speechbrain.Checkpointer By default, this will be used to load checkpoints, and will have the optimizer added to continue training if interrupted. Example ------- >>> from torch.optim import SGD >>> class SimpleBrain(Brain): ... def compute_forward(self, batch, stage): ... return self.modules.model(batch[0]) ... def compute_objectives(self, predictions, batch, stage): ... return torch.nn.functional.l1_loss(predictions, batch[0]) >>> model = torch.nn.Linear(in_features=10, out_features=10) >>> brain = SimpleBrain({"model": model}, opt_class=lambda x: SGD(x, 0.1)) >>> brain.fit(range(1), ([torch.rand(10, 10), torch.rand(10, 10)],)) Nc CsXd|_||_||_tD]L\}}|dur9||vr9|dur0||vr0td|dt||t||||q |durS||vrStd|dt||||q t|||q t j j t krft j j tkstdtt j j dtt j j dtt dtttjdduotjddu|_|jr|jrtd |jd kr|jd krt d |jd krtjd nd |jvrtjt|jd tj| |j|_!|j!D]}t"|j!|dr|j!| |j|j!|<q|durt#d(i||_$|j%r*|j&s*|jdur*t"|jdr*t'(} td| j)t*+| j)|j_,| |j_-d|_.|j/r9tdd|_0|j1rEtdd|_0|jdkr[|j0dksW|j2dkr[td|j0dkoed |jv} |j3rr| rrtdtd| d|j0dtjj4j5| d|_6d|_7|jdkr|j0dkrd|_7nd |jvr|j0dvrd|_7|j7r|jdur|jj8d|j6dd|9|jrttjd|_:t;s|j:d krtd td!td"d#|_|jdur|j8d$||j?st@A|jBd%|_Bd|_C|jDr*td&|jE|jFd'|_GtH|jF|jE|j$jI|_CdSdS))NzInfo: z* arg overridden by command line input to: z arg from hparam file is usedzDetected Python .z-. We suggest using SpeechBrain with Python >=RANKrzTo use data_parallel backend, start your script with: python experiment.py hyperparams.yaml --data_parallel_backend=True To use DDP backend, start your script with: torchrun [args] experiment.py hyperparams.yamlrzThe options `ckpt_interval_minutes` and `ckpt_interval_steps` are mutually exclusive. Please keep only one active per experiment run.rrtocheckpoints_dirzRSince debug mode is active, switching checkpointer output to temporary directory: ztThe option `--auto_mix_prec` is deprecated and will be removed in the future. Please use `--precision=fp16` instead.rKzxThe option `--bfloat16_mix_prec` is deprecated and will be removed in the future. Please use `--precision=bf16` instead.rLr"zThe option `--precision` or `--eval_precision` is set to fp16. This option is not yet supported on CPU. Please use `--precision=bf16` or `--eval_precision=bf16` instead to enable mixed precision on CPU.zThe option `skip_nonfinite_grads` will be ignored because GradScaler is enabled and will automatically skip nonfinite gradients.zGradscaler enabled: z. Using precision: )enabledFT)rKrLscaler) optional_loadz ================ WARNING ===============Please add sb.ddp_init_group() into your exp.pyTo use DDP backend, start your script with: torchrun [args] experiment.py hyperparams.yamlzMTo use DDP, please add sb.utils.distributed.ddp_init_group() into your exp.pyzAOnly the main process is alive, all other subprocess were killed.brainrz$Pytorch profiler has been activated.r:rT)Joptimizers_dict opt_class checkpointerrun_opt_defaultsrrxrrsetattrr{ version_infomajorPYTHON_VERSION_MAJORminorPYTHON_VERSION_MINORwarningrfrrdistributed_launchr#rQr7r8exitr!rMr set_devicernn ModuleDictrmoduleshasattrrhparamsrr tempfileTemporaryDirectoryrSpathlibPathrtempdir train_samplerr1r.r2r0r4amp GradScalerruse_ampadd_recoverableprint_trainable_parametersrankravg_train_lossstepoptimizer_stepr<dictfromkeysr=profilerrErHrFtot_prof_stepsr output_folder) rRrrrrrrrrrgradscaler_enabledrTrTrU__init__s                  zBrain.__init__c Csd}d}|jD]}||7}|jr||7}q |jj}|dkrCannot specify both shuffle=Trueand a sampler in loader_kwargsSB_GLOBAL_SEEDi&l!)seedsortingrandomT drop_last)rr+r& batch_sampler)rr&r+)rr&zDCannot automatically solve distributed sampling for IterableDataset.)rrrQrfrrrr rrrr)rrr rxr)rRr!r#r%r&r( shuffle_ddpr+rTrTrUrsd             zBrain._train_loader_specificscCs4||||jdur|jdSdS)aGets called at the beginning of ``fit()``, on multiple processes if ``distributed_count > 0`` and backend is ddp. Default implementation compiles the jit modules, initializes optimizers, and loads the latest checkpoint to resume training. N)_compile_wrap_distributedinit_optimizersrrecover_if_possiblerRrTrTrU on_fit_start^s   zBrain.on_fit_startcCsb|j}|jdur-|jrt|j}|||_d|ji|_|jdur/|jd|jdSdSdS)aCalled during ``on_fit_start()``, initialize optimizers after parameters are fully configured (e.g. DDP, jit). The default implementation of this method depends on an optimizer class being passed at initialization that takes only a list of parameters (e.g., a lambda or a partial function definition). This creates a single optimizer that optimizes all trainable params. Override this class if there are multiple optimizers. Nr optimizer) rrrrDrr4rrr)rR all_paramsrTrTrUr0ss     zBrain.init_optimizersFcCsP|jdur||jD]}|j|dq dS|jdur&|jj|ddSdS)a Sets the gradients of all optimized ``torch.Tensor``s to zero if ``set_to_none=False`` (default) or to None otherwise. Setting gradients to None should save the memory, e.g. during ``evaluate()`` and thus larger batch might be used. N set_to_none)rfreeze_optimizersvalues zero_gradrr4)rRr7optrTrTrUr:s  zBrain.zero_gradcCs"|jdur|jj||ddSdS)a)Gets called at the beginning of ``evaluate()`` Default implementation loads the best-performing checkpoint for evaluation, based on stored metrics. Arguments --------- max_key : str Key to use for finding best checkpoint (higher is better). By default, passed to ``self.checkpointer.recover_if_possible()``. min_key : str Key to use for finding best checkpoint (lower is better). By default, passed to ``self.checkpointer.recover_if_possible()``. Nmax_keymin_key)rr1)rRr=r>rTrTrUon_evaluate_starts  zBrain.on_evaluate_startc Cs&t|j}|j|jdk}||||| ]|jrMtj |j t |j j d| |tjj}|||tjj}Wdn1sGwYn| |tjj}|||tjj}|j||j}|||Wdn1szwY|r|||||||S)a]Fit one batch, override to do multiple updates. The default implementation depends on a few methods being defined with a particular behavior: * ``compute_forward()`` * ``compute_objectives()`` * ``optimizers_step()`` Also depends on having optimizers passed at initialization. Arguments --------- batch : list of torch.Tensors Batch of data to use for training. Default implementation assumes this batch has two elements: inputs and targets. Returns ------- detached loss rrJ device_typeN)rIrVr.rr9on_fit_batch_startno_syncrrMautocastrJr!rrrbrrrrscalecheck_loss_isfinitebackwardoptimizers_stepon_fit_batch_enddetachr")rRrr should_stepoutputsloss scaled_lossrTrTrU fit_batchs4      zBrain.fit_batchcCs>t|s|jd7_|j|jkrtdtddSdS)aCheck if the loss is finite. If the loss is not finite, log a helpful message and increment the `nonfinite_count`. If the `nonfinite_count` exceeds the `--nonfinite_patience` threshold, stop the training and raise an error. This check is particularly useful when the loss becomes NaN or inf, while the parameters and gradients remain finite. It helps prevent getting stuck in an infinite loop during training. Arguments --------- loss : tensor The loss tensor after ``backward()`` has been called but before the optimizers ``step()``. r:zLoss is not finite and patience is exhausted. To debug, wrap `fit()` with autograd's `detect_anomaly()`, e.g. with torch.autograd.detect_anomaly(): brain.fit(...)zPatience not yet exhausted.N)rMisfinitenonfinite_countr5rQrxr)rRrMrTrTrUrFs  zBrain.check_loss_isfinitecCsN|jD]}|jr$|jdur$t|js$d|_td|j dqdS)zXChecks if the gradients are finite. If not, it will emit a warning and set them to zero.Nz Gradients z% contain NaN or Inf. Setting to None.) rrrgradrMrPallrxrrS)rRparamrTrTrUcheck_gradientss zBrain.check_gradientscCs|S)zBy default, this method returns the passed optimizers. Override this method if you want to freeze some optimizers during training. To do so, return a of active optimizers. rT)rR optimizersrTrTrUr8 szBrain.freeze_optimizerscCs|jdur ||j}n |jdurd|ji}ndS|D]}|j|q|D]}tjj |j dd|j q*|j sG|jrG||D]}|j|qK|j|D]}|jddq]|jd7_dS)z~Performs a step of gradient descent on the optimizers. This method is called every ``grad_accumulation_factor`` steps.Nr4rparamsTr6r:)rr8rr4r9runscale_rMrrcclip_grad_norm_ param_groupsr3 is_enabledr4rUrupdater:r)rRvalid_optimizersr;rTrTrUrHs&        zBrain.optimizers_stepcCr)aCalled at the beginning of ``fit_batch()``. This method is not called under the AMP context manager. Do not assume automatic casting of the input batch to a lower precision (e.g. fp16). Arguments --------- batch : list of torch.Tensors Batch of data to use for training. Default implementation assumes this batch has two elements: inputs and targets. should_step : boolean Whether optimizer.step() was called or not. NrT)rRrrKrTrTrUrBArzBrain.on_fit_batch_startcCr)a Called after ``fit_batch()``. Arguments --------- batch : list of torch.Tensors Batch of data to use for training. Default implementation assumes this batch has two elements: inputs and targets. outputs : list or dictionary of torch.Tensors Returned value of compute_forward(). loss : torch.Tensor Returned value of compute_objectives(). should_step : boolean Whether optimizer.step() was called or not. NrT)rRrrLrMrKrTrTrUrIQszBrain.on_fit_batch_endcCst|j}|jr6tj|jt|jjd|j ||d}|j |||d}Wdn1s0wYn|j ||d}|j |||d}| S)a`Evaluate one batch, override for different procedure than train. The default implementation depends on two methods being defined with a particular behavior: * ``compute_forward()`` * ``compute_objectives()`` Arguments --------- batch : list of torch.Tensors Batch of data to use for evaluation. Default implementation assumes this batch has two elements: inputs and targets. stage : Stage The stage of the experiment: Stage.VALID, Stage.TEST Returns ------- detached loss r@rN) rIrVr0rrMrDrJr!rrrrJr")rRrrroutrMrTrTrUevaluate_batchbs  zBrain.evaluate_batchc Cs|tj||j|d|_|jdur$t|jdr$|j |t }d}t ||j d| |j ddy}|jdurC|j|D]d}|jrQtdnY|j d7_ |d7}||}|||j|_|j|jd|jdur|j |jj|jkrtd |jt|jr|j |jkrn|||r|t }d}qEWdn1swY|jdd |tj|j|d |_d|_ dS) Nr set_epochTrA)initial dynamic_ncolsdisablecolourzTrain iteration limit exceededr:) train_lossz+The profiler finished, training is stopped.r6r) rrrrrAr:rQrrratimerrr=rstart_optimizer_step_limit_exceededrxrrOupdate_averager set_postfixstep_numrstopquitrr_should_save_intra_epoch_ckpt_save_intra_epoch_ckptr) rR train_setrenablelast_ckpt_timesteps_since_ckpttrrMrTrTrU _fit_trainsj           + zBrain._fit_traincCs|jdurdS|jdkr|jdkrdSt|d}d|jko%|kn}|p6d|jko4|kn}ts<|S|g}tjj|dd|dS)zDetermines if an intra-epoch checkpoint should be saved. Returns True if there's a checkpointer and time or steps has exceeded limit. NFrgN@)src)rr7r8rgrrMrdbroadcast_object_list)rRrsrtelapsed_minutesdecisionbroadcast_listrTrTrUros z#Brain._should_save_intra_epoch_ckptcCs|durc|tj||jd}tCt|d| |jddD]"}|j d7_ |j |tjd}| ||}|j rE|j |j krEnq#d|_ |tj||WddS1s\wYdSdS)NrTrBrcrdrer:r^r)rrrrevalrMno_gradrr=rr`rjrrr)rR valid_setrrravg_valid_lossrrMrTrTrU _fit_valids*    "zBrain._fit_validc Cs|jr tddSt|ts!t|ts!|j|fdtjj i|}|dur=t|ts=t|ts=|j|ftjj dd|}| |durI|j }|oPtj j}|D] }|j|||d|j|||d|jrm||jksp|jrsdSqSdS)aEIterate epochs and datasets to improve objective. Relies on the existence of multiple functions that can (or should) be overridden. The following methods are used and expected to have a certain behavior: * ``fit_batch()`` * ``evaluate_batch()`` * ``update_average()`` If the initialization was done with distributed_count > 0 and the distributed_backend is ddp, this will generally handle multiprocess logic, like splitting the training data into subsets for each device and only saving a checkpoint on the main process. Arguments --------- epoch_counter : iterable Each call should return an integer indicating the epoch count. train_set : Dataset, DataLoader A set of data to use for training. If a Dataset is given, a DataLoader is automatically created. If a DataLoader is given, it is used directly. valid_set : Dataset, DataLoader A set of data to use for validation. If a Dataset is given, a DataLoader is automatically created. If a DataLoader is given, it is used directly. progressbar : bool Whether to display the progress of each epoch in a progressbar. train_loader_kwargs : dict Kwargs passed to `make_dataloader()` for making the train_loader (if train_set is a Dataset, not DataLoader). E.G. batch_size, num_workers. DataLoader kwargs are all valid. valid_loader_kwargs : dict Kwargs passed to `make_dataloader()` for making the valid_loader (if valid_set is a Dataset, not DataLoader). E.g., batch_size, num_workers. DataLoader kwargs are all valid. Returns ------- None z8Test only mode, skipping training and validation stages.Nr)rr")rqrrr)rrrr)rrxrr r rrrbrrrr3r6rcrdrervrrrri) rR epoch_counterrqr progressbartrain_loader_kwargsvalid_loader_kwargsrrrrTrTrUfitsX5 z Brain.fitcCs|jduo |j|jkSr)r;rr2rTrTrUri^s  z$Brain._optimizer_step_limit_exceededcCs$|jjddddtditjddS)z,Saves a CKPT with specific intra-epoch flag.Fr:cSs t|jvSr)INTRA_EPOCH_CKPT_FLAGmeta)crTrTrUjs z.Brain._save_intra_epoch_ckpt..T) end_of_epoch num_to_keepckpt_predicater verbosityN)rsave_and_keep_onlyrloggingDEBUGr2rTrTrUrpes zBrain._save_intra_epoch_ckptc Csjttd}|s|jdurtdt}|jr+|jdur!t|j}n t|j}tdt}|j rF|j durz*Brain._wrap_distributed..gloo) device_idsr&csrrrrrTrTrUrr) rr#rranyrr convert_sync_batchnormr$DDPr&r!DP)rRrSrrTrTrUr/s4     zBrain._wrap_distributedc Cs|dur|j }|otjj}t|ts)t|ts)d|d<|j|t j fi|}|j ||d|j t j dd|j d}t?t|d| |jddD]"}|jd 7_|j|t j d } || |}|jrr|j|jkrrnqP|t j |dWdn1swYd |_|S) aIterate test_set and evaluate brain performance. By default, loads the best-performing checkpoint (as recorded using the checkpointer). Arguments --------- test_set : Dataset, DataLoader If a DataLoader is given, it is iterated directly. Otherwise passed to ``self.make_dataloader()``. max_key : str Key to use for finding best checkpoint, passed to ``on_evaluate_start()``. min_key : str Key to use for finding best checkpoint, passed to ``on_evaluate_start()``. progressbar : bool Whether to display the progress in a progressbar. test_loader_kwargs : dict Kwargs passed to ``make_dataloader()`` if ``test_set`` is not a DataLoader. NOTE: ``loader_kwargs["ckpt_prefix"]`` gets automatically overwritten to ``None`` (so that the test DataLoader is not added to the checkpointer). Returns ------- average test loss Nr"r<)rrTrCr|r:r^r)r6rbrcrdrer r rrrrr?rrr}rMr~rr=rr`rjrrr) rRtest_setr=r>rtest_loader_kwargsrr avg_test_lossrrMrTrTrUevaluatesD"    zBrain.evaluatecCs.t|r|||j8}|t||j7}|S)a4Update running average of the loss. Arguments --------- loss : torch.tensor detached loss, a single float value. avg_loss : float current running average. Returns ------- avg_loss : float The average loss. )rMrPrr)rRrMavg_lossrTrTrUrjs zBrain.update_averageTccs~|r:g}|jD]}t|dsq ||jd|_q dVd}|jD]}t|ds.q&|||_|d7}q&dSdVdS)aoCopies pytorch's implementation for doing no_sync across all modules. Explanation: nn.module.no_sync() is a context manager for when one does not want to sync gradients, which happens when using both DDP and gradient accumulation. Speechbrain brain's class can contain multiple modules and calling no_sync on these individually would be very awkward, therefore this contextmanager exists. Arguments --------- use : bool If set to `False` will still sync gradients, useful to make behavior toggleable. Yields ------ None require_backward_grad_syncFNrr:)rr9rappendr)rRuseold_values_listrirTrTrUrC&s"      z Brain.no_synccCsR|j|j|jd}t|d}|t|WddS1s"wYdS)N)rrrr])rrrrkryamldump)rRrg save_dictr]rTrTrU_saveJs "z Brain._savecCs~t| }t|}Wdn1swY|d|_|d|_d|vrA|jj}d|d}|d7}t||j|_ dS|d|_ dS)Nrrrz'optimizer_step' not found in z checkpoint.z7 Using the saved 'step' value (BACKWARDS COMPATIBILITY)) rkr safe_loadrrrrWwarningswarnr)rRrgrrrclsnameMSGrTrTrU_recoverTs       zBrain._recover)NNNNNr)r)F)NN)T).rWrXrYrZrrrrrrrrr3r0r:r?rOrFrUr8rHrBrIrMr~r`rvrorrpropertyrirpr.r/rrjrrCrbrc checkpoints mark_as_saverrmark_as_loaderrrTrTrTrUr sn} Q/   JG  3 / !A b  @ G #  rr)OrZrrqrrfrror{rrgr contextlibr dataclassesrdatetimerenumrrtypesrrMr hyperpyyamlrtorch.nnr rr torch.nn.parallelr rtorch.utils.datar r r tqdm.contribr speechbrainrbspeechbrain.dataio.dataloaderrrspeechbrain.dataio.samplerrrspeechbrain.utils.distributedrspeechbrain.utils.loggerrspeechbrain.utils.optimizersrspeechbrain.utils.profilingrrcr} apply_quirksrWrxrgdirnamernrvDEFAULT_LOG_CONFIGrjrrrrrIrrzrrrrregister_checkpoint_hooksrrTrTrTrUs                      !"&( O