o %ݫi@sUdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl Z ddl Z ddlmZddlmmZddlmZmZmZmZddlmZeeZdZedZd Zd d d Z e e!e!fe"d <de e!e j#fde e!e!fde e!e j#ffddZ$de e!e j#fde e!e j#ffddZ%ddZ&d:ddZ'eddZ(ddZ)e j*j+e&e j,j-e&e j,j.j/e&ia0e j*j+e(e j,j-e(e j,j.j/e(ia1e2e j3e2dkre&t0e j,j.j4<e(t1e j,j.j4<ne&t0e j,j.j5<e(t1e j,j.j5<e2e j3e2dkre&t0e j6j7j8j9<e(t1e j6j7j8j9<ne&t0e j7j8j9<e(t1e j7j8j9<e j*j+e)ia:zddl;Z<[>> # Toy example Module: >>> class Recoverable(torch.nn.Module): ... def __init__(self, param): ... super().__init__() ... self.param = torch.nn.Parameter(torch.tensor([param])) ... def forward(self, x): ... return x * self.param >>> model = Recoverable(1.) >>> tempdir = getfixture('tmpdir') >>> # In simple cases, the module aims to have a terse syntax, >>> # consisting of three steps. >>> # 1. Specifying where to save checkpoints and what is included in a >>> # checkpoint: >>> checkpointer = Checkpointer(tempdir, {"network": model}) >>> # 2. Recover from the latest checkpoint, if one is found: >>> checkpointer.recover_if_possible() >>> # Run your experiment: >>> data = [(0.1, 0.9), (0.3, 0.8)] >>> for example, target in data: ... loss = (model(example) - target)**2 ... # 3. Save checkpoints, and keep by default just one, the newest: ... ckpt = checkpointer.save_and_keep_only() Authors * Aku Rouhe 2020 * Adel Moumen 2024 N)Dict)version) ddp_barrier ddp_broadcastif_main_processmain_process_only) get_loggerCKPTz.yamlz.ckptz.multihead_attnz.convs_intermediate)z.mutihead_attnz.convs_intermedite KEYS_MAPPING state_dictmappingreturnc Cs\|D]'\}}t|D]}||vr*|||}||||<td||||qq|S)a_ Maps the keys in the old state dictionary according to the provided mapping. NOTE: This function will remap all state_dict keys that contain the old key. For instance, if the state_dict is {'model.encoder.layer.0.atn.self.query.weight': ...} and the mapping is {'.atn': '.attn'}, the resulting state_dict will be {'model.encoder.layer.0.attn.self.query.weight': ...}. Since this effectively works as a mass substring replacement, partial key matches (e.g. in the middle of one layer name) will also work, so be careful to avoid false positives. Parameters ---------- state_dict : dict The old state dictionary to be mapped. mapping : dict A dictionary specifying the mapping between old and new keys. Returns ------- dict The modified state dictionary with mapped keys. z`Due to replacement compatibility rule '%s'->'%s', renamed `state_dict['%s']`->`state_dict['%s']`)itemslistkeysreplacepoploggerinfo)r r replacement_oldreplacement_newold_keynew_keyrQ/home/ubuntu/.local/lib/python3.10/site-packages/speechbrain/utils/checkpoints.pymap_old_state_dict_weightsXs  rcCst|t}|S)aHook to be called when loading a state_dict checkpoint. This hook is called when loading a state_dict checkpoint. It can be used to modify the state_dict before it is loaded into the model. By default, this hook will map the old state_dict keys to the new ones. Arguments --------- state_dict : dict The state_dict to be loaded. Returns ------- dict The modified state_dict. )rr )r altered_state_dictrrr%hook_on_loading_state_dict_checkpoints rcCsD~d}t||}z |j|ddWdSty!||YdSw)aLoads a torch.nn.Module state_dict from the given path instantly. This can be made the default for torch.nn.Modules with: >>> DEFAULT_LOAD_HOOKS[torch.nn.Module] = torch_recovery Arguments --------- obj : torch.nn.Module Instance for which to load the parameters. path : str, pathlib.Path Path where to load from. end_of_epoch : bool Whether the recovery comes from an end of epoch checkpoint. cpuTstrictN)torch_patched_state_dict_loadload_state_dict TypeError)objpath end_of_epochdevicer rrrtorch_recoverys  r(rcCstj||d}t|}|S)aLoads a `state_dict` from the given path using :func:`torch.load` and calls the SpeechBrain `state_dict` loading hooks, e.g. to apply key name patching rules for compatibility. The `state_dict` sees no further preprocessing and is not applied into a model, see :func:`~torch_recovery` or :func:`~torch_parameter_transfer`. Arguments --------- path : str, pathlib.Path Path where to load from. device : str Device where the loaded `state_dict` tensors should reside. This is forwarded to :func:`torch.load`; see its documentation for details. Returns ------- The loaded state dict.  map_location)torchloadr)r%r'r rrrr!sr!cCs4|}|std|d|dt||dS)a Saves the obj's parameters to path. Default save hook for torch.nn.Modules For saving torch.nn.Module state_dicts. Arguments --------- obj : torch.nn.Module Instance to save. path : str, pathlib.Path Path where to save to. zSaving an empty state_dict for z in .N)r rwarningr+save)r$r%r rrr torch_savesr0cCsd}t||}|j|dd}|jD]}td|d|dd|q|jD]}td|d|dd |q*d S) aNon-strict Torch Module state_dict load. Loads a set of parameters from path to obj. If obj has layers for which parameters can't be found, only a warning is logged. Same thing if the path has parameters for layers which don't find a counterpart in obj. Arguments --------- obj : torch.nn.Module Instance for which to load the parameters. path : str Path where to load from. rFrzDuring parameter transfer to z loading from z*, the transferred parameters did not have zparameters for the key: z1, the object could not use the parameters loaded zwith the key: N)r!r" missing_keysrr.unexpected_keys)r$r%r'r incompatible_keys missing_keyunexpected_keyrrrtorch_parameter_transfers(     r6z2.0.0z2.4.0cCs|t|dSN)r,str)r$r%rrr _load_spm"sr9cCHt|}z |ttdWn tyd}t|wd|_|S)aMethod decorator which marks given method as the checkpoint saving hook. See register_checkpoint_hooks for example. Arguments --------- method : callable Method of the class to decorate. Must be callable with signature (instance, path) using positional arguments. This is satisfied by for example: def saver(self, path): Returns ------- The decorated method, marked as a checkpoint saver. Note ---- This will not add the hook (not possible via a method decorator), you must also decorate the class with @register_checkpoint_hooks Only one method can be added as the hook. testpathz6Checkpoint saver must match signature (instance, path)T)inspect signaturebindobjectpathlibPathr#_speechbrain_savermethodsigMSGrrr mark_as_saver0s  rGcCsJt|}z |ttddWn tyd}t|wd|_|S)aMethod decorator which marks given method as checkpoint loading hook. Arguments --------- method : callable Method of the class to decorate. Must be callable with signature (instance, path, end_of_epoch) using positional arguments. This is satisfied by for example: `def loader(self, path, end_of_epoch):` Returns ------- The decorated method, registered as a checkpoint loader. Note ---- This will not add the hook (not possible via a method decorator), you must also decorate the class with @register_checkpoint_hooks Only one method can be added as the hook. r;Tz@Checkpoint loader must have signature (self, path, end_of_epoch))r<r=r>r?r@rAr#_speechbrain_loaderrCrrrmark_as_loaderPs  rIcCr:)a)Method decorator which marks given method as a parameter transfer hook. Arguments --------- method : callable Method of the class to decorate. Must be callable with signature (instance, path) using positional arguments. This is satisfied by for example: `def loader(self, path):` Returns ------- The decorated method, registered as a transfer method. Note ---- This will not add the hook (not possible via a method decorator), you must also decorate the class with @register_checkpoint_hooks Only one method can be added as the hook. Note ---- The transfer hook is prioritized over the loader hook by the ``Pretrainer`` However, if no transfer hook is registered, the Pretrainer will use the loader hook. r;z.Transfer hook must have signature (self, path)T)r<r=r>r?r@rAr#_speechbrain_transferrCrrrmark_as_transferos  rKTcCs|jD]@\}}t|dr#|rt|t|<n|t|<td|t|dr4|t|<td|t|drE|t|<td|q|S)aClass decorator which registers the load, save and transfer hooks. The hooks must have been marked with mark_as_loader and mark_as_saver, and possibly mark_as_transfer. Arguments --------- cls : class Class to decorate save_on_main_only : bool By default, the saver is only run on a single process. This argument provides the option to run the saver on all processes, needed for some savers where data is first gathered before saving. Returns ------- the decorated class with hooks registered Example ------- >>> @register_checkpoint_hooks ... class CustomRecoverable: ... def __init__(self, param): ... self.param = int(param) ... ... @mark_as_saver ... def save(self, path): ... with open(path, "w", encoding="utf-8") as fo: ... fo.write(str(self.param)) ... ... @mark_as_loader ... def load(self, path, end_of_epoch): ... del end_of_epoch # Unused here ... with open(path, encoding="utf-8") as fi: ... self.param = int(fi.read()) rBz$Registered checkpoint save hook for rHz$Registered checkpoint load hook for rJz'Registered parameter transfer hook for ) __dict__rhasattrrDEFAULT_SAVE_HOOKSrdebugDEFAULT_LOAD_HOOKSDEFAULT_TRANSFER_HOOKS)clssave_on_main_onlynamerDrrrregister_checkpoint_hookss(   rUcCs0tt|}|D] }||vr||Sq dS)afFinds the default save/load hook to use with the given object. Follows the Method Resolution Order, i.e., if no hook is registered for the class of the object itself, also searches classes which the object inherits from. Arguments --------- obj : instance Instance of a class. default_hooks : dict Mapping from classes to (checkpointing hook) functions. Returns ------- The correct method or None if no method is registered. Example ------- >>> a = torch.nn.Module() >>> get_default_hook(a, DEFAULT_SAVE_HOOKS) == torch_save True N)r<getmrotype)r$ default_hooksmrorRrrrget_default_hooks  rZ Checkpoint)r%meta paramfilesaNamedTuple describing one saved checkpoint To select a checkpoint to load from many checkpoint, Checkpoints are first filtered and sorted based on this namedtuple. Checkpointers put pathlib.Path in path and a dict in meta. You can essentially add any info you want to meta when saving a checkpoint. The only default key in meta is "unixtime". Checkpoint.paramfiles is a dict from recoverable name to parameter filepath. cCs t|jSr7)hashr%selfrrrs racCs |jdS)zRecency as Checkpoint importance metric. This function can also act as an example of how to make checkpoint importance keyfuncs. This is a named function, but as you can see it could be easily implemented as a lambda in a pinch. unixtimer\ckptrrr ckpt_recencys rfc @s.eZdZdZ    d-ddZ   d.ddZdd Zid dejfd d Z id dd d gggdejf ddZ    d/ddZ     d0ddZ    d/ddZ ddZddZd ddegdejdddZeeejfddZdd Zd!d"Zed#d$Zed%d&Zd'd(Zd)d*Zid fd+d,ZdS)1 Checkpointera Saves checkpoints and recovers from them. Arguments --------- checkpoints_dir : str, pathlib.Path Path to directory where to save checkpoints. recoverables : mapping, optional Objects to to recover. They need a (unique) name: this is used to connect the parameters in a checkpoint to the correct recoverable. The name is also used in the filename of the savefile for the objects parameters. These can also be added with add_recoverable or add_recoverables or just modifying checkpointer.recoverables directly. custom_load_hooks : mapping, optional A mapping from name [same as in recoverables] to function or method. Sets a custom loading hook for a particular object. The function/method must be callable with signature (instance, path) using positional arguments. This is satisfied by for example: `def loader(self, path)`. custom_save_hooks : mapping, optional Mapping from name [same as in recoverables] to function or method. Sets a custom saving hook for a particular object. The function/method must be callable with signature (instance, path) using positional arguments. This is satisfied by for example: def saver(self, path): allow_partial_load : bool, optional If True, allows loading a checkpoint where a savefile is not found for every registered recoverable. In that case, only the found savefiles are loaded. When False, loading such a save will raise RuntimeError. (default: False) Example ------- >>> import torch >>> #SETUP: >>> tempdir = getfixture('tmpdir') >>> class Recoverable(torch.nn.Module): ... def __init__(self, param): ... super().__init__() ... self.param = torch.nn.Parameter(torch.tensor([param])) ... def forward(self, x): ... return x * self.param >>> recoverable = Recoverable(1.) >>> recoverables = {'recoverable': recoverable} >>> # SETUP DONE. >>> checkpointer = Checkpointer(tempdir, recoverables) >>> first_ckpt = checkpointer.save_checkpoint() >>> recoverable.param.data = torch.tensor([2.]) >>> loaded_ckpt = checkpointer.recover_if_possible() >>> # Parameter has been loaded: >>> assert recoverable.param.data == torch.tensor([1.]) >>> # With this call, by default, oldest checkpoints are deleted: >>> checkpointer.save_and_keep_only() >>> assert first_ckpt not in checkpointer.list_checkpoints() NFcCsxt||_tj|jddi|_i|_|dur||i|_|dur*|j |i|_ |dur7|j |||_ dS)NTexist_ok) r@rAcheckpoints_dirosmakedirs recoverablesoptional_recoverablesadd_recoverablescustom_load_hooksupdatecustom_save_hooksallow_partial_load)r`rjrmrprrrsrrr__init__As     zCheckpointer.__init__cCs@||j|<||j|<|dur||j|<|dur||j|<dSdS)aRegister a recoverable with possible custom hooks. Arguments --------- name : str Unique name for recoverable. Used to map savefiles to objects. obj : instance The object to recover. custom_load_hook : callable, optional Called to load the object's savefile. The function/method must be callable with signature (instance, path) using positional arguments. This is satisfied by for example: def load(self, path): custom_save_hook : callable, optional Called to save the object's parameters. The function/method must be callable with signature (instance, path) using positional arguments. This is satisfied by for example: def saver(self, path): optional_load : bool, optional If True, allows for the optional loading of an object from a checkpoint. If the checkpoint lacks the specified object, no error is raised. This is particularly useful during transitions between different training configurations, such as changing precision from floating point 32 to 16. For example, suppose you have a training checkpoint that does not includes a `scaler` object. If you intend to continue pre-training in floating point 16, where the `scaler` object is needed, marking it as optional prevents loading errors. Without marking it as optional, attempting to load the `scaler` object from a checkpoint trained in floating point 32 would fail, as the `scaler` object is not present in that checkpoint. N)rmrnrprr)r`rTr$custom_load_hookcustom_save_hook optional_loadrrradd_recoverableWs $  zCheckpointer.add_recoverablecCs:t|tjjr|j|dSt|}d|d}t|)aUpdate the recoverables dict from the given mapping. Arguments --------- recoverables : mapping Objects to recover. They need a (unique) name: this is used to connect the parameters in a checkpoint to the correct recoverable. The name is also used in the filename of the savefile for the objects parameters. zBCheckpointer needs a mapping (e.g. dict), got z instead.N) isinstance collectionsabcMappingrmrqreprAttributeError)r`rmrecrFrrrros zCheckpointer.add_recoverablesTcCsd}tr#|dur|}n||}tj|dd||t||}t|dd}i}|j D]9\}}|t } || } | ||<||j vrO|j ||| q0t |t } | dur^| || q0dt|d} t| tr|rqdnd } t|d | d |t|||SdS) aSaves a checkpoint. The whole checkpoint becomes a directory. Saves each registered object's parameters in a separate file. Also a meta file is added. The meta file by default has just the unixtime (seconds since unix epoch), but you can add anything relevant yourself. The meta information is later used to pick the checkpoint to load. The value of end_of_epoch is saved in the meta. This can affect how epoch counters and dataset iterators load their state. For multi-process saving there are cases where we may want to run saving code on multiple processes (e.g. FSDP where we need to collect parameters before saving). This works by creating a save folder on the main process and communicating it to all processes, and then letting each saver/loader method control whether it should save on one or all processes. Arguments --------- meta : mapping, optional A mapping which is added to the meta file in the checkpoint. The key "unixtime" is included by default. end_of_epoch : bool, optional Whether the checkpoint is at the end of an epoch. True by default. May affect loading. name : str, optional Specify a custom name for your checkpoint. The name will still have a prefix added. If no name is given, a name is created from a timestamp and a random unique id. verbosity : logging level Set logging level this save. Returns ------- Checkpoint namedtuple [see above], the saved checkpoint, unless this is run on a non-main process, in which case it returns None. NTrhr)srczDon't know how to save O. Register default hook or add custom hook for this object. end-of-epochz intra-epochz Saved an z checkpoint in )r_new_checkpoint_dirpath_custom_checkpoint_dirpathrkrl_save_checkpoint_metafile METAFNAMErrmr PARAMFILE_EXTrrrZrNrW RuntimeErrorrlogr[)r`r\r&rT verbosityckpt_dir saved_metasaved_paramfilesr$objfnamesavepath default_hookrF ckpt_typerrrsave_checkpoints>+          zCheckpointer.save_checkpointc Cs:|j|||| d|r|t|j||||| | ddS)aSaves a checkpoint, then deletes the least important checkpoints. Essentially this combines ``save_checkpoint()`` and ``delete_checkpoints()`` in one call, providing short syntax. Arguments --------- meta : mapping, optional A mapping which is added to the meta file in the checkpoint. The key "unixtime" is included by default. end_of_epoch : bool, optional Whether the checkpoint is at the end of an epoch. True by default. May affect loading. name : str, optional Specify a custom name for your checkpoint. The name will still have a prefix added. If no name is given, a name is created from a timestamp and a random unique id. num_to_keep : int, optional Number of checkpoints to keep. Defaults to 1. This deletes all checkpoints remaining after filtering. Must be >=0. keep_recent : bool, optional Whether to keep the most recent ``num_to_keep`` checkpoints. importance_keys : list, optional A list of key functions used in sorting (see the sorted built-in). Each callable defines a sort order and num_to_keep checkpoints are kept for callable. The checkpoint with the highest keys are kept. The functions are passed Checkpoint namedtuples (see above). max_keys : list, optional A list of keys for which the *highest* value will be kept. min_keys : list, optional A list of keys for which the *lowest* value will be kept. ckpt_predicate : callable, optional Use this to exclude some checkpoints from deletion. Before any sorting, the list of checkpoints is filtered with this predicate. Only the checkpoints for which ckpt_predicate is True can be deleted. The function is called with Checkpoint namedtuples (see above). verbosity : int The logging level, default logging.INFO Note ---- Unlike save_checkpoint, this does not return anything, since we cannot guarantee that the saved checkpoint actually survives deletion. )r\r&rTr) num_to_keepmax_keysmin_keysimportance_keysckpt_predicaterN)rappendrfdelete_checkpoints) r`r\r&rTr keep_recentrrrrrrrrsave_and_keep_onlys:  zCheckpointer.save_and_keep_onlycCs$|j||||dd}|r|dSdS)aPicks a particular checkpoint from all available checkpoints. If none of ``importance_key``, ``max_key``, and ``min_key`` is used, then most recent checkpoint will be returned. No more than one of them may be used. Most functionality is actually implemented in ``find_checkpoints()`` but this is kept as a useful interface. Arguments --------- importance_key : callable, optional The key function used in sorting. The checkpoint with the highest returned value is picked. The function is called with Checkpoint namedtuples. max_key : str, optional The checkpoint with the highest value for this key will be returned. Only checkpoints with this key will be considered! min_key : str, optional The checkpoint with the lowest value for this key will be returned. Only checkpoints with this key will be considered! ckpt_predicate : callable, optional Before sorting, the list of checkpoints is filtered with this predicate. See the filter builtin. The function is called with Checkpoint namedtuples (see above). By default, all checkpoints are considered. Returns ------- Checkpoint If found. None If no Checkpoints exist/remain after filtering. N)importance_keymax_keymin_keyrmax_num_checkpointsr)find_checkpoints)r`rrrr ckpts_foundrrrfind_checkpoint8s*zCheckpointer.find_checkpointcs|durdurdurt}r!|s!fdd}|ffdd }nr4|s4fdd}|ffdd }ns8r.importance_keyc$|dur |jvo ||S|jvSzCheckpoints predicate.Nrcre old_predicaterrrr z5Checkpointer.find_checkpoints..ckpt_predicatecs |j Srrcrdrrrrs crrrcrrrrrrzDMust specify only one of 'importance_key', 'max_key', and 'min_key'.T)keyreverse)rf ValueErrorlist_checkpointsrfiltersorted)r`rrrrrckpts ranked_ckptsr)rrrrns**   zCheckpointer.find_checkpointscCs4|||||}|dur|||Std|S)aYPicks a checkpoint and recovers from that, if one is found. If a checkpoint is not found, no recovery is run. If none of ``importance_key``, ``max_key``, and ``min_key`` is used, then most recent checkpoint will be returned. No more than one of them may be used. Arguments --------- importance_key : callable, optional The key function used in sorting. The checkpoint with the highest returned value is loaded. The function is called with Checkpoint namedtuples. max_key : str, optional The checkpoint with the highest value for this key will be loaded. Only checkpoints with this key will be considered! min_key : str, optional The checkpoint with the lowest value for this key will be loaded. Only checkpoints with this key will be considered! ckpt_predicate : callable, optional Before sorting, the list of checkpoints is filtered with this predicate. See the filter builtin. The function is called with Checkpoint namedtuples (see above). By default, all checkpoints are considered. Returns ------- Checkpoint If found. None If no Checkpoints exist/remain after filtering. Nz1Would load a checkpoint here, but none found yet.)rload_checkpointrr)r`rrrr chosen_ckptrrrrecover_if_possibles)  z Checkpointer.recover_if_possiblecCs||dS)zLoads the specified checkpoint. Arguments --------- checkpoint : Checkpoint Checkpoint to load. N)_call_load_hooks)r` checkpointrrrrzCheckpointer.load_checkpointcCs||S)zList all checkpoints in the checkpoints directory. Returns ------- list List of Checkpoint namedtuple (see above). )_construct_checkpoint_objects_list_checkpoint_dirsr_rrrr rzCheckpointer.list_checkpoints)rrrrrrc Cs|dkrtdt}t}dd|pgD} | dd|pgD| dd|D| D]} || d<||jd i| ||jd d|i| q/t|D] } | |vr`tj| |d qStd S) a. Deletes least important checkpoints. Since there can be many ways to define importance (e.g. lowest WER, lowest loss), the user should provide a list of sort key functions, each defining a particular importance order. In essence, each importance key function extracts one importance metric (higher is more important). For each of these orders, num_to_keep checkpoints are kept. However if there is overlap between each orders' preserved checkpoints, the additional checkpoints are not preserved, so the total number of preserved checkpoints can be less than:: num_to_keep * len(importance_keys) Arguments --------- num_to_keep : int, optional Number of checkpoints to keep. Defaults to 10. You choose to keep 0. This deletes all checkpoints remaining after filtering. Must be >=0 min_keys : list, optional List of strings representing keys in the meta. The lowest of these values will be kept, up to num_to_keep. max_keys : list, optional List of strings representing keys in the meta. The highest of these values will be kept, up to num_to_keep. importance_keys : list, optional A list of key functions used in sorting (see the sorted built-in). Each callable defines a sort order and num_to_keep checkpoints are kept for callable. To be clear, those with the highest key are kept. The functions are called with Checkpoint namedtuples (see above). See also the default (ckpt_recency, above). The default deletes all but the latest checkpoint. ckpt_predicate : callable, optional Use this to exclude some checkpoints from deletion. Before any sorting, the list of checkpoints is filtered with this predicate. Only the checkpoints for which ckpt_predicate is True can be deleted. The function is called with Checkpoint namedtuples (see above). verbosity : logging level Set logging level for this deletion. Note ---- Must be called with keyword arguments, as a signoff that you know what you are doing. Deletion is permanent. rz/Number of checkpoints to keep must be positive.cSg|]}d|iqSrr.0rrrr Rz3Checkpointer.delete_checkpoints..cSrrrrrrrrSrcSr)rrrrrrrTrrr)rNr)rsetextendrqrrrg_delete_checkpoint) r`rrrrrrpotential_deletionsprotected_checkpointsr key_kwargsrerrrrs.9  zCheckpointer.delete_checkpointscCs8t|js tdt|jt|d|jdS)Nz.Checkpoint does not appear valid for deletion.zDeleted checkpoint in )rg_is_checkpoint_dirr%rshutilrmtreerr)rrrrrros  zCheckpointer._delete_checkpointc Cs"td|j|jd}|jD]{\}}z|j|}WnEtyc|jr*Yqd|vr?d|jd|}t |t Yq|j |rVd|jd|d}t |t Yqd|jd|}t |w||jvrs|j||||qt|t}|dur||||qd t|d }t |dS) NzLoading a checkpoint from r dataloaderzLoading checkpoint from z:, but missing a load path for zTrying to load checkpoint from z>, but missing a load path for zU. Skipping as this recoverable is marked as optional.zDon't know how to load r)rrr%r\rmrr]KeyErrorrswarningswarn UserWarningrnrrprZrPrW)r`rr&rTr$loadpathrFrrrrrwsF           zCheckpointer._call_load_hookscCsdd|jDS)NcSsg|] }t|r|qSr)rgr)rxrrrrs z6Checkpointer._list_checkpoint_dirs..)rjiterdirr_rrrrsz"Checkpointer._list_checkpoint_dirsc Csg}|D]>}t|tdd}tj|tjd}Wdn1s!wYi}|D] }|jtkr8|||j<q,| t |||q|S)Nutf-8encoding)Loader) openryamlr,rrsuffixrstemrr[)checkpoint_dirs checkpointsrfir\r]ckptfilerrrrs   z*Checkpointer._construct_checkpoint_objectscCs2t|}|s dS|jtsdS|tS)NF)r@rAis_dirrT startswith CKPT_PREFIXrexists)r%rrrrs   zCheckpointer._is_checkpoint_dircCst}tdt|}d}|jtd|d|dr3|d7}|jtd|d|ds|jtd|d|dS)Nz%Y-%m-%d+%H-%M-%Sr+02dr)timestrftime localtimerjrr)r`tstamp suffix_numrrrrsz$Checkpointer._new_checkpoint_dirpathcCs|jtd|S)Nr)rjr)r`rTrrrrsz'Checkpointer._custom_checkpoint_dirpathcCsft|d}||t|ddd}|d|t|Wd|S1s,wY|S)N)rbrwrrz# yamllint disable )rrqrwriterdump)r`fpathmeta_to_includer&r\forrrrs   z&Checkpointer._save_checkpoint_metafile)NNNF)NNF)NNNN)NNNNN)__name__ __module__ __qualname____doc__rtrxrologgingINFOrrrrrrrrfr staticmethodrrrrrrrrrrrrrrgs~;  +  [ K 8 a 2   \,    rgcCst|}zt|}Wn tytdwd}t6|D]}|D] \}}|||j7<q%|d7}q|D] \}}|jt|||<q>> # Consider this toy Module again: >>> class Recoverable(torch.nn.Module): ... def __init__(self, param): ... super().__init__() ... self.param = torch.nn.Parameter(torch.tensor([param])) ... def forward(self, x): ... return x * self.param >>> # Now let's make some checkpoints: >>> model = Recoverable(1.) >>> tempdir = getfixture('tmpdir') >>> checkpointer = Checkpointer(tempdir, {"model": model}) >>> for new_param in range(10): ... model.param.data = torch.tensor([float(new_param)]) ... _ = checkpointer.save_checkpoint() # Suppress output with assignment >>> # Let's average the 3 latest checkpoints >>> # (parameter values 7, 8, 9 -> avg=8) >>> ckpt_list = checkpointer.find_checkpoints(max_num_checkpoints = 3) >>> averaged_state = average_checkpoints(ckpt_list, "model") >>> # Now load that state in the normal way: >>> _ = model.load_state_dict(averaged_state) # Suppress output >>> model.param.data tensor([8.]) rc3s"|] }|jdVqdS)r)N)r])rrer'parameter_loaderrecoverable_namerr Hs  z&average_checkpoints..css|]}t|VqdSr7)r)rr rrrrLs  r)checkpoint_listrraveragerparameter_iteratoravg_ckptrrraverage_checkpointssAr )r)T)Prrzcollections.abcr<rrkr@rrrtypingrr+r packagingrspeechbrain.utils._workaroundsutils _workarounds__waspeechbrain.utils.distributedrrrrspeechbrain.utils.loggerrrrrrrr r8__annotations__Tensorrrr(r!r0r6nnModuleoptim Optimizer lr_schedulerReduceLROnPlateaurPrNparse __version__ _LRScheduler LRSchedulercudaamp grad_scaler GradScalerrQ sentencepiecespmr9SentencePieceProcessor ImportError_cycliclrsaverCyclicLR_cycliclrloaderrGrIrKrUrZ namedtupler[__hash__rfrgrr,r rrrrs3        +     "     %:   a%