o %ݫi~9@sdZddlZddlZddlZddlZddlmZddlmZddlZ ddl m Z e e Z dZdZdZd Zgd ZdZd Zd ZeegZiZd d ZGdddZed GdddeZedGdddeZddZddiZGdddZddZddZ ddZ!dS) uUtilities for hyperparameter optimization. This wrapper has an optional dependency on Oríon https://orion.readthedocs.io/en/stable/ https://github.com/Epistimio/orion Authors * Artem Ploujnikov 2021 N)datetime)load_hyperpyyaml) get_loggerz orion.clientz%Y%m%d%H%M%S%fhpoptgeneric)ORION_EXPERIMENT_NAMEORION_EXPERIMENT_VERSIONORION_TRIAL_ID hpopt_modetrial_idcsfdd}|S)aZA decorator to register a reporter implementation for a hyperparameter optimization mode Arguments --------- mode: str the mode to register Returns ------- f: callable a callable function that registers and returns the reporter class Example ------- >>> @hpopt_mode("raw") ... class RawHyperparameterOptimizationReporter(HyperparameterOptimizationReporter): ... def __init__(self, *args, **kwargs): ... super().__init__( *args, **kwargs) ... def report_objective(self, result): ... objective = result[self.objective_key] ... print(f"Objective: {objective}") >>> reporter = get_reporter("raw", objective_key="error") >>> result = {"error": 1.2, "train_loss": 7.2} >>> reporter.report_objective(result) Objective: 1.2 cs |t<|S)zA "Call the function that registers and returns the reporter class) _hpopt_modes)clsmodeK/home/ubuntu/.local/lib/python3.10/site-packages/speechbrain/utils/hpopt.pyfJszhpopt_mode..fr)rrrrrr +s c@s8eZdZdZddZddZeddZedd Zd S) "HyperparameterOptimizationReporterzA base class for hyperparameter fit reporters Arguments --------- objective_key: str the key from the result dictionary to be used as the objective cCs ||_dSN objective_keyselfrrrr__init__[s z+HyperparameterOptimizationReporter.__init__cCtS)a Reports the objective for hyperparameter optimization. Arguments --------- result: dict a dictionary with the run result. Returns ------- objective: dict A mapping from metric to score. )NotImplementedrresultrrrreport_objective^s z3HyperparameterOptimizationReporter.report_objectivecCsdS)z-Determines whether this reporter is availableTrrrrr is_availablemz/HyperparameterOptimizationReporter.is_availablecCr)z4The unique ID of this trial (used for folder naming))DEFAULT_TRIAL_IDrrrrr rr!z+HyperparameterOptimizationReporter.trial_idN) __name__ __module__ __qualname____doc__rrpropertyr r rrrrrRs rcs6eZdZdZd fdd ZddZeddZZS) )GenericHyperparameterOptimizationReportera A generic hyperparameter fit reporter that outputs the result as JSON to an arbitrary data stream, which may be read as a third-party tool Arguments --------- reference_date: datetime.datetime The date used to create trial id output: stream The stream to report the results to *args: tuple Arguments to be forwarded to parent class **kwargs: dict Arguments to be forwarded to parent class Ncs.tj|i||p tj|_||_d|_dSr)superrsysstdoutoutputreference_date _trial_id)rr-r,argskwargs __class__rrrs  z2GenericHyperparameterOptimizationReporter.__init__cCs tt|||jd|jdS)aReports the objective for hyperparameter optimization. Arguments --------- result: dict a dictionary with the run result. Example ------- >>> reporter = GenericHyperparameterOptimizationReporter( ... objective_key="error" ... ) >>> result = {"error": 1.2, "train_loss": 7.2} >>> reporter.report_objective(result) {"error": 1.2, "train_loss": 7.2, "objective": 1.2} ) objectiveN)jsondumpdictrr,rrrrrsz:GenericHyperparameterOptimizationReporter.report_objectivecCs*|jdur|jp t}|t|_|jS)awThe unique ID of this trial (used mainly for folder naming) Example ------- >>> import datetime >>> reporter = GenericHyperparameterOptimizationReporter( ... objective_key="error", ... reference_date=datetime.datetime(2021, 1, 3) ... ) >>> print(reporter.trial_id) 20210103000000000000 N)r.r-rnowstrftimeFORMAT_TIMESTAMP)rdaterrrr s  z2GenericHyperparameterOptimizationReporter.trial_idNN) r#r$r%r&rrr'r __classcell__rrr1rr(xs r(orioncsPeZdZdZfddZddZddZdd Zed d Z ed d Z Z S)'OrionHyperparameterOptimizationReporterzA result reporter implementation based on Orion Arguments --------- objective_key: str the key from the result dictionary to be used as the objective cs&tj|dd|_d|_|dS)Nr)r)r orion_clientr. _check_clientrr1rrrs z0OrionHyperparameterOptimizationReporter.__init__cCs8z tt|_WdStytdd|_YdSw)NzOrion is not available) importlib import_module MODULE_ORIONr? ImportErrorloggerwarningrrrrr@s    z5OrionHyperparameterOptimizationReporter._check_clientcCsddd|DS)zFormats the log message for output Arguments --------- result: dict the result dictionary Returns ------- message: str a formatted message z, css"|] \}}|d|VqdS)z = Nr).0keyvaluerrr s zJOrionHyperparameterOptimizationReporter._format_message..)joinitemsrrrr_format_messages z7OrionHyperparameterOptimizationReporter._format_messagecCsB||}td||jdur||j}|j|dSdS)zReports the objective for hyperparameter optimization. Arguments --------- result: dict a dictionary with the run result. zHyperparameter fit: N)rMrEinfor?rr)rrmessageobjective_valuerrrrs   z8OrionHyperparameterOptimizationReporter.report_objectivecCs&|jdurdddtD|_|jS)z;The unique ID of this trial (used mainly for folder naming)N-css|] }t|p dVqdS)NosgetenvrGnamerrrrJs zCOrionHyperparameterOptimizationReporter.trial_id..)r.rKORION_TRIAL_ID_ENVrrrrr s   z0OrionHyperparameterOptimizationReporter.trial_idcCs|jduo tddtDS)zDetermines if Orion is available. In order for it to be available, the library needs to be installed, and at least one of ORION_EXPERIMENT_NAME, ORION_EXPERIMENT_VERSION, ORION_TRIAL_ID needs to be set Ncss|]}t|VqdSrrSrVrrrrJs  zGOrionHyperparameterOptimizationReporter.is_available..)r?anyrXrrrrr sz4OrionHyperparameterOptimizationReporter.is_available) r#r$r%r&rr@rMrr'r r r<rrr1rr>s  r>cOsdt|}|durtd|dtt}||i|}|js0tdtt}||i|}|S)a Attempts to get the reporter specified by the mode and reverts to a generic one if it is not available Arguments --------- mode: str a string identifier for a registered hyperparameter optimization mode, corresponding to a specific reporter instance *args: tuple Arguments to forward to the reporter class. **kwargs: dict Arguments to forward to the reporter class. Returns ------- reporter: HyperparameterOptimizationReporter a reporter instance Example ------- >>> reporter = get_reporter("generic", objective_key="error") >>> result = {"error": 3.4, "train_loss": 1.2} >>> reporter.report_objective(result) {"error": 3.4, "train_loss": 1.2, "objective": 3.4} Nz hpopt_mode z' is not supported, reverting to genericzReverting to a generic reporter)r getrErFDEFAULT_REPORTERr )rr/r0 reporter_clsreporterrrr get_reporters   r^currentc@s6eZdZdZd ddZ d ddZdd Zd d ZdS)!HyperparameterOptimizationContexta A convenience context manager that makes it possible to conditionally enable hyperparameter optimization for a recipe. Arguments --------- reporter_args: list arguments to the reporter class reporter_kwargs: dict keyword arguments to the reporter class Example ------- >>> ctx = HyperparameterOptimizationContext( ... reporter_args=[], ... reporter_kwargs={"objective_key": "error"} ... ) NcCs.|pg|_|pi|_d|_d|_ddi|_dS)NFr3g) reporter_argsreporter_kwargsr]enabledr)rrarbrrrrGs  z*HyperparameterOptimizationContext.__init__TcCs|durg}t|}t|\}}}|rt|ni}|td}|tp&t} |rd|_t | g|j Ri|j |_ t |trtj|rt|dd7} t} t| d| idd} t| fi|}tt} |sk| t| D] }||vrz||vrz||=qmWdn1swY|||fS)a5A version of speechbrain.parse_arguments enhanced for hyperparameter optimization. If a parameter named 'hpopt' is provided, hyperparameter optimization and reporting will be enabled. If the parameter value corresponds to a filename, it will be read as a hyperpyyaml file, and the contents will be added to "overrides". This is useful for cases where the values of certain hyperparameters are different during hyperparameter optimization vs during full training (e.g. number of epochs, saving files, etc) Arguments --------- arg_list: list a list of arguments pass_hpopt_args: enumerable forces arguments that are normally suppressed and only used for hyperparameter optimization to be passed into overrides pass_trial_id: bool whether the "trial_id" argument is passed through (enabled by default) Returns ------- param_file : str The location of the parameters file. run_opts : dict Run options, such as distributed, device, etc. overrides : dict The overrides to pass to ``load_hyperpyyaml``. Example ------- >>> ctx = HyperparameterOptimizationContext() >>> arg_list = ["hparams.yaml", "--x", "1", "--y", "2"] >>> hparams_file, run_opts, overrides = ctx.parse_arguments(arg_list) >>> print(f"File: {hparams_file}, Overrides: {overrides}") File: hparams.yaml, Overrides: {'x': 1, 'y': 2} NFTzutf-8)encodingr ) overridesoverrides_must_match)setsbparse_argumentsrrZ KEY_HPOPTKEY_HPOPT_MODEr[rcr^rarbr] isinstancestrrTpathexistsopen get_trial_idr6list HPOPT_KEYSappend KEY_TRIAL_ID)rarg_listpass_hpopt_args pass_trial_id hparams_filerun_optsoverrides_yamlrerr hpopt_filer hpopt_overrideskeysrHrrrriNsF+   z1HyperparameterOptimizationContext.parse_argumentscCs |td<|SNr_)_contextrrrr __enter__sz+HyperparameterOptimizationContext.__enter__cCsN|dur!|jdur!|j}|sttg|jRi|j}||jdtd<dSr)rr]r^r[rarbrr)rexc_type exc_value tracebackr]rrr__exit__s  z*HyperparameterOptimizationContext.__exit__r;)NT)r#r$r%r&rrirrrrrrr`3s  H r`cOst||}|S)aaInitializes the hyperparameter optimization context Arguments --------- *args : tuple Arguments to forward to HyperparameterOptimizationContext **kwargs : dict Arguments to forward to HyperparameterOptimizationContext Returns ------- HyperparameterOptimizationContext Example ------- >>> import sys >>> with hyperparameter_optimization(objective_key="error", output=sys.stdout) as hp_ctx: ... result = {"error": 3.5, "train_loss": 2.1} ... report_result(result) ... {"error": 3.5, "train_loss": 2.1, "objective": 3.5} )r`)r/r0hpfitrrrhyperparameter_optimizations rcCstd}|r ||_dSdS)a]Reports the result using the current reporter, if available. When not in hyperparameter optimization mode, this function does nothing. Arguments --------- result: dict A dictionary of stats to be reported Example ------- >>> result = {"error": 3.5, "train_loss": 2.1} >>> report_result(result["error"]) r_N)rr)rctxrrr report_results rcCs td}|r |jj}|St}|S)a Returns the ID of the current hyperparameter optimization trial, used primarily for the name of experiment folders. When using a context, the convention for identifying the trial ID will depend on the reporter being used. The default implementation returns a fixed value ("hpopt") Returns ------- trial_id: str the trial identifier Example ------- >>> trial_id = get_trial_id() >>> trial_id 'hpopt' r_)rr]r r")rr rrrrqs  rq)"r&rAr4rTr*r hyperpyyamlr speechbrainrhspeechbrain.utils.loggerrr#rErCr9r"r[rXrjrkrursr r rr(r>r^rr`rrrqrrrrsH    '& C J)t