o rric9@sUdZddlmZddlmZmZddlZddlmZddl Z ddl m Z ddl Z ddlZddlZddlZddlmZmZmZdd lmZmZmZdd lmZdd lmZmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%ddl&Z'eedZe j(e)gdfe*d<ddZ+eGdddZ,ddZ-de j.defddZ/ede,fdedede)dede j0ede,de j1e fdd Z2d!d"Z3d#e j1e)ded$e j1e de j1e fd%d&Z4de j.deded'e j1e d(e j(de5f d)d*Z6dS)+z This defines the `dora grid` action, and provides a `run_grid` API that can be used from a notebook or any other script. When using the API, you can provide the equivalent of the command line flags with the `RunGridArgs` dataclass. )ProcessPoolExecutor) dataclassfieldN)partial)Path) SlurmConfig SubmitRulesupdate_from_args)ExplorerLauncherHerd) DecoratedMain)colorize simple_logfatal)SheepShepherd)import_or_fatalreliable_rmtreetry_loadzGrid:logcOsdSN)argskwargsrr=/home/ubuntu/.local/lib/python3.10/site-packages/dora/grid.pyno_print%src@seZdZUdZeedZeje e d<dZ e e d<dZ ee d<dZejee d <d Ze e d <d Ze e d <d Ze e d <d Ze e d<d Ze e d<d Zeje e d<d Ze e d<dZejee d<dZejee d<dZejee d<d Ze e d<dS) RunGridArgsay Arguments to tune the behavior of the `run_grid` function. Args: patterns (list[str]): List of patterns used to filter by name the XPs. monitor (bool): if True, will monitor the advances of the XPs every `interval` minutes, stopping only when all runs completed or failed. interval (float): interval in minutes to wait between updates. trim (int or None): if provided, will trim all XP logs to the epoch of the XP with the provided index. Useful to compare XP started at different times. trim_last (bool): if True, will trim all XP to the least advanced XP. silent (bool): if True, do not print anything (e.g. API usage). dry_run (bool): if True, Dora will simulate the run of the grid, without scheduling or canceling any XP. cancel (bool): if True, will cancel all XPs in the grid. If `patterns` is provided, only XP matching the patterns will be canceled. clear (bool): This will cancel any previous job, clear the XP folder, and reschedule a new experiment. )default_factorypatternsTmonitorintervalNtrimF trim_lastsilentdry_runcancelclearinitjupyterfolderrtail_from_commandline)__name__ __module__ __qualname____doc__rlistr tpListstr__annotations__r!boolr#floatr$Optionalintr%r&r'r(r)r*r+r,rr-r.rrrrr)s"         rcCs|jj}|dur |jd}t|}|jdur'|jddd}t|jj|}|jdus0| sg}t|jj}t |D]0\}} } | D](} t||  |} | j drl| j dsl| j| j} |t| ddqDq=|jdur| std|d|dtd d |td |d|j} t| }z|j}Wntyt| d Ynwt|tst|d |S)Nz.grids./z.py_z No grid file z in package z. Maybe you made a typo?zPotential grids are: z, rz( has no exploration function `explorer`.z' must be an instance of `dora.Explorer`)dora grid_packagepackagergridreplacer__file__parentexistsoswalk relative_tonameendswith startswithstemappendr6rjoinsysexitexplorerAttributeErrorr isinstancer )rmainr@grids grid_filename grid_file candidatespkg_rootrootfoldersfilesfilefullpath grid_namerBrRrrr _get_explore[s@        rarrUcCsZt||}|}t||t}t||t}d|_t||t|||j|||dS)NT)raget_slurm_configr r rr.run_gridrB)rrUrRslurmrules grid_argsrrr grid_actions    rgrRr`rerdreturnc" st|tsJ|dur|}|jj|jj|}|jdddt}t|t d}|j rQt d} t |||| d} || | Wdn1sKwYn t |||} || |t|j} t|j|| } |jr|jrwtdt dt| d |d |jrtd } | d krtd t d| D] } | jdur|| jq|t d| D]} | jj rt!| jjd| _qg}g}|"D]g}|j#|jvr4|$|z|%|j#}WnDt&y'}z7t d|j#d|||jj'j(}t)|}|dur|d}t d|j*d|j#d||WYd}~qd}~ww|dus/J|$|q||j+sdd| D|j,D]O}fdd|D}|sWqG|d}|j-|jj.}t|dkrr|/|||qG|0||D] } |/| ||qzWdn 1swYqG|D]-}|1s|jdusJ||j|2|j}t d|jj*d|jj.d|q|j+r| D]-} | 1s| jdusJ|2| j}t d| jj*d| jj.d||| jq|js>| D].} || jj.}| s|3r'|3r#|4| jj4ks%Jq|5| jjq||D]}|6q6|j7rN| D] } |8| jqD|j+rT| S| s]t d| Sdd|j|j |j9fD}|r|jsut:d t|dks~J|d}z| |} Wnt;ytd!|jYnw|2| j}|jdurt<| jj| S|j9dur| j std"| j d#t=>d$d$d%d&d'| j g| S| j std(|dz t?@tA| j tBjCW| StDyY| Sw|jErtF}nt<}|d)| |jGr|jEsdd*lHmI}|jJdd+|tK|||| |r. | S|jKs5 | StL|jMd,}|tN|D]#} d-|| d.d/}!|| d0krZtO|!d1}!||!d2d3tPQdqC|d4q )5a Run a grid search, this is the API underlying the `dora grid` command, so that it can be used from a notebook. You can also provide patterns to filter out XPs to be displayed. Args: main (DecoratedMain): main training function, decorated with Dora. explorer (Explorer): explorer instance that will define the XPs to launch. grid_name (str): this must be a unique name for the grid. rules (SubmitRules): see `dora.conf.SubmitRules`, those defines the rules for rescheduling failed XP etc. slurm (SlurmConfig or None): if provided, this will override the default Slurm config defined my the `main` argument. Returns: A list of `dora.shep.Sheep`. NT)exist_okparents)r)poolz'--dry_run is incompatible with --clear.zYou are about to restart z experiments from the grid z' from scratch. This cannot be reverted.zConfirm [yN]: yzAbort...zCanceling all current jobs...zDeleting XP folders...z$Error when trying to load old sheep z: rzCanceling job z from unloadable sheep r<cSsi|]}|jj|qSr)xpsig.0sheeprrr szrun_grid..csg|] }|vr|qSrr)rqro sheep_maprr szrun_grid..rz for no longer required sheep r=z for sheep zNo sheep to handle.cSsg|]}|dur|qSrr)rqactionrrrrvzIThe folder, log, and tail flags are only supported from the command line.zInvalid index zLog z does not existr-z-n200z-fz"Log file does not exist for sheep zMonitoring Grid )display)wait<zNext update in z.0fz seconds 31 )endz< )RrTr rbr?dir_gridsmkdirr rr_slowrr completeupdater3sheepsvalues_filter_grid_sheepsr r)r'rlenr.inputlowerjob cancel_lazycommitrnr,rFriterdirrJrNget_sheep_from_sig Exceptionshepjob_filerjob_idr( job_arrays slurm_configsromaybe_submit_lazy job_arrayis_doneget_name is_symlinkresolve symlink_tounlinkr*init_xpr- RuntimeError IndexErrorprintrGexecvpshutil copyfileobjopenrPstdoutBrokenPipeErrorr&rr+IPythonrz clear_outputr!r;r#rangertimesleep)"rUrRr`rerdr grid_folderherdshepherdrllauncherrreplrr to_unlink old_sheepschild old_sheeperrorrjobsrr array_sheepsfirstrJlinkactionsindex maybe_printrzrelaoutrrtrrcsJ                     "  (              rccCsh|sdS|D]+}d}|dddkr|dd}d}t|d|d}|r,|r+dSq|s1dSqdS)NTFr!*)fnmatch)rJr patternnegresultrrr _match_namePs  rr rc sg}t|D]}z |t|Wn tyYqw||qg|D]}||j}t||r7|q%|rCfdd|DS)Ncsg|]}|qSrr)rqidxrrrrvqsz'_filter_grid_sheeps..)r3rNr; ValueErrorremoverrnr)r rUrindexesprrrJrrrras"      rrrc sdd|D\}}fdd|D}d|jdur$t||jn |jr0tdd|Ddur=fdd|D}g}d} tt|||D]V\} \} } } | }| sZd } |durad }n|dd }| | | j rp| j j nd | j j |d }i}||d<z| | | }Wnty|| }Ynw||||qI|r|d|tjdtd|g|d}|tj|||d| S)zSingle iteration of monitoring of the jobs in a Grid. Returns `True` if all jobs are done or failed, and `False` otherwise. cSsg|]}|jqSr)rnrprrrrvzszmonitor..csg|]}|jqSr)get_xp_historyrnrp)rUrrrv{rxNcss|]}t|VqdSr)rrqmetricsrrr szmonitor..csg|]}|dqSrrr)r$rrrvrxTFzN/A)rJrsidrostateMetaz Base name: )shortengroups)colors) get_namesr$rr%min enumerateziprrrrrnro process_sheepNotImplementedErrorprocess_historyrrNtttablegroup get_grid_metaget_grid_metrics treetable get_colors)rrUrRrrnames base_name historieslinesfinishedrrrhistoryrJrmetalineotherrr)rUr$rr!usX      r!)7r2concurrent.futuresr dataclassesrrr functoolsrrGpathlibrtypingr4rrPrconfrr r explorer r r rUrrrrrrrrutilsrrrrrCallabler6r7rrraAnyrgr:r5rcrrr8r!rrrrs^    1&  D