o -wii@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddl mZddlmZddlmZmZmZmZmZmZmZmZddlZddlZddlZddlmZddlm Z dd l!m"Z"dd l#m$Z$dd l%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,erddl-m.m/Z/ddl0m1Z1ddl-m2Z2m3Z3ddl4m3Z5e6e7Z8ej9ddddZ:dZ;GdddeZeGdddZ?Gddde Z@dS) zAbstract Scheduler class.N)ABCabstractmethod) dataclass)Enum) TYPE_CHECKINGAnyDictIteratorListOptionalTupleUnion) CommError) launch_add) LaunchError)SchedulerError)create_sweep_command_argsmake_launch_sweep_entrypoint)event_loop_thread_exec%strip_resource_args_and_template_vars) generate_idApi) QueuedRunRun)rzsched:cyan)fg g@c@s,eZdZdZdZdZdZdZdZdZ dZ d S) SchedulerStaterN) __name__ __module__ __qualname__PENDINGSTARTINGRUNNING FLUSH_RUNS COMPLETEDFAILEDSTOPPED CANCELLEDr1r1^/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/sdk/launch/sweeps/scheduler.pyr0src@steZdZdZdZdZdZdZdZdZ dZ d Z d e d e d e d dfddZddeded dfddZed efddZdS)RunState)runningalive)pendingr5) preemptingr5)crasheddead)failedr9)killedr9)finishedr9) preemptedr9)unknownr5clsargskwdsreturncOst|}|d|_|S)Nr)object__new___value_)r?r@rAobjr1r1r2rDHs  zRunState.__new__r>_lifeNcCs ||_dSN_life)selfrGrHr1r1r2__init__Ms zRunState.__init__cCs |jdkS)Nr5rJrLr1r1r2is_aliveP zRunState.is_alive)r>)r&r'r(r+r) PREEMPTINGCRASHEDr.KILLEDFINISHED PREEMPTEDUNKNOWNrr rDstrrMpropertyboolrOr1r1r1r2r3;sr3c@s&eZdZUeeefed<eed<dS)_Worker agent_configagent_idN)r&r'r(rrWr__annotations__r1r1r1r2rZUs  rZc@sheZdZUeed<eed<ejZeed<dZ e ded<dZ e e ee fed<dZe eeed<dS) SweepRunid worker_idstateNzpublic.QueuedRun queued_runr@logs)r&r'r(rWr]intr3r+rarbr r@rrrcr r1r1r1r2r^[s r^c@seZdZdZdZdZgdZdddddddddd eed ee d ee d ee d ee dee dee e e fdeefddZ ede deefddZeddddZeddddZeddddZeddddZedefdd Zejd!eddfd"d Zedefd#d$Zedefd%d&Zede fd'd(Zedee effd)d*Zedee effd+d,Z ded.d/Z!ddd0d1Z"d2ee ddfd3d4Z#ddd5d6Z$ddd7d8Z%ddd9d:Z&d;e'ee efde fdd?Z)ddd@dAZ*de+e,e effdBdCZ-dDe'e ddfdEdFZ.dddGdHZ/dIe defdJdKZ0dddLdMZ1dddNdOZ2dIe de'efdPdQZ3dIe dee effdRdSZ4e5j6fdIe dTe5de5fdUdVZ7dee effdWdXZ8dYe ddfdZd[Z9d\e de fd]d^Z:d_ede,ee'e ee ee effffd`daZ;d_edefdbdcZsz/Scheduler.available_workers..)rritemsrNr1rNr2available_workerss  zScheduler.available_workersSdkRuncCs,tjdd}tjd|jd|j|d}|S)z8Controls resume or init logic for a scheduler wandb run.T)disable_job_creationz Scheduler.allow)rresumersrv)rfSettingsinitrr)rLrvrunr1r1r2rs  zScheduler._init_wandb_runcCs tj|_dS)zStop the sweep.N)rr/rrNr1r1r2 stop_sweep s zScheduler.stop_sweeperrcCstj|_|r t|dS)z%Fail the sweep w/ optional exception.N)rr.rr)rLrr1r1r2 fail_sweepszScheduler.fail_sweepcCsttd|js ttd|jjd|dSt j |_ | s6ttd|dS| t||dS)zAStart a scheduler, confirms prerequisites, begins execution loop.zScheduler starting.zSweep already in end state (z ). Exiting...Nz1No 'job' or 'image_uri' loaded from sweep config.)rfrrrO termerrorrarlowerexitrr*r_try_load_executablerasyncior_register_agentsrNr1r1r2starts" zScheduler.startc Csttdtj|_z ||jsnttd|| |jtj krA|j dkr:ttdnrt |jq |jD]`}|jr[ttd|jdtj |_nJz ||}|sfWn?Wn0tyw}zt|d}~wty}zttd |tj|_WYd}~nd}~ww||r|jd 7_qDt |jqWn8tyttd tj|_|YdSty}zttd |tj|_|d}~ww|jtj kr|jrtj|_|dS) zMain run function.zScheduler runningTzPolling for new runs to launchrzDone polling on runs, exitingzSweep at run_cap ()NzFailed to get next sweep run: rz-Scheduler received KeyboardInterrupt. Exitingz Scheduler failed with exception )rfrrrr+ra_update_scheduler_run_staterO_update_run_statesrr,rtimesleeprrrrrrrrr._add_to_launch_queueKeyboardInterrupttermwarnr/rr-)rLr`rrr1r1r2r.sx         ) z Scheduler.runc Cs|z|WntyttdtYnwd}|jt j kr0| dd}n2|jt j kr>| dd}n$|jt j t jfvrS| dd}|nt j|_| d d }|ttd ||jdS) NzFailed to save state: PAUSEDpausedrT completedCANCELED cancelledrRr8z Scheduler )rrrrfrr traceback format_excrarr,_set_sweep_stater-r0r/ _stop_runsr.rrfinish)rLstatusr1r1r2rls2         zScheduler.exitrtc CsZd}|D]&}|dddvr&|ds&td|dd|dd |q|d 7}q|S) z.Returns the number of valid runs in the sweep.rrar)r;r8summaryMetricszexcluding run: rz with state: z from run cap r)r|rr)rLrtcountrr1r1r2rs z Scheduler._get_num_runs_launchedcCs||jdr4z|j|jd}ttd|jdWdSty3t tt YdSw|jdr|j|jEdHWddS1swYdS)z)Thread-safe way to iterate over the runs.N)rrrrNr1r1r2rs"zScheduler._yield_runsruns_to_removecCsT|j|D]}ttd|d|j|=qWddS1s#wYdS)zHelper for removing runs from memory. Can be overloaded to prevent deletion of runs, which is useful for debugging or when polling on completed runs. zCleaning up finished run (rN)rrfrrr)rLrrun_idr1r1r2 _cleanup_runss  "zScheduler._cleanup_runscCsdg}|D] \}}||g7}q|D]}ttd|d||s/ttd|dqdS)NzStopping run (rzFailed to stop run ()rrfrr _stop_runr)rL to_deleterrGr1r1r2rs  zScheduler._stop_runsrc Cs||jvrtd|d|jdS|j|}|j|=|js+td|jddS|jjs1dStd|d|j d|j   d }z|j j|d }|r^ttd |d WdSWdSty~}ztd |d|WYd}~dSd}~ww)z.Stops a run and removes it from the scheduler.zrun: z not in _runs: Fz2tried to _stop_run but run not queued yet (run_id:rTzRun:v1::utf-8)rz Stopped run .zerror stopping run (): N)rrrrbr_rarObase64standard_b64encoderr~encodedecoderxstop_runrfrrr)rLrrencoded_run_idsuccessrr1r1r2rs:    zScheduler._stop_runc Cs||jj}|tjkrtj|_n|tjtj fvrtj|_n |tj kr'tj |_z |j |j|j|j}WntyO}ztd|WYd}~dSd}~ww|dkrZtj |_dS|dvrdtj|_dS|dkrntj|_dSdS)zGUpdate the scheduler state from state of scheduler run and sweep state.zsweep state error: NrT)r0r/r)_get_run_staterr_r3rSrr/rar.rRrTr-rxget_sweep_staterr~rrrrr0r,)rLra sweep_staterr1r1r2rs.        z%Scheduler._update_scheduler_run_statec Csg}|D]S\}}|||j|_z |jr|jjnd}Wnttfy;}ztd|d}WYd}~nd}~ww|jjrD|dkrYtd|d|jd|d| |q| |dS)ztIterate through runs. Get state from backend and deletes runs if not in running state. Threadsafe. Nz Failed to get queued_run.state: r:(z ) states: (z, r) rrrarbrrrrrOrr)rLrrr rqi_staterr1r1r2rs  zScheduler._update_run_statesc sz6|j|j}|s gWS|j|jd|jd|}|jdd|jdgd}fdd|D}|WStyQ}zt d|WYd }~gSd }~ww) zUse the public api to get metrics from a run. Uses the metric name found in the sweep config, any misspellings will result in an empty list. /metricr_step)keyscsg|]}|qSr1r1rx metric_namer1r2 Csz3Scheduler._get_metrics_from_run..z[_get_metrics_from_run] N) rrbryrrkrlr scan_historyrrr)rLrrbapi_runhistorymetricsrr1rr2_get_metrics_from_run3s  zScheduler._get_metrics_from_runc Cs\z|j|j|j|}|r|WSWiSty-}ztd|WYd}~iSd}~ww)z+Use the public api to get info about a run.z[_get_run_info] N)rx get_run_infor~rrrr)rLrinforr1r1r2 _get_run_infoJs zScheduler._get_run_infoprev_run_statec Csd}z|j|j|j|}t|}W|StyQ}z2td|d||tjkr.z*Sweep command contains unresolved macros: z', see launch docs for supported macros.)rr@rrr|rrindexcopydeepcopyrrsupdaterfr)rLrr@ entry_point macro_argspidx launch_config unresolvedr1r1r2_make_entry_and_launch_configs8        z'Scheduler._make_entry_and_launch_configc Csz|jdp |jd}|jd}|jdp|}|dur-|dur-ttd|jd|dur<|dur   "   " #re)Ar>rrr(loggingrzrrrrabcrr dataclassesrenumrtypingrrrr r r r r clickrrf wandb.errorsrwandb.sdk.launch._launch_addrwandb.sdk.launch.errorsrwandb.sdk.launch.sweepsrwandb.sdk.launch.sweeps.utilsrrwandb.sdk.launch.utilsrrwandb.sdk.lib.runidrrwapispublicwandb.apis.internalrrrwandb.sdk.wandb_runr getLoggerr&rstylerrrr3rZr^rer1r1r1r2sN  (