o
    -wiha                     @   s2  d Z ddlZddlZddlZddlZddlZddlmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ed	d
dZejjZdejej fddZG dd de Z!G dd dZ"de	ej# dee$e%f fddZ&dd Z'de(dee(ej)f de	ej# de$fddZ*dS )a9  Sweep controller.

This module implements the sweep controller.

On error an exception is raised:
    ControllerError

Example:
    import wandb

    #
    # create a sweep controller
    #
    # There are three different ways sweeps can be created:
    # (1) create with sweep id from `wandb sweep` command
    sweep_id = 'xyzxyz2'
    tuner = wandb.controller(sweep_id)
    # (2) create with sweep config
    sweep_config = {}
    tuner = wandb.controller()
    tuner.configure(sweep_config)
    tuner.create()
    # (3) create by constructing programmatic sweep configuration
    tuner = wandb.controller()
    tuner.configure_search('random')
    tuner.configure_program('train-dummy.py')
    tuner.configure_parameter('param1', values=[1,2,3])
    tuner.configure_parameter('param2', values=[1,2,3])
    tuner.configure_controller(type="local")
    tuner.create()
    #
    # run the sweep controller
    #
    # There are three different ways sweeps can be executed:
    # (1) run to completion
    tuner.run()
    # (2) run in a simple loop
    while not tuner.done():
        tuner.step()
        tuner.print_status()
    # (3) run in a more complex loop
    while not tuner.done():
        params = tuner.search()
        tuner.schedule(params)
        runs = tuner.stopping()
        if runs:
            tuner.stop_runs(runs)
    N)CallableDictListOptionalTupleUnionenv)InternalApi)wandb_sweep)handle_sweep_config_violations0sweep_config_err_text_from_jsonschema_violations)
get_modulesweepsz^wandb[sweeps] is required to use the local controller. Please run `pip install wandb[sweeps]`.)required
   c                    s   d  fddt| D S )N c                 3   s    | ]}t  V  qd S N)randomchoice).0_chars S/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/wandb_controller.py	<genexpr>U   s    z _id_generator.<locals>.<genexpr>)joinrange)sizer   r   r   r   _id_generatorT   s   r    c                   @   s   e Zd ZdZdS )ControllerErrorzBase class for sweep errors.N)__name__
__module____qualname____doc__r   r   r   r   r!   X   s    r!   c                   @   s  e Zd ZdZd_ddZdeeeeee	j
f ee	j gee	j f f fddZdeeeeee	j
f ee	j gee	j f f fd	d
Zd`ddZdd Zdd Zdd Z										daddZdd Zdd Zedeee	j
f fddZedefddZdbddZd eddfd!d"Zd eddfd#d$Zd eddfd%d&Zd eddfd'd(Zdbd)d*Zd+e defd,d-Z!dcd/e"defd0d1Z#	.	2	.	.ddd3e"d4e"d5e"d6e"ddf
d7d8Z$dee fd9d:Z%dbd;d<Z&dbd=d>Z'd?d@ Z(dbdAdBZ)dbdCdDZ*de"fdEdFZ+dee	j fdGdHZ,dee	j fdIdJZ-dee	j fdKdLZ.dee	j fdMdNZ/dOee	j ddfdPdQZ0dRee	j ddfdSdTZ1dbdUdVZ2dbdWdXZ3dbdYdZZ4dbd[d\Z5dbd]d^Z6dS )e_WandbControlleraA  Sweep controller class.

    Internal datastructures on the sweep object to coordinate local controller with
    cloud controller.

    Data structures:
        controller: {
            schedule: [
                { id: SCHEDULE_ID
                  data: {param1: val1, param2: val2}},
            ]
            earlystop: [RUN_ID, ...]
        scheduler:
            scheduled: [
                { id: SCHEDULE_ID
                  runid: RUN_ID},
            ]

    `controller` is only updated by the client
    `scheduler` is only updated by the cloud backend

    Protocols:
        Scheduling a run:
        - client controller adds a schedule entry on the controller.schedule list
        - cloud backend notices the new entry and creates a run with the parameters
        - cloud backend adds a scheduled entry on the scheduler.scheduled list
        - client controller notices that the run has been scheduled and removes it from
          controller.schedule list

    Current implementation details:
        - Runs are only schedule if there are no other runs scheduled.

    Nc                 C   s~  d | _ i | _d | _d | _d | _d | _d | _d | _d | _d | _	d | _
d | _d | _d| _d| _d| _d| _d| _g | _g | _tj}|rHtj||d |rQtj||d t|d| _t|tr`|| _ nNt|tskt|tjrt|| _t ddgdd	gD ]\}}t!|| jv o| j| rt"| || j|  d
| j|< qz| j#dd| _ n|d u rd| _d S t$d| % }|d u rt$d|| _d S )NFr   r   r   )environmethodearly_terminate_custom_search_custom_stoppingcustomT)	from_dictUnhandled sweep controller typezCan not find sweep)&	_sweep_id_creater*   r+   _program_function
_sweep_obj_sweep_config_sweep_metric_sweep_runs_sweep_runs_map
_scheduler_controller_controller_prev_step_started_done_scheduling_defer_sweep_creation_logged_laststatus_log_actions
_log_debugosr'   r	   
set_entityset_projectr
   _api
isinstancestrdictr   SweepConfigzipcallablesetattrcreater!   _sweep_object_read_from_backend)selfsweep_id_or_configentityprojectr'   
config_keycontroller_attr	sweep_objr   r   r   __init__   sd   


z_WandbController.__init__searchc                 C   sD   |    t|tr|| jd< d S t|rd| jd< || _d S td)Nr(   r,   zUnhandled search type.)_configure_checkrE   rF   r0   rJ   r*   r!   )rN   rV   r   r   r   configure_search   s   



z!_WandbController.configure_searchstoppingc                 K   s   |    t|tr)| jdi  || jd d< | D ]\}}|| jd |< qd S t|rB||| _| jdi  d| jd d< d S td)Nr)   typer,   zUnhandled stopping type.)	rW   rE   rF   r0   
setdefaultitemsrJ   r+   r!   )rN   rY   kwargskvr   r   r   configure_stopping   s   

z#_WandbController.configure_stoppingc                 C   s>   |    | jdi  || jd d< |r|| jd d< d S d S )NmetricnamegoalrW   r0   r[   )rN   ra   rc   r   r   r   configure_metric  s   z!_WandbController.configure_metricc                 C   sH   |    t|tr|| jd< d S t|r d| jd< || _tdtd)Nprogram__callable__z'Program functions are not supported yetzUnhandled sweep program type)rW   rE   rF   r0   rJ   r1   r!   )rN   rf   r   r   r   configure_program  s   

z"_WandbController.configure_programc                 C      |    || jd< d S )Nrb   rW   r0   )rN   rb   r   r   r   configure_name     z_WandbController.configure_namec                 C   ri   )Ndescriptionrj   )rN   rm   r   r   r   configure_description  rl   z&_WandbController.configure_descriptionc                 C   sJ  |    | jdi |i  |d us#|d u r,|d u r,|d u r,|d u r,|| jd | d< |d ur9|| jd | d< |d urF|| jd | d< |d urS|| jd | d< |d ur`|| jd | d< |d urm|| jd | d< |d urz|| jd | d< |	d ur|	| jd | d	< |
d ur|
| jd | d
< |d ur|| jd | d< d S d S )N
parametersvaluevaluesdistributionminmaxmusigmaqabrd   )rN   rb   rq   rp   rr   rs   rt   ru   rv   rw   rx   ry   r   r   r   configure_parameter   s0    z$_WandbController.configure_parameterc                 C   s,   |    | jdi  | jd d| dS )z1Configure controller to local if type == 'local'.
controllerrZ   Nrd   )rN   rZ   r   r   r   configure_controllerG  s   z%_WandbController.configure_controllerc                 C   sL   |    | jrtdt|tr|| _d S t|tr"t|| _d S td)NzAlready configured.r.   )rW   r0   r!   rE   rG   rF   yaml	safe_load)rN   sweep_dict_or_configr   r   r   	configureM  s   


z_WandbController.configurereturnc                 C      | j S r   )r3   rN   r   r   r   sweep_configX     z_WandbController.sweep_configc                 C   r   r   )r/   r   r   r   r   sweep_id\  r   z_WandbController.sweep_idc                 C   s   |  j d7  _ d S )N   )r=   r   r   r   r   _log`  s   z_WandbController._logsc                 C      t d| |   d S )NzERROR:printr   rN   r   r   r   r   _errorc     
z_WandbController._errorc                 C   r   )NzWARN:r   r   r   r   r   _warng  r   z_WandbController._warnc                 C   r   )NzINFO:r   r   r   r   r   _infok  r   z_WandbController._infoc                 C   r   )NzDEBUG:r   r   r   r   r   _debugo  r   z_WandbController._debugc                 C   s   | j rtdd S )Nz/Can not configure after sweep has been started.)r:   r!   r   r   r   r   rW   s  s   z!_WandbController._configure_checkconfigc                 C   s*   t |}t|dkrt|}|S d}|S )Nr   r   )r   &schema_violations_from_proposed_configlenr   )rN   r   
violationsmsgr   r   r   	_validatew  s   
z_WandbController._validateFr-   c                 C   s   | j rtd| js|std| jstdt| j| _| j| j\}}t| t	d| t
| j|}|r>t	d| || _d| _|S )Nz,Can not create after sweep has been started.z,Can not use create on already created sweep.z#Must configure sweep before create.zCreate sweep with ID:z
Sweep URL:F)r:   r!   r<   r0   r   rH   rD   upsert_sweepr   r   r   _get_sweep_urlr/   )rN   r-   r   warnings	sweep_urlr   r   r   rL     s    


z_WandbController.createTverboseprint_statusprint_actionsprint_debugc                 C   sf   |rd}d}d}|    |  s1|r|   |   |r |   |r&|   td |  rd S d S )NT   )_start_if_not_starteddoner   stepr   r   timesleep)rN   r   r   r   r   r   r   r   run  s   
z_WandbController.runc           
      C   s  i }| j rdg}|| j  |dd}t|}| j| j|}|s$d S || _t	|d | _
| j
di d| _ g }|d D ]h}| }d|v rW|d rWt|d |d< d|vr_td	t|d |d< d
|v rt|d
 trdd |d
 D |d
< n	td|d
 d|v rg }|d D ]}	||	7 }q||d< |tjdi | q@|| _dd | jD | _t|dpd| _t|dpd| _| j | _|S )N_stepi )keyssamplesr   ra   rb   runssummaryMetricszsweep object is missing confighistoryc                 S   s   g | ]}t |qS r   )jsonloads)r   dr   r   r   
<listcomp>      zD_WandbController._sweep_object_read_from_backend.<locals>.<listcomp>z8Invalid history value: expected list of json strings: {}sampledHistoryc                 S   s   i | ]}|j |qS r   rb   r   rr   r   r   
<dictcomp>      zD_WandbController._sweep_object_read_from_backend.<locals>.<dictcomp>r{   z{}	schedulerr   )r4   appendr   dumpsrD   sweepr/   r2   r}   r~   r3   getcopyr   
ValueErrorrE   listformatr   SweepRunr5   r6   r8   r7   r9   )
rN   
specs_jsonr^   specsrT   r5   r   rrsampled_historyhistoryDictListr   r   r   rM     sR   


z0_WandbController._sweep_object_read_from_backendc                 C   sV   | j | jkrd S | jd }t| j }| jj| j||d\}}t| | j 	 | _d S )Nid)r{   obj_id)
r8   r9   r2   r   r   rD   r   r3   r   r   )rN   sweep_obj_idr{   r   r   r   r   r   _sweep_object_sync_to_backend  s   

z._WandbController._sweep_object_sync_to_backendc                 C   sd   | j rd S | jrtd|  }|sd S | jdi ddk}|s&tdd| _ i | _|   d S )Nz9Must specify or create a sweep before running controller.r{   rZ   localz<Only sweeps with a local controller are currently supported.T)r:   r<   r!   rM   r3   r   r8   r   )rN   objis_localr   r   r   r     s"   z&_WandbController._start_if_not_startedc           
      C   s   | j dpg }g }g }g }|D ]7}|d}|d}| j|}|s%q|jr-|| |j}	|jtkr8|	s8q|| |jdkrG|| q|||fS )N	scheduledrunidr   running)r7   r   r6   stoppedr   summary_metricsstateSWEEP_INITIAL_RUN_STATE)
rN   scheduled_liststarted_idsstopped_runs	done_runsr   r   objidr   summaryr   r   r   _parse_scheduled  s(   






z!_WandbController._parse_scheduledc                    s   |    |   |  \ | jdg }fdd|D }|| jd< | jdg } fdd|D }|| jd< g | _g | _d S )Nschedulec                    s   g | ]}| d  vr|qS )r   )r   r   r   )r   r   r   r         z*_WandbController._step.<locals>.<listcomp>	earlystopc                    s    g | ]}|vr| vr|qS r   r   r   )r   r   r   r   r      s    )r   rM   r   r8   r   r?   r@   )rN   schedule_listnew_schedule_listearlystop_listnew_earlystop_listr   )r   r   r   r   r     s   


z_WandbController._stepc                 C   s@   |    |  }| | |  }t|dkr| | d S d S Nr   )r   rV   r   rY   r   	stop_runs)rN   
suggestionto_stopr   r   r   r   )  s   
z_WandbController.stepc                 C   sB   |    | jd}|dd tjjjtjtjjjfD v rdS dS )Nr   c                 S   s   g | ]}|  qS r   )upperr   r   r   r   r   4  s    z)_WandbController.done.<locals>.<listcomp>FT)	r   r2   r   r   RunState
preemptingrp   r   r   )rN   r   r   r   r   r   1  s   z_WandbController.donec                 C   s0   | j ptj}|| j| jpg }|d u rd| _|S )NT)r*   r   next_runr3   r5   r;   )rN   rV   r   r   r   r   _search?  s
   z_WandbController._searchc                 C   s   |    |  }|S r   )r   r   )rN   r   r   r   r   rV   F  s   z_WandbController.searchc                 C   sP   d| j vrg S | jptj}|| j| jpg }dd |D }|r&|  j|7  _|S )Nr)   c                 S   s0   g | ]}|j d urddd |j  D qS )N c                 S   s   g | ]\}}| d | qS )=r   r   r^   r_   r   r   r   r   R  r   z9_WandbController._stopping.<locals>.<listcomp>.<listcomp>)early_terminate_infor   r\   r   r   r   r   r   r   Q  s
    
z._WandbController._stopping.<locals>.<listcomp>)r   r+   r   r   r3   r5   r@   )rN   stopperr   debug_linesr   r   r   	_stoppingK  s   
z_WandbController._stoppingc                 C   s   |    |  S r   )r   r   r   r   r   r   rY   [  s   z_WandbController.stoppingr   c                 C   s   |    | jr| jdrd S t }|d u r|dd idg}n dd t|j D }| jdd	|f |d|jidg}|| jd< | 
  d S )Nr   args)r   datac                 S   s"   g | ]\}}d  ||dqS )z{}={}rp   )r   r   r   r   r   r   r   k  s    z-_WandbController.schedule.<locals>.<listcomp>,)r   r8   r   r    sortedr   r\   r?   r   r   r   )rN   r   schedule_idr   
param_listr   r   r   r   _  s   
z_WandbController.scheduler   c                 C   s>   t dd |D }| jdd|f || jd< |   d S )Nc                 S   s   h | ]}|j qS r   r   r   r   r   r   	<setcomp>w  s    z-_WandbController.stop_runs.<locals>.<setcomp>stopr   r   )r   r?   r   r   r8   r   )rN   r   r   r   r   r   r   v  s   
z_WandbController.stop_runsc                 C   s:   t | j| j| j}| j|ks| jrt| || _d| _d S r   )_sweep_statusr2   r3   r5   r>   r=   r   )rN   statusr   r   r   r   |  s
   
z_WandbController.print_statusc                 C   s4   | j D ]\}}| |  d| d qg | _ d S )N ())r?   r   
capitalize)rN   actionliner   r   r   r     s   
z_WandbController.print_actionsc                 C   s    | j D ]}| | qg | _ d S r   )r@   r   )rN   r  r   r   r   r     s   

z_WandbController.print_debugc                 C      |  d d S NzMethod not implemented yet.r   r   r   r   r   print_space     z_WandbController.print_spacec                 C   r  r  r  r   r   r   r   print_summary  r	  z_WandbController.print_summary)NNNr   )
NNNNNNNNNN)r   N)F)FTFF)7r"   r#   r$   r%   rU   r   rF   r   rG   r   rH   r   r   r   rX   r`   re   rh   rk   rn   rz   r|   r   propertyr   r   r   r   r   r   r   rW   r   r   boolrL   r   rM   r   r   r   r   r   r   r   rV   r   rY   r   r   r   r   r   r  r
  r   r   r   r   r&   \   s    
"\



'

	

0






r&   r   r   c                 C   sn   i }dd t jj D dg }| D ]"}|j}d}|D ]
}||kr%|} nq||d ||  d7  < q|S )Nc                 S      g | ]\}}|qS r   r   r   rb   r   r   r   r   r     r   z#_get_run_counts.<locals>.<listcomp>unknownr   r   )r   r   __members__r\   r   r[   )r   metrics
categoriesr   r   foundcr   r   r   _get_run_counts  s   r  c                 C   s`   dd t jj D dg }g }|D ]}| |sq||  d| |   qd|}|S )Nc                 S   r  r   r   r  r   r   r   r     r   z$_get_runs_status.<locals>.<listcomp>r  z: z, )r   r   r  r\   r   r   r  r   )r  r  mlistr  r   r   r   r   _get_runs_status  s   

r  rT   
sweep_conf
sweep_runsc                 C   s"  | d }| d }t |}t|}t dd |D }t dd |D }d}	|s(|r7d| }	|r7|	d| d	7 }	t|}
|d
d}|dd }g }|| |rY||dd d|}g }|d| d| d	 |
r{|d| d|
 d	 n|d|  |	r||	 d|}|S )Nrb   r   c                 S      g | ]}|j r|qS r   )r   r   r   r   r   r     r   z!_sweep_status.<locals>.<listcomp>c                 S   r  r   )should_stopr   r   r   r   r     r   r   z	Stopped: z (Stopping: r  r(   r  r)   rZ   r   zSweep: r   zRuns: z | )r   r  r  r   r   r   )rT   r  r  r   r   	run_countrun_type_countsr   rY   stopstrruns_statusr(   sweep_optionssectionsr   r   r   r     s8   




r   )+r%   r   rA   r   stringr   typingr   r   r   r   r   r   r}   wandbr	   
wandb.apisr
   	wandb.sdkr   wandb.sdk.launch.sweeps.utilsr   r   
wandb.utilr   r   r   pendingr   ascii_lowercasedigitsr    	Exceptionr!   r&   r   rF   intr  r  rG   rH   r   r   r   r   r   <module>   sH    1      <