o
    -wiQ                     @   sl  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZmZ d dlZd dlZd dlmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ e  e!Z"G dd	 d	e#Z$G d
d dZ%G dd dZ&G dd dZ'	dddZ(				dde)dee dee) dee) dee* ddfddZ+d a,dd Z-dS )    N)AnyCallableDictListOptional)util	wandb_lib	wandb_sdk)pyagent)InternalApi)utils)ipythonc                   @   s   e Zd ZdS )
AgentErrorN)__name__
__module____qualname__ r   r   N/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/wandb_agent.pyr      s    r   c                   @   sD   e Zd ZdZ	dddZdd Zdd Zd	d
 Zdd Zdd Z	dS )AgentProcesszLaunch and manage a process.Nc                 C   s   d | _ d | _t | _d| _|r?t dkrtt	j
d}nttjd}|tjjr1|tjj t	j|fd|i|| _ d S |rWtj| j| j||||fd| _| j  d S td)NFWindows)creationflags)
preexec_fnenv)targetargsz*Agent Process requires command or function)_popen_procmultiprocessingQueue_finished_q_proc_killedplatformsystemdict
subprocessCREATE_NEW_PROCESS_GROUPossetpgrpgetwandbr   SERVICEpopPopenProcess_startstartr   )selfr   commandfunctionrun_id
in_jupyterkwargsr   r   r   __init__"   s$   
zAgentProcess.__init__c           	      C   sl   |r|  D ]	\}}|tj|< qtd|  |r|  td| d tj}|r/t  |d d S )NzAgent Started Run: zAgent Finished Run: 
T)itemsr&   environr)   termlogrunjoinput)	r0   
finished_qr   r2   r3   r4   kvr;   r   r   r   r.   ;   s   zAgentProcess._startc                 C   sZ   | j r| j  S | jr| j  dS z| jdd}|rW dS W d S  tjy,   Y d S w )NTFr   )	r   pollr    r   r<   r   r(   queueEmpty)r0   finishedr   r   r   rA   N   s   

zAgentProcess.pollc                 C   sJ   | j r t dkr	 | j  }|d ur|S td q
| j  S | j S )Nr   T   )	r   r!   r"   rA   timesleepwaitr   r<   )r0   pr   r   r   rH   ]   s   



zAgentProcess.waitc                 C   s8   | j r| j  S | jj}|rt|tj}d| _|S d S )NT)r   killr   pidr&   signalSIGKILLr    )r0   rK   retr   r   r   rJ   i   s   
zAgentProcess.killc                 C   s4   | j rt dkr| j tjS | j  S | j S )Nr   )r   r!   r"   send_signalrL   CTRL_C_EVENT	terminater   r0   r   r   r   rQ   s   s
   

zAgentProcess.terminateNNNNN)
r   r   r   __doc__r6   r.   rA   rH   rJ   rQ   r   r   r   r   r      s    

r   c                   @   s   e Zd ZU dZdZdZdZdZdZg dZ	e
e ed< edZ		dd
dZdd Zdd Zdd Zdd Zdd Zdd Zdd Zd	S )Agent   r      <      )z${env}z${interpreter}z
${program}z${args}DEFAULT_SWEEP_COMMANDz\$\{envvar\:([A-Z0-9_]*)\}Nc                 C   s   || _ || _i | _g | _|| _|| _g | _d| _d | _|| _	t
j| j| _t
j| j| _d| _d| _|| _g | _t
j| j| _| jd u rKtd| jd u rTtdtjdd u ritjt tjd< d S d S )NTr   zInvalid agent report intervalzInvalid agent kill delay	WANDB_DIR) _api_queue_run_processes_server_responses	_sweep_id_in_jupyter_log_running_last_report_time	_functionr)   r   get_agent_report_intervalREPORT_INTERVAL_report_intervalget_agent_kill_delay
KILL_DELAY_kill_delay	_finished_failed_count_sweep_commandget_agent_max_initial_failuresMAX_INITIAL_FAILURES_max_initial_failuresr   r&   r9   r(   pathabspathgetcwd)r0   apirB   sweep_idr2   r4   countr   r   r   r6      s8   

zAgent.__init__c                 C   s:   t tjjdkrdS t tj| j k r| j| j	kS dS )zDetermine if the process is flapping.

        Flapping occurs if the agents receives FLAPPING_MAX_FAILURES non-0 exit codes in
        the first FLAPPING_MAX_SECONDS.
        trueFN)
r&   getenvr)   r   AGENT_DISABLE_FLAPPINGrF   
START_TIMEFLAPPING_MAX_SECONDSrm   FLAPPING_MAX_FAILURESrR   r   r   r   is_flapping   s
   zAgent.is_flappingc                 C   s   | j | jko| j| j kS N)rm   rl   rr   rR   r   r   r   
is_failing   s   
zAgent.is_failingc                 C   sP  | j | jd}|r'|d}|r't|}|r'|d}|r't|tr'|| _| j j	t
 | jd}|d }zz| jr4t| jd| j}|D ]}|d | | qHt }	| jd u sk| jdkry|	| j| j krytd	t| j  |	| _i }
t| j D ]\}}| }|d u rd
|
|< qt|tst|tr|dkr|  jd7  _|  rt d| j!| j" td d| _ nE| # rt d| j$ td d| _ n0td| t%t&drd}t|tr|}nt|trd}t&'| | j|= d | _|  j(d7  _(q| j)r| j(| j)ks| jsd| _q9| j *|i |
}g | _+|D ]}| j+,| | q$| js=W n+ t-y`   zt&.d | j D ]	\}}|/  qHW n
 t-y]   Y nw Y nw W z7| j0slt&.d | j D ]\}}z|1  W qq t2y   Y qqw | j D ]	\}}|/  qW d S  t-y   t&.d | j D ]\}}z|3  W q t2y   Y qw Y d S w z6| j0st&.d | j D ]\}}z|1  W q t2y   Y qw | j D ]	\}}|/  qW w  t-y'   t&.d | j D ]\}}z|3  W q t2y$   Y qw Y w w )Nz{}configr1   )rw   idd   
resp_queuer   zRunning runs: %sTrE   z?Detected %i failed runs in the first %i seconds, shutting down.z;To disable this check set WANDB_AGENT_DISABLE_FLAPPING=trueFz0Detected %i failed runs in a row, shutting down.z=To change this value set WANDB_AGENT_MAX_INITIAL_FAILURES=valzCleaning up finished run: %steardownzNCtrl-c pressed. Waiting for runs to end. Press ctrl-c again to terminate them.z3Terminating and syncing runs. Press ctrl-c to kill.zKilling runs and quitting.)4r\   sweepr`   r(   yaml	safe_load
isinstancelistro   register_agentsocketgethostnamerc   r   read_many_from_queuer]   POLL_INTERVALr=   _process_commandstopwatch_nowrd   rh   loggerinfor^   keysr8   rA   boolintrm   r   errorr~   r}   r   rr   hasattrr)   r   rl   rn   agent_heartbeatr_   appendKeyboardInterruptr:   rH   ra   rQ   OSErrorrJ   )r0   	sweep_obj
sweep_yamlsweep_configsweep_commandagentagent_idcommandsr1   now
run_statusr3   run_processpoll_result	exit_code_r   r   r   r;      s  






	


O




z	Agent.runc                 C   s
  t dd|v r|d nd  |dd d}z9|d }|dkr&| |}n%|dkr0| |}n|dkr:| |}n|d	krD| |}ntd
| ||d< W n* tyz   t d| t	
 \}}}|j dt| |d< t||d< ~Y nw | j||f |S )NzAgent received command: %stypeUnknownr   )r   resultr;   stopexitresumezNo such command: r   z&Exception while processing command: %s: 	exception	traceback)r   r   r(   _command_run_command_stop_command_exitr   	Exceptionr   sysexc_infor   strr   	format_tbrb   r   )r0   r1   responsecommand_typer   ex_typeextbr   r   r   r   8  s:   zAgent._process_commandc                 C   sf  t dddd |d  D   | jr0td|d ddd	d |d  D   t	| j
}|d}tjtjj}tjd
d| d| d }tjd
d| d| d }|tjtjj< tjtjjd}tj||}|tjtjj< tj||d  ttj}	t|}
d|v rt|d}||
d d  W d    n1 sw   Y  | jrt  t| j|	|| jd}n_dg|
d< |d g|
d< |g|
d< t dksdg|
d< g }|D ](}t|}| dr|!dr|
|dd }||pg 7 }q||g7 }qt d "d!d"d# |D  t||	d$}|| j#|< d | j#| _$d | _%d S )%Nz Agent starting run with config:
r7   c                 S   s    g | ]\}}d  ||d qS )z	{}: {}value)format.0r?   r@   r   r   r   
<listcomp>]  s     z&Agent._command_run.<locals>.<listcomp>r   zAgent Starting Run: r3   z with config:
c                 S   s$   g | ]\}}d | d|d  qS )	r   r   r   r   r   r   r   r   d  s   $ r)   zsweep-zconfig-z.yamlz.json z${args_json_file}w	args_jsonr   )r2   r   r3   r4   pythoninterpreterprogramargs_json_filer   z/usr/bin/envr   z${}   r   zAbout to run command: {} c                 s   s(    | ]}d |v rd| dn|V  qdS )r   "Nr   )r   cr   r   r   	<genexpr>  s   & z%Agent._command_run.<locals>.<genexpr>)r1   r   )&r   r   r<   r8   ra   r)   r:   r(   sweep_utilscreate_sweep_commandro   r&   r9   r   SWEEP_IDrs   RUN_IDDIRSWEEP_PARAM_PATHr   config_utilsave_config_file_from_dictr#   create_sweep_command_argsopenwritere   r   r   r!   r"   r   
startswithendswithr   r^   last_sigterm_timerd   )r0   r1   r   r3   rw   config_file	json_filebase_dirsweep_param_pathr   
sweep_varsfpproccommand_listr   replace_listr   r   r   r   Y  s   







zAgent._command_runc                 C   s   |d }|| j v rT| j | }t }|jd u r2||_td| z|  W d S  ty1   Y d S w ||j| j krRtd| z|	  W d S  tyQ   Y d S w d S t
d| d S )Nr3   zStop: %szKill: %szRun %s not running)r^   r   r   r   r   r   rQ   r   rk   rJ   r   )r0   r1   r3   r   r   r   r   r   r     s*   


zAgent._command_stopc              	   C   sF   t d | j D ]\}}z|  W q
 ty   Y q
w d| _d S )Nz1Received exit command. Killing runs and quitting.F)r   r   r^   r8   rJ   r   rc   )r0   r1   r   r   r   r   r   r     s   

zAgent._command_exitNNNN)r   r   r   r   rg   rj   r}   r~   rq   rZ   r   r   __annotations__recompileSWEEP_COMMAND_ENV_VAR_REGEXr6   r   r   r;   r   r   r   r   r   r   r   r   rU   |   s$   
 

 |!QrU   c                   @   s   e Zd Zdd Zdd ZdS )AgentApic                 C   s   || _ d| _t | _d S )Nr   )r]   _command_idr   Manager_multiproc_manager)r0   rB   r   r   r   r6     s   zAgentApi.__init__c                 C   s   d|d< d| j  |d< |  j d7  _ | j }||d< | j| | }td| d|v rFtd	 |d
 D ]}t|  q7t|d  |S )Nlocaloriginzlocal-r   rE   r   zresult:r   z(Exception occurred while running commandr   )r   r   r   r]   r=   r(   printstrip)r0   r1   r   r   liner   r   r   r1     s   

zAgentApi.commandN)r   r   r   r6   r1   r   r   r   r   r     s    r   c              	   C   s  t ||| d}t|}|rt| d S |dp|}|dp"|}|dp)| } |r2tj| |r:tj| | rC| t	j
tjj< ttj t }tj}	|rUtj}	||	 td}
||
 z"t| t }t }t||| |||d}|  W t| d S t| w )N)entityprojectnamer   r   r   z4%(asctime)s - %(name)s - %(levelname)s - %(message)s)rw   r2   r4   rx   )r#   r   parse_sweep_idr)   	termerrorr(   r   
set_entityset_projectr&   r9   r   r   setLevelloggingDEBUGStreamHandlerERROR	FormattersetFormatter
addHandlerr   r   r   rU   r;   removeHandler)rw   r2   r4   r   r   rx   partserrch	log_level	formatterrv   rB   r   r   r   r   	run_agent  sL   





r  rw   r2   r   r   rx   returnc                 C   sb   t d7 a z'tjjdd |rt| ||||W t d8 a S t| |t |||dW t d8 a S t d8 a w )a  Start one or more sweep agents.

    The sweep agent uses the `sweep_id` to know which sweep it
    is a part of, what function to execute, and (optionally) how
    many agents to run.

    Args:
        sweep_id: The unique identifier for a sweep. A sweep ID
            is generated by W&B CLI or Python SDK.
        function: A function to call instead of the "program"
            specified in the sweep config.
        entity: The username or team name where you want to send W&B
            runs created by the sweep to. Ensure that the entity you
            specify already exists. If you don't specify an entity,
            the run will be sent to your default entity,
            which is usually your username.
        project: The name of the project where W&B runs created from
            the sweep are sent to. If the project is not specified, the
            run is sent to a project labeled "Uncategorized".
        count: The number of sweep config trials to try.
    rE   T)_silent)r2   r4   r   r   rx   )
_INSTANCESr	   wandb_login_loginr
   r  r   r4   )rw   r2   r   r   rx   r   r   r   r     s   

	r   c                   C   s   t tS r   )r   r  r   r   r   r   _is_runningC  s   r  rS   r   ).r  r   r&   r!   rB   r   rL   r   r$   r   rF   r   typingr   r   r   r   r   r   r)   r   r   r	   wandb.agents.pyagentr
   
wandb.apisr   wandb.sdk.launch.sweepsr   r   wandb.sdk.libr   	getLoggerr   r   r   r   r   rU   r   r  r   r   r   r  r  r   r   r   r   <module>   s`    
]  P
2
/