o
    wib                     @  s4  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlmZmZ d dlZd dlmZ d dlmZ d dlmZmZ eeZG dd deZG d	d
 d
ZG dd dZG dd dZ 						d#ddZ!					d$d%dd Z"d a#d!d" Z$dS )&    )annotationsN)AnyCallable)util)wandb_login)config_utilipythonc                   @  s   e Zd ZdS )
AgentErrorN)__name__
__module____qualname__ r   r   E/home/ubuntu/.local/lib/python3.10/site-packages/wandb/wandb_agent.pyr	      s    r	   c                   @  sV   e Zd ZdZ						dddZdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dS )AgentProcesszLaunch and manage a process.NFc           
   	   C  s  d | _ d | _t | _d| _i | _|rUttdd ttdd h}|	d  t
 D ],}||v r/q(ttt t|| j|< t|| j W d    n1 sOw   Y  q(|rt dkryttjd}	|tjjd  tj|fd|i|	| _ d S tjdkrtdd	}	nttjd
}	|tjjd  tj|f|tjd|	| _ | j j !  d S |rtj"| j#| j||||fd| _| j$  d S t%d)NFSIGKILLSIGSTOPWindows)creationflagsenv)      r   )process_group)
preexec_fn)r   stdin)targetargsz*Agent Process requires command or function)&_popen_procmultiprocessingQueue_finished_q_proc_killed_original_handlersgetattrsignaldiscardvalid_signals
contextlibsuppressOSError
ValueError	getsignal_forward_signalplatformsystemdict
subprocessCREATE_NEW_PROCESS_GROUPpopwandbr   SERVICEPopensysversion_infoossetpgrpPIPEr   closeProcess_startstartr	   )
selfr   commandfunctionrun_id
in_jupyterforward_signalsskip_signalssignumkwargsr   r   r   __init__!   sV   	




zAgentProcess.__init__c                 C  s   | j rt dkr|tjtjfv r| j tj n| j | | jr8t	tdr2|tj
kr2| j  n| j| | j|}|rKt|rM||| dS dS dS )zOForward a received signal to any child process, mirroring the agent's behavior.r   r   N)r   r-   r.   r$   SIGINTSIGTERMsend_signalCTRL_BREAK_EVENTr   hasattrr   killr"   getcallable)r?   rF   frameoriginal_handlerr   r   r   r,   r   s   zAgentProcess._forward_signalc           	      C  sl   |r|  D ]	\}}|tj|< qtd|  |r|  td| d tj}|r/t  |d d S )NzAgent Started Run: zAgent Finished Run: 
T)itemsr8   environr3   termlogrunjoinput)	r?   
finished_qr   rA   rB   rC   kvrW   r   r   r   r=      s   zAgentProcess._startc                 C  sZ   | j r| j  S | jr| j  dS z| jdd}|rW dS W d S  tjy,   Y d S w )NTFr   )	r   pollr!   r   rX   r    rO   queueEmpty)r?   finishedr   r   r   r]      s   

zAgentProcess.pollc                 C  sJ   | j r t dkr	 | j  }|d ur|S td q
| j  S | j S )Nr   T   )	r   r-   r.   r]   timesleepwaitr   rX   )r?   pr   r   r   rd      s   



zAgentProcess.waitc                 C  s8   | j r| j  S | jj}|rt|tj}d| _|S d S )NT)r   rN   r   pidr8   r$   r   r!   )r?   rf   retr   r   r   rN      s   
zAgentProcess.killc                 C  s4   | j rt dkr| j tjS | j  S | j S )Nr   )r   r-   r.   rK   r$   CTRL_C_EVENT	terminater   r?   r   r   r   ri      s
   

zAgentProcess.terminateNNNNNF)r
   r   r   __doc__rH   r,   r=   r]   rd   rN   ri   r   r   r   r   r      s    
Q
r   c                   @  s   e Zd ZU dZdZdZdZdZdZg dZ	de
d< ed	Z	
	
	
	
	dddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zd
S )Agent   r      <   r   )z${env}z${interpreter}z
${program}z${args}z	list[str]DEFAULT_SWEEP_COMMANDz\$\{envvar\:([A-Z0-9_]*)\}NFc                 C  s   || _ || _i | _g | _|| _|| _g | _d| _d | _|| _	t
j| j| _t
j| j| _d| _d| _|| _g | _t
j| j| _|| _| jd u rNtd| jd u rWtdtjdd u rltjt  tjd< d S d S )NTr   zInvalid agent report intervalzInvalid agent kill delay	WANDB_DIR)!_api_queue_run_processes_server_responses	_sweep_id_in_jupyter_log_running_last_report_time	_functionr3   r   get_agent_report_intervalREPORT_INTERVAL_report_intervalget_agent_kill_delay
KILL_DELAY_kill_delay	_finished_failed_count_sweep_commandget_agent_max_initial_failuresMAX_INITIAL_FAILURES_max_initial_failures_forward_signalsr	   r8   rU   rO   pathabspathgetcwd)r?   apir^   sweep_idrA   rC   countrD   r   r   r   rH      s:   


zAgent.__init__c                 C  s:   t tjjdkrdS t tj| j k r| j| j	kS dS )zDetermine if the process is flapping.

        Flapping occurs if the agents receives FLAPPING_MAX_FAILURES non-0 exit codes in
        the first FLAPPING_MAX_SECONDS.
        trueFN)
r8   getenvr3   r   AGENT_DISABLE_FLAPPINGrb   
START_TIMEFLAPPING_MAX_SECONDSr   FLAPPING_MAX_FAILURESrj   r   r   r   is_flapping   s
   zAgent.is_flappingc                 C  s   | j | jko| j| j kS N)r   r   r   rj   r   r   r   
is_failing  s   
zAgent.is_failingc                 C  sX  dd l }| j| jd}|r+|d}|r+||}|r+|d}|r+t|tr+|| _| jj	t
 | jd}|d }zz| jr8t| jd| j}|D ]}	|	d | |	 qLt }
| jd u so| jdkr}|
| j| j kr}td	t| j  |
| _i }t| j D ]\}}| }|d u rd
||< qt|tst|tr|dkr|  jd7  _|  rt d| j!| j" td d| _ nE| # rt d| j$ td d| _ n0td| t%t&drd}t|tr|}nt|trd}t&'| | j|= d | _|  j(d7  _(q| j)r| j(| j)ks| jsd| _q=| j*|i |}g | _+|D ]}	| j+,| |	 q(| jsAW n+ t-yd   zt&.d | j D ]	\}}|/  qLW n
 t-ya   Y nw Y nw W z7| j0spt&.d | j D ]\}}z|1  W qu t2y   Y quw | j D ]	\}}|/  qW d S  t-y   t&.d | j D ]\}}z|3  W q t2y   Y qw Y d S w z6| j0st&.d | j D ]\}}z|1  W q t2y   Y qw | j D ]	\}}|/  qW w  t-y+   t&.d | j D ]\}}z|3  W q t2y(   Y qw Y w w )Nr   z{}configr@   )r   idd   
resp_queuezRunning runs: %sTra   z?Detected %i failed runs in the first %i seconds, shutting down.z;To disable this check set WANDB_AGENT_DISABLE_FLAPPING=trueFz0Detected %i failed runs in a row, shutting down.z=To change this value set WANDB_AGENT_MAX_INITIAL_FAILURES=valzCleaning up finished run: %steardownzNCtrl-c pressed. Waiting for runs to end. Press ctrl-c again to terminate them.z3Terminating and syncing runs. Press ctrl-c to kill.zKilling runs and quitting.)4yamlrs   sweeprw   rO   	safe_load
isinstancelistr   register_agentsocketgethostnamerz   r   read_many_from_queuert   POLL_INTERVALrY   _process_commandstopwatch_nowr{   r   loggerinforu   keysrT   r]   boolintr   r   errorr   r   r   r   rM   r3   r   r   r   agent_heartbeatrv   appendKeyboardInterruptrV   rd   rx   ri   r)   rN   )r?   r   	sweep_obj
sweep_yamlsweep_configsweep_commandagentagent_idcommandsr@   now
run_statusrB   run_processpoll_result	exit_code_r   r   r   rW     s  






	


Q




z	Agent.runc                 C  s  t d|dd |dd d}z9|d }|dkr#| |}n%|dkr-| |}n|dkr7| |}n|d	krA| |}ntd
| ||d< W n* tyw   t 	d| t
 \}}}|j dt| |d< t||d< ~Y nw | j||f |S )NzAgent received command: {}typeUnknownr   )r   resultrW   stopexitresumezNo such command: r   z&Exception while processing command: %s: 	exception	traceback)r   r   formatrO   _command_run_command_stop_command_exitr	   	Exceptionr   r6   exc_infor
   strr   	format_tbry   r   )r?   r@   responsecommand_typer   ex_typeextbr   r   r   r     s2   zAgent._process_commandc                 C  sx  ddl m} tdddd |d  D   | jr6td|	d	 d
ddd |d  D   |
| j}|	d	}tj	tjj}tjdd| d| d }tjdd| d| d }|tjtjj< tj	tjjd}tj||}	|	tjtjj< t|	|d  ttj}
||}d|v rt|d}||d d  W d    n1 sw   Y  | jrt  t| j|
|| j| jd}nadg|d< |d g|d< |g|d< t dkrdg|d< g }|D ](}t |}|!dr|"dr|	|dd  }||pg 7 }q||g7 }qtd!#d"d#d$ |D  t||
| jd%}|| j$|< d | j$| _%d | _&d S )&Nr   utilsz Agent starting run with config:
rS   c                 S  s    g | ]\}}d  ||d qS )z	{}: {}value)r   .0r[   r\   r   r   r   
<listcomp>  s     z&Agent._command_run.<locals>.<listcomp>r   zAgent Starting Run: rB   z with config:
c                 S  s$   g | ]\}}d | d|d  qS )	r   r   r   r   r   r   r   r     s   $ r3   zsweep-zconfig-z.yamlz.json z${args_json_file}w	args_json)rA   r   rB   rC   rD   pythoninterpreterprogramargs_json_filer   z/usr/bin/envr   z${}   r   zAbout to run command: {} c                 s  s(    | ]}d |v rd| dn|V  qdS )r   "Nr   )r   cr   r   r   	<genexpr>  s   & z%Agent._command_run.<locals>.<genexpr>)r@   r   rD   )'wandb.sdk.launch.sweepsr   r   r   rX   rT   rx   r3   rV   rO   create_sweep_commandr   r8   rU   r   SWEEP_IDr   RUN_IDDIRSWEEP_PARAM_PATHr   save_config_file_from_dictr/   create_sweep_command_argsopenwriter|   r   r   r   r-   r.   r   
startswithendswithr   ru   last_sigterm_timer{   )r?   r@   sweep_utilsr   rB   r   config_file	json_filebase_dirsweep_param_pathr   
sweep_varsfpproccommand_listr   replace_listr   r   r   r     s   







zAgent._command_runc                 C  s   |d }|| j v rT| j | }t }|jd u r2||_td| z|  W d S  ty1   Y d S w ||j| j krRtd| z|	  W d S  tyQ   Y d S w d S t
d| d S )NrB   zStop: %szKill: %szRun %s not running)ru   r   r   r   r   r   ri   r)   r   rN   r   )r?   r@   rB   r   r   r   r   r   r     s*   


zAgent._command_stopc              	   C  sF   t d | j D ]\}}z|  W q
 ty   Y q
w d| _d S )Nz1Received exit command. Killing runs and quitting.F)r   r   ru   rT   rN   r)   rz   )r?   r@   r   r   r   r   r   r     s   

zAgent._command_exitNNNNF)r
   r   r   r   r~   r   r   r   r   rq   __annotations__recompileSWEEP_COMMAND_ENV_VAR_REGEXrH   r   r   rW   r   r   r   r   r   r   r   r   rm      s.   
 

( Trm   c                   @  s   e Zd Zdd Zdd ZdS )AgentApic                 C  s   || _ d| _t | _d S )Nr   )rt   _command_idr   Manager_multiproc_manager)r?   r^   r   r   r   rH   $  s   zAgentApi.__init__c                 C  s   d|d< d| j  |d< |  j d7  _ | j }||d< | j| | }td| d|v rFtd	 |d
 D ]}t|  q7t|d  |S )Nlocaloriginzlocal-r   ra   r   zresult:r   z(Exception occurred while running commandr   )r	  r  r   rt   rY   rO   printstrip)r?   r@   r   r   liner   r   r   r@   )  s   

zAgentApi.commandN)r
   r   r   rH   r@   r   r   r   r   r  #  s    r  Fc              	   C  s4  ddl m} ddlm} t||| d}	||	}
|
r!t|
 d S |	dp'|}|	dp.|}|	dp5| } |r>tj	
| |rFtj	| | rO| tjtj	j< ttj t }tj}|ratj}|| td}|| z#t| | }t }t||| ||||d	}|  W t| d S t| w )
Nr   )InternalApir   )entityprojectnamer  r  r  z4%(asctime)s - %(name)s - %(levelname)s - %(message)s)r   rA   rC   r   rD   )
wandb.apisr  r   r   r/   parse_sweep_idr3   	termerrorrO   r   
set_entityset_projectr8   rU   r   r   setLevelloggingDEBUGStreamHandlerERROR	FormattersetFormatter
addHandlerr   r   rm   rW   removeHandler)r   rA   rC   r  r  r   rD   r  r   partserrch	log_level	formatterr   r^   r   r   r   r   	run_agent:  sR   	





	r(  r   r   rA   Callable | Noner  
str | Noner  r   
int | NonerD   r   returnNonec              	   C  sn   ddl m} td7 az'tjdd |r || ||||W td8 aS t| |t ||||dW td8 aS td8 aw )ad  Start one or more sweep agents.

    The sweep agent uses the `sweep_id` to know which sweep it
    is a part of, what function to execute, and (optionally) how
    many agents to run.

    Args:
        sweep_id: The unique identifier for a sweep. A sweep ID
            is generated by W&B CLI or Python SDK.
        function: A function to call instead of the "program"
            specified in the sweep config.
        entity: The username or team name where you want to send W&B
            runs created by the sweep to. Ensure that the entity you
            specify already exists. If you don't specify an entity,
            the run will be sent to your default entity,
            which is usually your username.
        project: The name of the project where W&B runs created from
            the sweep are sent to. If the project is not specified, the
            run is sent to a project labeled "Uncategorized".
        count: The number of sweep config trials to try.
        forward_signals: Whether to forward signals the agent receives
            to the child processes. Only supported by CLI agent.

    r   )pyagentra   T)_silent)rA   rC   r  r  r   rD   )wandb.agents.pyagentr.  
_INSTANCESr   _loginr(  r   rC   )r   rA   r  r  r   rD   r.  r   r   r   r   t  s"    

r   c                   C  s   t tS r   )r   r1  r   r   r   r   _is_running  s   r3  rk   r  )r   r   rA   r)  r  r*  r  r*  r   r+  rD   r   r,  r-  )%
__future__r   r'   r  r   r8   r-   r^   r  r$   r   r0   r6   rb   r   typingr   r   r3   r   	wandb.sdkr   wandb.sdk.libr   r   	getLoggerr
   r   r   r	   r   rm   r  r(  r   r1  r3  r   r   r   r   <module>   sT    
 ,  \
<6