o
    -wi                     @   sL  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZmZ ddlZddlZddlmZ ddlmZmZ dd	l m!Z! dd
l"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? ddl@mAZA erddlBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZM ed ZNeOePZQdZRdeeeSeSf ddf fddZTdddefddZUdd defd!d"ZVG d#d$ d$ZWG d%d& d&ZXG d'd( d(ZYdS ))zsender.    N)defaultdict)datetime)Queue)
TYPE_CHECKINGAnyDict	GeneratorListLiteralOptionalTupleTypeUnion)util)	CommError
UsageError)ProtobufErrorHandler)
DirWatcher)wandb_internal_pb2)ArtifactSaver)	interface)InterfaceQueue)context	datastorefile_streaminternal_apisender_config)
FilePusher)
JobBuilder)SettingsStatic)config_util	filenames
filesystem
proto_utilredirectretry	telemetry)message_to_dict)ArtifactManifestArtifactManifestEntryArtifactRecordEnvironmentRecordHttpResponse	LocalInfoRecordResultRunExitResult	RunRecordSummaryRecord)stdoutstderr   returnc                   c   s    g dE d H  d S )N))lightgbmr7   )catboostr8   )xgboostr9   )transformers_huggingfacehuggingface)transformersr;   )pytorch_igniteignite)r>   r>   )pytorch_lightning	lightning)fastairA   )torchrB   )kerasrC   )
tensorflowrD   )sklearnrE    rF   rF   rF   V/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/sdk/internal/sender.py_framework_priorityU   s   rH   manifestr(   c                 C   s   | j dkr>| jr5i }t| jd}|D ]}t|}|d}|||< qW d    n1 s/w   Y  ndd | jD }ntd| j  | j | j	dd | j
D |dS )	N   rtpathc                 S   s   i | ]}|j t|qS rF   )rL   _manifest_entry_from_proto).0contentrF   rF   rG   
<dictcomp>q   s    
z-_manifest_json_from_proto.<locals>.<dictcomp>z#unknown artifact manifest version: c                 S      i | ]
}|j t|jqS rF   keyjsonloads
value_json)rN   configrF   rF   rG   rP   {   s    )versionstoragePolicystoragePolicyConfigcontents)rX   manifest_file_pathgzipopenrT   rU   popr[   
ValueErrorstorage_policystorage_policy_config)rI   r[   fline
entry_jsonrL   rF   rF   rG   _manifest_json_from_protog   s,   



rf   entryr)   c                 C   s\   | j r| j nd }| j|| jr| jnd | jd ur| jnd | jr!| jnd | jdd | jD dS )Nc                 S   rQ   rF   rR   )rN   extrarF   rF   rG   rP      s    z._manifest_entry_from_proto.<locals>.<dictcomp>)digestbirthArtifactIDrefsize
local_path
skip_cacherh   )birth_artifact_idri   rk   rl   rm   rn   rh   )rg   ro   rF   rF   rG   rM      s   rM   c                   @   s   e Zd ZU eed< eed< eed< eed< eed< eed< ee ed< eee	e
f  ed< eee	e
f  ed	< eee	  ed
< dddZde	fddZdS )ResumeStateresumedstephistoryeventsoutputruntimewandb_runtimesummaryrW   tagsr6   Nc                 C   s@   d| _ d| _d| _d| _d| _d| _d | _d | _d | _d | _	d S )NFr   )
rq   rr   rs   rt   ru   rv   rw   rx   rW   ry   selfrF   rF   rG   __init__   s   
zResumeState.__init__c                 C   s(   d tdd t|  }d| dS )N,c                 S   s   | d  d| d  S )Nr   =rJ   rF   )itrF   rF   rG   <lambda>   s    z%ResumeState.__str__.<locals>.<lambda>zResumeState())joinmapvarsitems)r{   objrF   rF   rG   __str__   s   zResumeState.__str__r6   N)__name__
__module____qualname__bool__annotations__intfloatr   r   strr   r	   r|   r   rF   rF   rF   rG   rp      s   
 
rp   c                   @   s\   e Zd ZU ejed< ejed< ej	ed< ej
ed< ej
ed< deddfd	d
ZdddZdS )_OutputRawStream_stopped_queue	_emulator_writer_thr_reader_thrstreamsmSendManagerc                 C   sf   t  | _t | _t | _t j	|j
t|ddd| d| _t j	|jt|ddd| d| _d S )N)r   Tz	OutRawWr-)targetkwargsdaemonnamez	OutRawRd-)	threadingEventr   queuer   r   r$   TerminalEmulatorr   Thread_output_raw_writer_threaddictr   _output_raw_reader_threadr   )r{   r   r   rF   rF   rG   r|      s   


z_OutputRawStream.__init__r6   Nc                 C   s   | j   | j  d S N)r   startr   rz   rF   rF   rG   r      s   
z_OutputRawStream.startr   )r   r   r   r   r   r   r   r   r$   r   r   r   r|   r   rF   rF   rF   rG   r      s   
 




r   c                   @   s  e Zd ZU dZeed< dZeed< eed< ded< ded	< eed
< e	e
e
f ed< e	e
e
f ed< ejed< ejed< ded< ed ed< ed ed< ee
 ed< ee
 ed< ed ed< ed ed< ed ed< ed ed< eed< ee	e
ef  ed < e	e
ef ed!< e	e
ef ed"< ee	e
ef  ed#< eej ed$< e	d%ef ed&< eej ed'< eed(< eed)< eed*< eed+< d,ed-dd.dd/ed0ejd1d2fd3d4Zed5e
d6ed2ee
f d1d fd7d8Zd1efd9d:Zdd;d<Zd=ee e!  d>ee! d?ee"j# d1e$d@ fdAdBZ%dCedDe
d1d2fdEdFZ&ddHdIZ'ddKdLZ(ddMdNZ)ddOdPZ*ddSdTZ+dUe	d1d2fdVdWZ,dXed1d2fdYdZZ-d[ed1d2fd\d]Z.dd^d_Z/dd`daZ0ddbed1d2fdcddZ1ddbed1d2fdedfZ2ddged1d2fdhdiZ3ddjdkZ4ddldmZ5ddndoZ6ddpdqZ7ddrdsZ8ddtduZ9ddvdwZ:ddxdyZ;dzdd1ed{ fd|d}Z<d1e
fd~dZ=d1e>j?fddZ@de>j?d1d2fddZAdddZBdeCfddZDdddZEdd ZF	G			{	z		1	2dddZGd	dGddee
 d1d2fddZHdedefddZIdzddee>j? d1eCfddZJd	dee
 d1d2fddZKde	e
ef d1d2fddZLdddZMd
ddZNdddZOdddZPdddZQdddZRdddZSdddZTdddZU	2d	dd%dee
 d1d2fddZVdddZWdddZXdddZYdd%de
d1d2fddZZdddZ[dddĄZ\dddƄZ]dejd1d2fddɄZ^ddd˄Z_ddd̈́Z`	ΐddeajbddd1d2fddӄZcdddՄZddddׄZedddلZfdddۄZgdddބZhdddZidddZjdddZkdddZldddZm	2d	dddee d1ee	 fddZndddZodddZpd1ee
 fddZqdddZrd1e	e
ef fddZsd1e	e
ef fddZtdddZudddZvddd ZwewZxd2S (  r      UPDATE_CONFIG_TIME   UPDATE_STATUS_TIME	_settingszQueue[Record]	_record_qzQueue[Result]	_result_q
_interface_api_settings_partial_output_context_keeper_telemetry_objr+   _environment_objzfile_stream.FileStreamApi_fsr1   _run_entity_projectr   _dir_watcherr   _pusherr.   _record_exitr0   _exit_result_resume_state_rewind_response_cached_server_info_cached_viewer_server_messages_dsStreamLiterals_output_raw_streams_output_raw_file_send_record_num_send_end_offset_debounce_config_time_debounce_status_timesettingsrecord_qresult_qr   context_keeperr6   Nc                 C   s@  || _ || _|| _|| _|| _d | _d| _d| _d | _d | _	d | _
d | _d | _d | _d | _t | _d| _t | _t | _g | _t | _t | _i | _i | _t | _ t | _!t | _"g | _#t$ | _%d | _&d | _'d | _(t)j*|| j+d| _,t | _-t./ | _0d| _1t | _2d| _3t | _4d | _5t6|| _7t89 }|| _:|| _;d S )Nr   )default_settingsretry_callbackF)<r   r   r   r   r   r   r   r   r   r   r   r   _flagsr   r   r   ConfigState_consolidated_config_start_timer&   TelemetryRecordr   r   r+   r   _config_metric_pbdict_listr   _metadata_summaryr   _cached_summary_config_metric_index_dict_config_metric_dict_consolidated_summaryr   r   r   rp   r   r   r   r   r   Apir   _apir   r   r   _retry_q_config_needs_debouncer   
_exit_coder   r   r   _job_buildertime	monotonicr   r   )r{   r   r   r   r   r   time_nowrF   rF   rG   r|      s\   





zSendManager.__init__root_dirresumec           	      C   s`   t j|d}tj|||dddd}t }t }t|d}t	 }t
t| ||||dS )zQSet up a standalone SendManager.

        Exclusively used in `sync.py`.
        filesTFr   )x_files_dirr   r   x_syncdisable_job_creationx_file_stream_timeout_seconds)r   )r   r   r   r   r   )osrL   r   wandbSettingsr   r   r   r   ContextKeeperr   r   to_proto)	clsr   r   	files_dirr   r   r   publish_interfacer   rF   rF   rG   setup@  s(   



zSendManager.setupc                 C   s
   | j  S r   )r   qsizerz   rF   rF   rG   __len__a     
zSendManager.__len__c                 C   s   | S r   rF   rz   rF   rF   rG   	__enter__d     zSendManager.__enter__exc_type	exc_valueexc_tracebackFc                 C   s&   | rt | }| | | s|   dS )NF)nextsendfinish)r{   r   r   r   datarF   rF   rG   __exit__g  s   
zSendManager.__exit__statusresponse_textc                 C   s$   t  }||_||_| j| d S r   )r   r,   http_status_codehttp_response_textr   put)r{   r  r  responserF   rF   rG   r   s  s   zSendManager.retry_callbackrecordc                 C   s   |  |j | |jj |d}|sJ d| }t| |d }|dvr,td|  |s5J d| t	
|}| j|}z5z| j| || W n tjye   td|  | j| Y n	w W | j  d S W | j  d S | j  w )Nrecord_typesend_>   ru   request
output_rawzsend: zunknown send handler: zRecord cancelled: )_update_record_numnum_update_end_offsetcontrol
end_offset
WhichOneofgetattrloggerdebugr   context_id_from_recordr   getr   set_local_contextr%   RetryCancelledErrorreleaseclear_local_context)r{   r  r  handler_strsend_handler
context_idapi_contextrF   rF   rG   r   y  s,   

zSendManager.send_c                 C   s   | j r
| j   d S d S r   )r   enqueue_preemptingr{   r  rF   rF   rG   send_preempting  s   zSendManager.send_preemptingc                 C   s   | j dd d S )NTalways)_maybe_report_statusr!  rF   rF   rG   send_request_sender_mark     z$SendManager.send_request_sender_markc                 C   s^   |j d}|s
J d| }t| |d }|dkr td|  |s)J d| || d S )Nrequest_typesend_request_network_statuszsend_request: zunknown handle: )r
  r  r  r  r  )r{   r  r(  r  r  rF   rF   rG   send_request  s   zSendManager.send_requestresultr/   c                 C   s&   t |}| j| | j| d S r   )r   context_id_from_resultr   r  r   r  )r{   r,  r  rF   rF   rG   _respond_result  s   
zSendManager._respond_result
dictionaryc                 C   sh   t |tr0t| D ]&\}}t |tr/| | || | D ]\}}|||d | < q"qd S d S )N.)
isinstancer   listr   _flattenr_   )r{   r/  kvk2v2rF   rF   rG   r3    s   



zSendManager._flatten
record_numc                 C   s4   |sd S | j js| j js|| jd ksJ || _d S )NrJ   )r   _offliner   r   )r{   r8  rF   rF   rG   r    s
   
zSendManager._update_record_numr  c                 C   s   |sd S || _ d S r   )r   )r{   r  rF   rF   rG   r    s   
zSendManager._update_end_offsetc                 C   s   | j d u rt | _ | j | jj |jjj}|jjj	}| j 
| d}||k rQ| j  }|s1J | j  }t }|| | | | | |   ||k s(| jdd d S )Nr   Tr#  )r   r   	DataStoreopen_for_scanr   	sync_filer
  sender_readstart_offsetfinal_offsetseek	scan_data
get_offsetr   r.   ParseFromStringr  r   debouncer%  )r{   r  r>  r?  current_end_offsetr   send_recordrF   rF   rG   send_request_sender_read  s$   








z$SendManager.send_request_sender_readc              
   C   s   t |}|jj}d|_| jrA| jrA| jrA| jjrAz| j	
| j| j| jj|_W n ty@ } ztd| W Y d }~nd }~ww | | d S )NFz)Failed to check stop requested status: %s)r#   _result_from_recordr  stop_status_responserun_should_stopr   r   r   run_idr   check_stop_requested	Exceptionr  warningr.  r{   r  r,  status_resperF   rF   rG   send_request_stop_status  s   

z$SendManager.send_request_stop_statusr$  c                 C   s8   t  }|s|| j| j k rd S | jr|   || _d S r   )r   r   r   r   r   _debounce_config)r{   r$  r   rF   rF   rG   _maybe_update_config  s   
z SendManager._maybe_update_configc                 C   sr   t  }|s|| j| j k rd S || _tj| j| jd}t   }|j	t
|d  | jj|d}| j| d S )N)r8  sent_offset    .A)status_report)r   r   r   r   r   StatusReportRequestr   r   	sync_timeFromMicrosecondsr   r   _make_request_publish)r{   r$  r   rW  status_timer  rF   rF   rG   r%    s   z SendManager._maybe_report_statusfinalc                 C   s   | j |d | j|d d S )Nr#  )r%  rT  )r{   r^  rF   rF   rG   rD    s   zSendManager.debouncec                 C   s@   |   }| jr| jjd| jj|d| j | | d| _d S )N)r   rW   FrF   )_config_backend_dictr   r   
upsert_runrK  r   _config_saver   )r{   config_value_dictrF   rF   rG   rS    s   

zSendManager._debounce_configc              
   C   s   t |}|jj}	 z|j| j  W n# tj	y   Y n t
y8 } ztd|  W Y d }~nd }~ww q
| | d S )NTzError emptying retry queue: )r#   rH  r  network_status_responsenetwork_responsesappendr   
get_nowaitr   EmptyrM  r  rN  r.  rO  rF   rF   rG   send_request_network_status  s   
z'SendManager.send_request_network_statusc                 C   s`   || _ |j}|j| _td|j |j}td|j || jd< |   td | j	
  d S )Nzhandling exit code: %szhandling runtime: %srv   z
send defer)r   exit	exit_coder   r  inforv   r   _update_summaryr   publish_defer)r{   r  run_exitrv   rF   rF   rG   	send_exit'  s   

zSendManager.send_exitc                 C      d S r   rF   r{   r  rF   rF   rG   
send_final8  r   zSendManager.send_finalc                 C   rp  r   rF   rz   rF   rF   rG   
_flush_run;  r   zSendManager._flush_runc                 C   rp  r   rF   rq  rF   rF   rG   send_request_status_report>     z&SendManager.send_request_status_reportc                    s*  |j j  j}td|  d	 fdd}d}| jkr"|  n| jkr/  |  n| jkr8|  n| j	krA|  n| j
krJ|  n| jkrS|  n| jkrbjdd |  n| jkro  |  nz| jkr|  |  nm| jkrjrj  d _|  nY| jkrjrj| nJ|  nF| jkrjrj  |  n5| jkrʈjrƈjj d _|  n| jkr݈j  j  |  n| j krd}nt!d|sd S t"# }|_$j%rj%j&j'rt()j%}|j*+| ,| d S d S d S )
Nzhandle sender defer: r6   c                     s*    j d } td|   j|  d S )NrJ   zsend defer: )stater  rk  r   rm  )rv  deferr{   rF   rG   transition_stateG  s   
z8SendManager.send_request_defer.<locals>.transition_stateFT)r^  zunknown stater   )-r
  rx  rv  r  rk  BEGIN	FLUSH_RUNrs  FLUSH_STATSFLUSH_PARTIAL_HISTORYFLUSH_TB	FLUSH_SUMFLUSH_DEBOUNCERrD  FLUSH_OUTPUT_output_raw_finish	FLUSH_JOB
_flush_job	FLUSH_DIRr   r   FLUSH_FPr   JOIN_FPr   FLUSH_FSr   r   FLUSH_FINALr   publish_finalpublish_footerENDAssertionErrorr   r0   r   r   r  mailbox_slotr#   rH  exit_resultCopyFromr.  )r{   r  rv  ry  doner  r,  rF   rw  rG   send_request_deferB  sz   


















zSendManager.send_request_deferc                 C   s   |j js
|j js
d S t|}| jrE| j \}}| j }|jj	}|j
|j_
|j|j_|j|j_|j|j_|j|j_|j|j_|j|j_| jrVd|jj	_|jj	j| j | | d S NT)r  req_respr  r#   rH  r   
get_statusfile_counts_by_categoryr  poll_exit_responseuploaded_bytespusher_statstotal_bytesdeduped_bytesr   file_countswandb_countmediamedia_countartifactartifact_countotherother_countr   r  r  r  r.  )r{   r  r,  _aliver  r  resprF   rF   rG   send_request_poll_exit  s$   









z"SendManager.send_request_poll_exitrunwandb_internal_pb2.ErrorInfoc              	   C   sx  | j jsdS |jp| j}td||j|j | jj	||j|jd}|r+d|
ddvrG| j jdkrEt }tjjj|_d|j d	|_|S dS | j jd
krat }tjjj|_d|j d|_|S i }i }i }i }z]d}	d}
t|d }|rt|d }|
dd}
t|d }|rt|d }|
dd}	t|d pd}t|d pd}|
di 
dd}|dur|| j_|
dpg }W n, ttfy   td | j jdkrt }tjjj|_d|j d|_| Y S Y nw t|	|
| j_|
dd}|d }|dkr|d n|| j_|| j_|d | j_|d | j_|| j_|| j_|| j_ d | j_!td!| j  dS )"zEQueries the backend for a run; fail if the settings are incompatible.Nz#checking resume status for %s/%s/%s)entityproject_namer   z"t":wandbConfig mustztYou provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming a run (z) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`.neverzuYou provided an invalid value for the `resume` argument. The value 'never' is not a valid option for resuming a run (zk) that already exists. Please check your inputs and try again with a valid value for the `resume` argument.r   historyTail_runtime
eventsTailrW   {}summaryMetrics_wandbrv   ry   zunable to load resume tailsz$resume='must' but could not resume (z) _stephistoryLineCountrJ   eventsLineCountlogLineCountTzconfigured resuming with: )"r   r   r  r   r  rk  projectrK  r   run_resume_statusr  r   	ErrorInfo	ErrorCodeUSAGEcodemessagerT   rU   r   rw   
IndexErrorr`   	exceptionmaxrv   rr   rs   rt   ru   rW   rx   ry   rq   )r{   r  r  resume_statuserrorrs   rt   rW   rx   	events_rt
history_rtnew_runtimery   	last_stephistory_line_countrF   rF   rG   _setup_resume  s   

zSendManager._setup_resumec                    sN   | j dr| j j n| j dr| j j ndS t fddt D d}|S )z1Get telemetry data for internal config structure.imports_finishimports_initr  c                 3   s$    | ]\}}t  |d r|V  qdS )FN)r  )rN   rc   nimportsrF   rG   	<genexpr>  s   " z7SendManager._telemetry_get_framework.<locals>.<genexpr>)r   HasFieldr  r  r   rH   )r{   	frameworkrF   r  rG   _telemetry_get_framework  s   

z$SendManager._telemetry_get_frameworkc                 C   s.   | j pt }|j| j|  | j| j| jdS )N)telemetry_recordr  start_time_millismetric_pbdictsenvironment_record)	r   r   r   to_backend_dictr   r  r   r   r   )r{   rW   rF   rF   rG   r_    s   z SendManager._config_backend_dictrb  c                 C   s"   t j| jjd}t|| d S )Nzconfig.yaml)r   rL   r   r   r   r    save_config_file_from_dict)r{   rb  config_pathrF   rF   rG   ra  $  s   zSendManager._config_savec                 C   s   | j sdS z:tj}| jjd|dd | jj d| j j d| j j	 d| j j
 }tj|ddd	 |d
|ddd W dS  tjyJ   Y dS w )zSync this run with spell.N)r  	spell_urlSPELL_RUN_URL)rS   val/z/runs/SPELL_API_URLzhttps://api.spell.runz
/wandb_urlWANDB_ACCESS_TOKEN)access_tokenurlr5   )rT   timeout)r   r   environr   publish_configr  r   app_urlr  r  rK  requestsr  RequestException)r{   envr  rF   rF   rG   _sync_spell+  s    
*zSendManager._sync_spell
server_runc                 C   sT   | j sJ | j jsJ t| j jjd }|| j_|dd| j_d| j _|| j _	d S )NrJ   r  r   T)
r   branch_pointr   valuer   rr   r  rs   forkedstarting_step)r{   r  
first_steprF   rF   rG   _setup_fork>  s   
zSendManager._setup_forkc                 C   sp   |j sJ | jj|j|jpd |jpd |j j|j j| jj	pd d| _
| j
dd| j_t| j
dd| j_d S )N)run_namer  r  metric_namemetric_valueprogram_pathr  r   rW   r  )r  r   
rewind_runrK  r  r  metricr  r   programr   r  r   rs   rT   rU   rW   )r{   r  rF   rF   rG   _load_rewind_stateG  s   

zSendManager._load_rewind_statec                 C   sN   | j sJ | j jsJ | jsJ t| j jjd }|| j_d| j _|| j _d S )NrJ   T)	r   r  r   r   r  r   rr   r  r  )r{   r  rF   rF   rG   _install_rewind_stateV  s   

z!SendManager._install_rewind_stater  c                 C   sV   |j js|j jr"t|}|jj| |jj| | 	| d S t
d|j d S )NzGot error in async mode: %s)r  r  r  r#   rH  
run_resultr  r  r  r.  r  r  )r{   r  r  r  r,  rF   rF   rG   _handle_errord  s   
zSendManager._handle_errorfile_dirc              
   C   sx  |j }d }| jd u }t|j d | _|jr| j|j | j	j
r'd| jj_d }|jr<| j|j |  }| | |jj |jk}| oK|jj dk}t| j	j}	t|||	g}
|
dkrqt }tjjj|_d|_| ||| |r|jdkrt | j	j!|_|	r| "|}n|r| #|}|d ur| ||| d S | j$jd ur| j%t&'| j$j |  }| | |s|  }| | z| (||}W n( t)t*fy } zt+j,|dd t-.|}| ||| W Y d }~d S d }~ww | jsJ |r| /|}|d ur| ||| d S |j0j1s|j0j2r(t34|}|j5j 6| j | 7| |r2| 8| d S t+9d| jj d S )NrV  Tr  rJ   zfMultiple resume options specified. Please specify only one of `fork_from`, `resume`, or `resume_from`.)exc_infozupdated run: %s):r  r   r   
start_timeToMicrosecondsr   r&   r   	MergeFromr   r   featuresyncrW   r   update_from_protor_  ra  r  rK  r   r   sumr   r  r  r  r  r  r  r  r   auto_project_namer  r  r  r   merge_resumed_configr    dict_strip_value_dict	_init_runr   r   r  r  r   from_exceptionr  r  r  r  r#   rH  r  r  r.  _start_run_threadsrk  )r{   r  r  r  r  is_wandb_initrb  	do_rewinddo_fork	do_resumenum_resume_options_setr  rQ  r,  rF   rF   rG   send_runr  s   












zSendManager.send_runis_rewindinginsertedc                 C   s`   | j sJ | jjrd| j _| jjd ur| jj| j _d S d S |r%|   d S |s.d| jj_d S d S r  )	r   r   rq   rw   rv   r  r   r  maybe_run_overwrite)r{   r  r  rF   rF   rG   _update_resume_state  s   
z SendManager._update_resume_stateconfig_dictc                 C   s  |j  d | jj }| jr| jjr|js|j| jj t| jj}|r2| j	s*J | j	}d }d}nE| j
j|j|jp;d |jp?d |jpCd |jpGd |jpKd |jpOd |jd d  pWd |pZd |jp^d |jpbd | jjpgd |jjpld |jjpqd d\}}}|jr~d| j_|pg | _|| _| jjr|rtd| || | jj | j_!| jj "t#|d  | jj$%| j&'| | jj(d ur| jj(%| j&)| jj( |*d}|r|| j_+|*d}	|	r| j
,|	 |*d}
|
r|
| j_|*d}|r.|*d}|r|| j_|| _-|| j.d< | j
/d| |*d	}|r.|*d}|r.|| j_|| _0|| j.d	< | j
/d	| |*d
}|r:|| j_t12drD| 3  |S )NrV  T)r   r  r  groupjob_typedisplay_namenotesry   rW   
sweep_namehostr  repocommitzcCannot attempt to rewind and resume a run - only one of `resume` or `resume_from` can be specified.idr   displayNamer  r  	sweepNamer  )4r   r  r   rv   ry   extendr   r   resume_fromr   r   r`  rK  r  r  	run_groupr  r  r  sweep_idr  r  git
remote_urlr  r   disabler   r   rq   r`   r  rr   r  rZ  r   rW   r  r   _make_configrx   _make_summary_from_dictr  
storage_idset_current_run_idr   r   set_settingr   r   getenvr  )r{   r  r  r   r  r  server_messagesr  r,  r   r  r  r  r  entity_namer&  rF   rF   rG   r
    s   














zSendManager._init_runc                 C   s  | j sJ tj| j| j j| j j d | jjpd| j	d| _
| j
dt  | j
dtj| jjd | j
dtj| jjd | j
dtj| jjd t| j }t| j}|| tjj|d	d
 | j
  t| j| j
| jd| _t| j| j|| _td| j j| j j d  d S )NrV  r   )r  r   zwandb-summary.jsonzwandb-history.jsonl)start_chunk_idzwandb-events.jsonlz
output.loginternal)ry   process_context)r   z"run started: %s with start time %s) r   r   FileStreamApir   rK  r   r  r   r   r   r   set_file_policySummaryFilePolicyJsonlFilePolicyr   rs   rt   CRDedupeFilePolicyru   r'   r   updater   _sentryconfigure_scoper   r   r   r   r   r  rk  )r{   r  run_settingsr   rF   rF   rG   r  I  sB   





zSendManager._start_run_threadshistory_dictc                 C   s$   | j r| j tjt| d S d S r   )r   pushr!   HISTORY_FNAMErT   dumps)r{   r>  rF   rF   rG   _save_historyq  s   zSendManager._save_historyc                 C   s    |j }t|j}| | d S r   )rs   r#   dict_from_proto_listitemrB  )r{   r  rs   r>  rF   rF   rG   send_historyu  s   zSendManager.send_historyrx   r2   c                 C   s   t |j}|| _|   d S r   )r#   rC  r:  r   rl  )r{   rx   summary_dictrF   rF   rG   _update_summary_recordz  s   z"SendManager._update_summary_recordc                 C      |  |j d S r   )rG  rx   rq  rF   rF   rG   send_summary  r'  zSendManager.send_summaryc                 C      |  |jjj d S r   )rG  r
  summary_recordrx   rq  rF   rF   rG   send_request_summary_record     z'SendManager.send_request_summary_recordc                 C   s   | j  }|dd  | jr| j|d< | j| t| j}| jr*| j	t
j| tj| jjt
j}t|d}|| W d    n1 sIw   Y  | tt
j d S )Nr  w)r   copyr_   r   r   r:  rT   rA  r   r?  r!   SUMMARY_FNAMEr   rL   r   r   r   r^   write
_save_filer   GlobStr)r{   rF  json_summarysummary_pathrc   rF   rF   rG   rl    s   

zSendManager._update_summaryc              	   C   s   |j }|jtjjjkrd S | jsd S | jsd S |j	 }| jj
	 }t }|jD ]}zt|j||j< W q( tjyF   td|j Y q(w t|d}| | d|d< |d |d< || d |d< | jtjt| d S )Nzerror decoding stats json: %s)systemTr  rV  
_timestampr  )stats
stats_typer   StatsRecord	StatsTypeSYSTEMr   r   	timestampr  r   r   rD  rT   rU   rV   rS   JSONDecodeErrorr  r  r3  r?  r!   EVENTS_FNAMErA  )r{   r  rX  now_usstart_usdrD  rowrF   rF   rG   
send_stats  s,   



zSendManager.send_statsc                 C   s   | j  D ])\}}|j  |jjdd |j r$td |j  |j	  | 
| qi | _ | jr?| j  d | _d S d S )Nr   )r  zprocessing output...)r   r   r   setr   r   is_aliver  rk  r   _output_raw_flushr   closer{   r   r  rF   rF   rG   r    s   






zSendManager._output_raw_finishr   c              
   C   s   	 | j | }|j r|j rd S td q g }|j s,||j  |j r|j rQt	t
t|dkrQtd | | |D ]}| || qFd S z|jd| W n tyv } ztd|  W Y d }~nd }~ww q)NTg      ?i z6Terminal output too large. Logging without processing.r  z(problem writing to output_raw emulator: )r   r   emptyr   is_setr   sleepre  r  r  r   lenr  rN  rg  r   rQ  r   rM  )r{   r   r  r   rd   rQ  rF   rF   rG   r     s0   







z%SendManager._output_raw_writer_threadc                 C   sN   | j | }|j r|j s%| | tt |j r|j rd S d S r   )	r   r   rk  r   rj  rg  r   rl  _OUTPUT_MIN_CALLBACK_INTERVALri  rF   rF   rG   r     s
   


z%SendManager._output_raw_reader_threadr   c              
   C   s   |d u r+| j | }z|j }W n ty* } ztd|  W Y d }~nd }~ww |rA| || | jrC| j|	d d S d S d S )Nz*problem reading from output_raw emulator: zutf-8)
r   r   readrM  r  rN  _send_output_liner   rQ  encode)r{   r   r   r  rQ  rF   rF   rG   rg    s   
zSendManager._output_raw_flushc                 C   sz   dd l }ddlm} tdd |jjjD }t|j	| j
j|d}|d	| W d    d S 1 s6w   Y  d S )Nr   )REQUIREMENTS_FNAMEc                 s   s"    | ]}|j  d |j V  qdS )z==N)r   rX   )rN   rrF   rF   rG   r    s    
z;SendManager.send_request_python_packages.<locals>.<genexpr>rN  
)r   wandb.sdk.lib.filenamesrr  sortedr
  python_packagespackager^   rL   r   r   r   rQ  )r{   r  r   rr  installed_packages_listrc   rF   rF   rG   send_request_python_packages  s   "z(SendManager.send_request_python_packagesc                 C   s>   | j sd S |j}d}|jtjjjkrd}|j}| || d S )Nr3   r4   )	r   ru   output_typer   OutputRecord
OutputTypeSTDERRrd   rp  )r{   r  outr   rd   rF   rF   rG   send_output  s   zSendManager.send_outputc           	   
   C   s   | j sd S |j}d}|jtjjjkrd}|j}| j	|}|sgt
|| d}|| j|< | jsctj| jjtj}d }z
tt|d}W n ty] } ztd|  W Y d }~nd }~ww |rc|| _|  |j| d S )Nr3   r4   )r   r   wbz could not open output_raw_file: )r   r  r{  r   OutputRawRecordr}  r~  rd   r   r  r   r   r   rL   r   r   r   r!   OUTPUT_FNAMEr"   CRDedupedFiler^   OSErrorr  rN  r   r   r  )	r{   r  r  r   rd   r  output_log_pathoutput_raw_filerQ  rF   rF   rG   send_output_raw  s8   

zSendManager.send_output_rawrd   c                 C   s   d}|dkrd}| ds)| j|d |drd| j|< | j|  |7  < dS t }t| d }| j|d}| | | | }| j	rR| j	
tj| d| j|< dS )zyCombined writer for raw and non raw output lines.

        This is combined because they are both post emulator.
        r  r4   zERROR rt   N)endswithr   
setdefault
startswithr   r   utcfromtimestamp	isoformatr  r   r?  r!   r  )r{   r   rd   prependcur_timer]  prev_strrF   rF   rG   rp    s   


zSendManager._send_output_linec                 C   s
   d| _ d S r  )r   rz   rF   rF   rG   _update_config=  r   zSendManager._update_configc                 C   s   | j |j |   d S r   )r   r  rW   r  rq  rF   rF   rG   send_config@  s   zSendManager.send_configc           	      C   s   |j }|jrtd d S | j|jt }|j	j
r!|| n|| || j|j< |}|jrS| j|j}|d urSt }|j | |j }|d |d |_t|}| j|j}|d uri|| j|< nt| j}| j| || j|j< |   d S )Nz(Seen metric with glob (shouldn't happen)step_metricrJ   )r  	glob_namer  rN  r   r  r   r   MetricRecord_control	overwriter  r  r  r   r.   
ClearFieldstep_metric_indexr#   proto_encode_to_dictr   rm  re  rS  )	r{   r  r  
old_metricfind_step_idxrecmdfind_idxnext_idxrF   rF   rG   send_metricD  s8   






zSendManager.send_metricr&   c                 C      | j | |   d S r   )r   r  rS  )r{   r&   rF   rF   rG   _update_telemetry_recordk     z$SendManager._update_telemetry_recordc                 C   rH  r   )r  r&   rq  rF   rF   rG   send_telemetryo  r'  zSendManager.send_telemetryc                 C   rJ  r   )r  r
  r  r&   rq  rF   rF   rG   send_request_telemetry_recordr  rM  z)SendManager.send_request_telemetry_recordendfnamepolicyzinterface.PolicyNamec                 C   s*   t d|| | jr| j|| d S d S )Nzsaving file %s with policy %s)r  rk  r   update_policy)r{   r  r  rF   rF   rG   rR  u  s   zSendManager._save_filec                 C   s8   |j }|j D ]}| tt|jt|j qd S r   )	r   rR  r   rS  globescaperL   file_enum_to_policyr  )r{   r  r   r4  rF   rF   rG   
send_files|  s   

zSendManager.send_filesc                 C   rp  r   rF   rq  rF   rF   rG   send_header  r   zSendManager.send_headerc                 C   rp  r   rF   rq  rF   rF   rG   send_footer  r   zSendManager.send_footerc                 C   rp  r   rF   rq  rF   rF   rG   send_tbrecord  ru  zSendManager.send_tbrecordenvironmentc                 C   r  r   )r   r  rS  )r{   r  rF   rF   rG   _update_environment_record  r  z&SendManager._update_environment_recordc                 C   s|   |  |j tt| j}ttj	
| jjtjd}|| W d   n1 s,w   Y  | jttjdd dS )z>Inject environment info into config and upload as a JSON file.rN  Nnow)r  )r  r  rT   rA  r#   r'   r   r^   r   rL   r   r   r   r!   METADATA_FNAMErQ  rR  r   rS  )r{   r  environment_jsonrc   rF   rF   rG   send_environment  s   zSendManager.send_environmentc                 C   s2  |j js|j jstd|j t|}|jj}|j}|j	}|j
}|j}|j}|j}	|j}
td| d| d| d| d| d|
  |sI|r|r|r|rz| j||||||	|
}|d |jj_W n- ty } z!|
po|}d	| d
| d
| d| |jj_td| W Y d }~nd }~ww | | d S )Nz3Expected either `req_resp` or `mailbox_slot`, got: z!link_artifact params - client_id=z, server_id=z, portfolio_name=z	, entity=z
, project=z, organization=versionIndexzerror linking artifact to "r  z
"; error: z(Failed to link artifact to portfolio: %s)r  r  r  r`   r#   rH  r
  link_artifact	client_id	server_idportfolio_nameportfolio_entityportfolio_projectportfolio_aliasesportfolio_organizationr  r  r   r  link_artifact_responseversion_indexrM  error_messagerN  r.  )r{   r  r,  linkr  r  r  r  r  aliasesorganizationr  rQ  org_or_entityrF   rF   rG   send_request_link_artifact  sl   

	z&SendManager.send_request_link_artifactc                 C   sB   |j }|jdkr|jjsd| j_dS |jjr| j|j dS dS )z|Pretend to send a used artifact.

        This function doesn't actually send anything, it is just used internally.
        jobTN)use_artifacttypepartialjob_namer   r)  set_partial_source_idr   )r{   r  userF   rF   rG   send_use_artifact  s   zSendManager.send_use_artifactc              
   C   s   t |}|jjj}|jjj}z!| ||}|sJ d|d |jj_	t
d|j d|  W n" tyR } zd|j d|j d| |jj_W Y d }~nd }~ww | | d S )NzUnable to send artifactr   zlogged artifact  - zerror logging artifact "r  z": )r#   rH  r
  log_artifactr  history_step_send_artifactr  log_artifact_responseartifact_idr  rk  r   rM  r  r  r.  )r{   r  r,  r  r  resrQ  rF   rF   rG   send_request_log_artifact  s   


z%SendManager.send_request_log_artifactc              
   C   s`   |j }z| |}td|j d|  W d S  ty/   td|j d|j d Y d S w )Nzsent artifact r  z$send_artifact: failed for artifact "r  ")r  r  r  rk  r   rM  r  r  )r{   r  r  r  rF   rF   rG   send_artifact  s   

zSendManager.send_artifactr  r*   r  c                 C   sp  ddl m} | jsJ t| j|jt|j| j|jd}|j	r5| 
 }|d u s.|||dk r5td d S |jr>t|jnd }|jdi d|jd|jd|jd	|jd
|jd|jd|d|jpdd d|jpjd d|jd|jd|jd|j	d|jd|jd|d|jpd }| j || |jj!rt"#t$ t%&|jj! W d    |S 1 sw   Y  |S )Nr   parse)apiri   manifest_jsonfile_pusheris_user_createdz0.10.16zlThis W&B Server doesn't support distributed artifacts, have your administrator install wandb/local >= 0.9.37r  r  r  r   r  sequence_client_idmetadatattl_duration_secondsdescriptionr  ry   use_after_commitdistributed_idfinalizeincrementalr  base_idrF   )'packaging.versionr  r   r   r   ri   rf   rI   user_createdr  _max_cli_versionr  rN  r  rT   rU   saver  r  r  r   r  r  r  r  r  ry   r  r  incremental_beta1r  r   _handle_server_artifactr\   
contextlibsuppressFileNotFoundErrorr   remove)r{   r  r  r  savermax_cli_versionr  r  rF   rF   rG   r    s~   

	

zSendManager._send_artifactc                 C   s   ddl m} |j}|  }|d u s|||dk r td d S z| jj|j|j	|j
|jd W d S  tyD   td|j Y d S w )Nr   r  z0.10.9z]This W&B server doesn't support alerts, have your administrator install wandb/local >= 0.9.31)titletextlevelwait_durationzsend_alert: failed for alert )r  r  alertr  r  rN  r   notify_scriptable_run_alertr  r  r  r  rM  r  )r{   r  r  r  r  rF   rF   rG   
send_alert%  s"   zSendManager.send_alertc                 C   sp   t d |   | jr| j  d | _| jr$| j  | j  d | _| jr1| j| j d | _t	j
  d S )Nzshutting down sender)r  rk  r  r   r   r   r   r   r   r   r;  end_sessionrz   rF   rF   rG   r   :  s   



zSendManager.finishc                 C   s.   |   }|di dd }t|tsd S |S )NcliVersionInfor  )get_server_infor  r1  r   )r{   server_infor  rF   rF   rG   r  K  s   
zSendManager._max_cli_versionc                 C   s&   | j r| jrd S | j \| _| _ d S r   )r   r   r   viewer_server_inforz   rF   rF   rG   get_viewer_server_infoT  s   z"SendManager.get_viewer_server_infoc                 C      | j s|   | j S r   )r   r  rz   rF   rF   rG   get_viewer_infoY     zSendManager.get_viewer_infoc                 C   r  r   )r   r  rz   rF   rF   rG   r  ^  r  zSendManager.get_server_infor-   c                 C   sd   t  }| jjrd|_|S d}|  }|di }|du r"d|_|S |dd|_|d||_|S )a  Queries the server to get the local version information.

        First, we perform an introspection, if it returns empty we deduce that the
        docker image is out-of-date. Otherwise, we use the returned values to deduce the
        state of the local server.
        FlatestlatestLocalVersionInfoN	outOfDateTlatestVersionString)r   r-   r   r9  out_of_dater  r  rX   )r{   
local_infolatest_local_versionr  latest_local_version_inforF   rF   rG   get_local_infoc  s   zSendManager.get_local_infoc                 C   s   | j js| jjr
d S | j | j  | j }|	dd  | j 
| | j j| jd}|d urn| jd urp| j|}| jj|_| jj|_| jj|_|jd | j jD ]}|j| qTd|_d|_d|_| j| d S d S d S )Nr  )r  r  T)r   r)  r   r9  
set_configr   non_internal_configr   rO  r_   set_summarybuildr   r   r   _make_artifactrK  r  r  r  re  _aliasesr  r  r  _publish_artifact)r{   rF  r  proto_artifactaliasrF   rF   rG   r  ~  s(   



zSendManager._flush_jobc                 C   s   | j jddS )NT)block)r   r  rz   rF   rF   rG   __next__  s   zSendManager.__next__)r6   r   )r  r.   r6   N)r  r.   r6   N)r,  r/   r6   N)Fr   )r  r1   )r  r.   r  r  r  r1   r6   Nr   )rx   r2   r6   N)r   r   r6   N)r  )r  r+   r6   N)r6   r-   )r6   r.   )yr   r   r   r   r   r   r   r   r   r   r   r   r   r&   r   r   rp   r   r	   r   r:  r   r"   r  r   r|   classmethodr   r   r   r   r   r   BaseException	tracebackTracebackExceptionr
   r   r   r   r"  r&  r+  r.  r3  r  r  rG  rR  rT  r%  rD  rS  rh  ro  rr  rs  rt  r  r  r  r  r   BackendConfigDictr_  ra  r  r   r  r  r  r  r  r  r
  r  rB  rE  rG  rI  rL  rl  rd  r  r   r   rg  rz  r  r  rp  r  r  r  r  r  r  r   rS  rR  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r   rF   rF   rF   rG   r      sr  
 


R 


	T
]

	b
a(

 '
	*
3	r   )Z__doc__r  r  r]   rT   loggingr   r   r   r   r  collectionsr   r   r   typingr   r   r   r   r	   r
   r   r   r   r   r  r   r   wandb.errorsr   r   wandb.errors.utilr   wandb.filesync.dir_watcherr   wandb.protor   "wandb.sdk.artifacts.artifact_saverr   wandb.sdk.interfacer   #wandb.sdk.interface.interface_queuer   wandb.sdk.internalr   r   r   r   r   wandb.sdk.internal.file_pusherr   wandb.sdk.internal.job_builderr   "wandb.sdk.internal.settings_staticr   wandb.sdk.libr    r!   r"   r#   r$   r%   r&   wandb.sdk.lib.proto_utilr'   wandb.proto.wandb_internal_pb2r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r   	getLoggerr   r  rn  r   rH   rf   rM   rp   r   r   rF   rF   rF   rG   <module>   sT    0$	4
 