o
    niG                     @   sH  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZmZ ddlZddlZddlmZ ddlmZmZ dd	lm Z  dd
l!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z= ddl>m?Z? ddl@mAZA erddlBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZL ed ZMeNeOZPdZQdeeeReRf ddf fddZSdddefddZTd d!defd"d#ZUG d$d% d%ZVG d&d' d'ZWG d(d) d)ZXdS )*zsender.    N)defaultdict)datetime)Queue)
TYPE_CHECKINGAnyDict	GeneratorListLiteralOptionalTupleTypeUnion)util)	CommError
UsageError)ProtobufErrorHandler)
DirWatcher)wandb_internal_pb2)ArtifactSaver)	interface)InterfaceQueue)context	datastorefile_streaminternal_apisender_config)
FilePusher)
JobBuilder)SettingsStatic)config_util	filenames
filesystem
proto_utilredirect	telemetry)ContextCancelledError)message_to_dict)
ArtifactManifestArtifactManifestEntryArtifactRecordHttpResponse	LocalInfoRecordResultRunExitResult	RunRecordSummaryRecord)stdoutstderr   returnc                   c   s    g dE d H  d S )N))lightgbmr6   )catboostr7   )xgboostr8   )transformers_huggingfacehuggingface)transformersr:   )pytorch_igniteignite)r=   r=   )pytorch_lightning	lightning)fastair@   )torchrA   )kerasrB   )
tensorflowrC   )sklearnrD    rE   rE   rE   W/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/internal/sender.py_framework_priorityS   s   rG   manifestr(   c                 C   s   | j dkr>| jr5i }t| jd}|D ]}t|}|d}|||< qW d    n1 s/w   Y  ndd | jD }ntd| j  | j | j	dd | j
D |dS )	N   rtpathc                 S   s   i | ]}|j t|qS rE   )rK   _manifest_entry_from_proto).0contentrE   rE   rF   
<dictcomp>o   s    
z-_manifest_json_from_proto.<locals>.<dictcomp>z#unknown artifact manifest version: c                 S      i | ]
}|j t|jqS rE   keyjsonloads
value_json)rM   configrE   rE   rF   rO   y   s    )versionstoragePolicystoragePolicyConfigcontents)rW   manifest_file_pathgzipopenrS   rT   poprZ   
ValueErrorstorage_policystorage_policy_config)rH   rZ   fline
entry_jsonrK   rE   rE   rF   _manifest_json_from_protoe   s,   



re   entryr)   c                 C   s\   | j r| j nd }| j|| jr| jnd | jd ur| jnd | jr!| jnd | jdd | jD dS )Nc                 S   rP   rE   rQ   )rM   extrarE   rE   rF   rO      s    z._manifest_entry_from_proto.<locals>.<dictcomp>)digestbirthArtifactIDrefsize
local_path
skip_cacherg   )birth_artifact_idrh   rj   rk   rl   rm   rg   )rf   rn   rE   rE   rF   rL      s   rL   c                   @   s   e Zd ZU eed< eed< eed< eed< eed< eed< ee ed< eee	e
f  ed< eee	e
f  ed	< eee	  ed
< dddZde	fddZdS )ResumeStateresumedstephistoryeventsoutputruntimewandb_runtimesummaryrV   tagsr5   Nc                 C   s@   d| _ d| _d| _d| _d| _d| _d | _d | _d | _d | _	d S )NFr   )
rp   rq   rr   rs   rt   ru   rv   rw   rV   rx   selfrE   rE   rF   __init__   s   
zResumeState.__init__c                 C   s(   d tdd t|  }d| dS )N,c                 S   s   | d  d| d  S )Nr   =rI   rE   )itrE   rE   rF   <lambda>   s    z%ResumeState.__str__.<locals>.<lambda>zResumeState())joinmapvarsitems)rz   objrE   rE   rF   __str__   s   zResumeState.__str__r5   N)__name__
__module____qualname__bool__annotations__intfloatr   r   strr   r	   r{   r   rE   rE   rE   rF   ro      s   
 
ro   c                   @   s\   e Zd ZU ejed< ejed< ej	ed< ej
ed< ej
ed< deddfd	d
ZdddZdS )_OutputRawStream_stopped_queue	_emulator_writer_thr_reader_thrstreamsmSendManagerc                 C   sf   t  | _t | _t | _t j	|j
t|ddd| d| _t j	|jt|ddd| d| _d S )N)r   Tz	OutRawWr-)targetkwargsdaemonnamez	OutRawRd-)	threadingEventr   queuer   r   r$   TerminalEmulatorr   Thread_output_raw_writer_threaddictr   _output_raw_reader_threadr   )rz   r   r   rE   rE   rF   r{      s   


z_OutputRawStream.__init__r5   Nc                 C   s   | j   | j  d S N)r   startr   ry   rE   rE   rF   r      s   
z_OutputRawStream.startr   )r   r   r   r   r   r   r   r   r$   r   r   r   r{   r   rE   rE   rE   rF   r      s   
 




r   c                   @   sd  e Zd ZU dZeed< dZeed< eed< ded< ded	< eed
< e	e
e
f ed< e	e
e
f ed< ejed< ejed< ed ed< ed ed< ee
 ed< ee
 ed< ed ed< ed ed< ed ed< ed ed< eed< ee	e
ef  ed< e	e
ef ed< e	e
ef ed < ee	e
ef  ed!< eej ed"< e	d#ef ed$< eej ed%< eed&< eed'< eed(< eed)< d*ed+dd,dd-ed.ejd/d0fd1d2Zed3e
d4ed0ee
f d/d fd5d6Zd/efd7d8Zdd9d:Zd;ee e!  d<ee! d=ee"j# d/e$d> fd?d@Z%dAedBe
d/d0fdCdDZ&ddFdGZ'ddIdJZ(ddKdLZ)ddMdNZ*ddQdRZ+dSe	d/d0fdTdUZ,dVed/d0fdWdXZ-dYed/d0fdZd[Z.dd\d]Z/dd^d_Z0d d`ed/d0fdadbZ1d d`ed/d0fdcddZ2d deed/d0fdfdgZ3ddhdiZ4ddjdkZ5ddldmZ6ddndoZ7ddpdqZ8ddrdsZ9ddtduZ:ddvdwZ;ddxdyZ<dzdd/ed{ fd|d}Z=d/e
fd~dZ>d/e?j@fddZAde?j@d/d0fddZBdddZCdeDfddZEdddZFdd ZG	E			{	z		/	0dddZHddEddee
 d/d0fddZIdedefddZJdzddee?j@ d/eDfddZKddee
 d/d0fddZLde	e
ef d/d0fddZMdddZNdddZOdddZPdddZQdddZRdddZSdddZTdddZUdddZV	0ddd#dee
 d/d0fddZWdddZXdddZYdddZZdd#de
d/d0fddZ[dddZ\dddĄZ]dddƄZ^dejd/d0fddɄZ_ddd˄Z`ddd̈́Za	ΐddebjcddd/d0fddӄZddddՄZedddׄZfdddلZgdddۄZhddd݄Ziddd߄ZjdddZkdddZl	0ddddee d/ee	 fddZmdddZndddZod/ee
 fddZpdddZqd/e	e
ef fddZrd/e	e
ef fddZsdddZtdddZud	ddZvevZwd0S (
  r      UPDATE_CONFIG_TIME   UPDATE_STATUS_TIME	_settingszQueue[Record]	_record_qzQueue[Result]	_result_q
_interface_api_settings_partial_output_context_keeper_telemetry_objzfile_stream.FileStreamApi_fsr0   _run_entity_projectr   _dir_watcherr   _pusherr-   _record_exitr/   _exit_result_resume_state_rewind_response_cached_server_info_cached_viewer_server_messages_dsStreamLiterals_output_raw_streams_output_raw_file_send_record_num_send_end_offset_debounce_config_time_debounce_status_timesettingsrecord_qresult_qr   context_keeperr5   Nc                 C   s6  || _ || _|| _|| _|| _d | _d| _d| _d | _d | _	d | _
d | _d | _d | _d | _t | _d| _t | _g | _t | _t | _i | _i | _t | _t | _t | _g | _ t! | _"d | _#d | _$d | _%t&j'|| j(d| _)t | _*t+, | _-d| _.t | _/d| _0t | _1d | _2t3|| _4t56 }|| _7|| _8d S )Nr   )default_settingsretry_callbackF)9r   r   r   r   r   r   r   r   r   r   r   r   _flagsr   r   r   ConfigState_consolidated_config_start_timer%   TelemetryRecordr   _config_metric_pbdict_listr   _metadata_summaryr   _cached_summary_config_metric_index_dict_config_metric_dict_consolidated_summaryr   r   r   ro   r   r   r   r   r   Apir   _apir   r   r   _retry_q_config_needs_debouncer   
_exit_coder   r   r   _job_buildertime	monotonicr   r   )rz   r   r   r   r   r   time_nowrE   rE   rF   r{      sZ   




zSendManager.__init__root_dirresumec           	      C   sh   t | tj|d}tj|||dddd}t }t }t|d}t	
 }tt| ||||dS )zQSet up a standalone SendManager.

        Exclusively used in `sync.py`.
        filesTFr   )x_files_dirr   r   x_syncdisable_job_creationx_file_stream_timeout_seconds)r   )r   r   r   r   r   )printosrK   r   wandbSettingsr   r   r   r   ContextKeeperr   r   to_proto)	clsr   r   	files_dirr   r   r   publish_interfacer   rE   rE   rF   setup<  s*   



zSendManager.setupc                 C   s
   | j  S r   )r   qsizery   rE   rE   rF   __len__^     
zSendManager.__len__c                 C   s   | S r   rE   ry   rE   rE   rF   	__enter__a     zSendManager.__enter__exc_type	exc_valueexc_tracebackFc                 C   s&   | rt | }| | | s|   dS )NF)nextsendfinish)rz   r   r   r   datarE   rE   rF   __exit__d  s   
zSendManager.__exit__statusresponse_textc                 C   s$   t  }||_||_| j| d S r   )r   r+   http_status_codehttp_response_textr   put)rz   r   r  responserE   rE   rF   r   p  s   zSendManager.retry_callbackrecordc                 C   s   |  |j | |jj |d}|sJ d| }t| |d }|dvr,td|  |s5J d| t	
|}| j|}z4z| j| || W n tyd   td|  | j| Y n	w W | j  d S W | j  d S | j  w )Nrecord_typesend_>   rt   request
output_rawzsend: zunknown send handler: zRecord cancelled: )_update_record_numnum_update_end_offsetcontrol
end_offset
WhichOneofgetattrloggerdebugr   context_id_from_recordr   getr   set_local_contextr&   releaseclear_local_context)rz   r  r  handler_strsend_handler
context_idapi_contextrE   rE   rF   r   v  s,   

zSendManager.send_c                 C   s   | j r
| j   d S d S r   )r   enqueue_preemptingrz   r  rE   rE   rF   send_preempting  s   zSendManager.send_preemptingc                 C   s   | j dd d S )NTalways)_maybe_report_statusr  rE   rE   rF   send_request_sender_mark     z$SendManager.send_request_sender_markc                 C   s^   |j d}|s
J d| }t| |d }|dkr td|  |s)J d| || d S )Nrequest_typesend_request_network_statuszsend_request: zunknown handle: )r	  r  r  r  r  )rz   r  r&  r  r  rE   rE   rF   send_request  s   zSendManager.send_requestresultr.   c                 C   s&   t |}| j| | j| d S r   )r   context_id_from_resultr   r  r   r  )rz   r*  r  rE   rE   rF   _respond_result  s   
zSendManager._respond_result
dictionaryc                 C   sh   t |tr0t| D ]&\}}t |tr/| | || | D ]\}}|||d | < q"qd S d S )N.)
isinstancer   listr   _flattenr^   )rz   r-  kvk2v2rE   rE   rF   r1    s   



zSendManager._flatten
record_numc                 C   s4   |sd S | j js| j js|| jd ksJ || _d S )NrI   )r   _offliner   r   )rz   r6  rE   rE   rF   r    s
   
zSendManager._update_record_numr  c                 C   s   |sd S || _ d S r   )r   )rz   r  rE   rE   rF   r    s   
zSendManager._update_end_offsetc                 C   s   | j d u rt | _ | j | jj |jjj}|jjj	}| j 
| d}||k rQ| j  }|s1J | j  }t }|| | | | | |   ||k s(| jdd d S )Nr   Tr!  )r   r   	DataStoreopen_for_scanr   	sync_filer	  sender_readstart_offsetfinal_offsetseek	scan_data
get_offsetr   r-   ParseFromStringr  r   debouncer#  )rz   r  r<  r=  current_end_offsetr   send_recordrE   rE   rF   send_request_sender_read  s$   








z$SendManager.send_request_sender_readc              
   C   s   t |}|jj}d|_| jrA| jrA| jrA| jjrAz| j	
| j| j| jj|_W n ty@ } ztd| W Y d }~nd }~ww | | d S )NFz)Failed to check stop requested status: %s)r#   _result_from_recordr  stop_status_responserun_should_stopr   r   r   run_idr   check_stop_requested	Exceptionr  warningr,  rz   r  r*  status_resperE   rE   rF   send_request_stop_status  s   

z$SendManager.send_request_stop_statusr"  c                 C   s8   t  }|s|| j| j k rd S | jr|   || _d S r   )r   r   r   r   r   _debounce_config)rz   r"  r   rE   rE   rF   _maybe_update_config  s   
z SendManager._maybe_update_configc                 C   sr   t  }|s|| j| j k rd S || _tj| j| jd}t   }|j	t
|d  | jj|d}| j| d S )N)r6  sent_offset    .A)status_report)r   r   r   r   r   StatusReportRequestr   r   	sync_timeFromMicrosecondsr   r   _make_request_publish)rz   r"  r   rU  status_timer  rE   rE   rF   r#    s   z SendManager._maybe_report_statusfinalc                 C   s   | j |d | j|d d S )Nr!  )r#  rR  )rz   r\  rE   rE   rF   rB    s   zSendManager.debouncec                 C   s@   |   }| jr| jjd| jj|d| j | | d| _d S )N)r   rV   FrE   )_config_backend_dictr   r   
upsert_runrI  r   _config_saver   )rz   config_value_dictrE   rE   rF   rQ    s   

zSendManager._debounce_configc              
   C   s   t |}|jj}	 z|j| j  W n# tj	y   Y n t
y8 } ztd|  W Y d }~nd }~ww q
| | d S )NTzError emptying retry queue: )r#   rF  r  network_status_responsenetwork_responsesappendr   
get_nowaitr   EmptyrK  r  rL  r,  rM  rE   rE   rF   send_request_network_status  s   
z'SendManager.send_request_network_statusc                 C   sp   | j   |  }|  }|rtd|  |d| _|jj	r6t
|}| jr/| j|jj_| | d S d S )NzLogin server info: entity)r   reauthget_viewer_infoget_server_infor  infor  r   r  req_respr#   rF  r  login_responseactive_entityr,  )rz   r  viewerserver_infor*  rE   rE   rF   send_request_login$  s   

zSendManager.send_request_loginc                 C   s`   || _ |j}|j| _td|j |j}td|j || jd< |   td | j	
  d S )Nzhandling exit code: %szhandling runtime: %sru   z
send defer)r   exit	exit_coder   r  rk  ru   r   _update_summaryr   publish_defer)rz   r  run_exitru   rE   rE   rF   	send_exit5  s   

zSendManager.send_exitc                 C      d S r   rE   rz   r  rE   rE   rF   
send_finalF  r   zSendManager.send_finalc                 C   rx  r   rE   ry   rE   rE   rF   
_flush_runI  r   zSendManager._flush_runc                 C   rx  r   rE   ry  rE   rE   rF   send_request_status_reportL     z&SendManager.send_request_status_reportc                    s*  |j j  j}td|  d	 fdd}d}| jkr"|  n| jkr/  |  n| jkr8|  n| j	krA|  n| j
krJ|  n| jkrS|  n| jkrbjdd |  n| jkro  |  nz| jkr|  |  nm| jkrjrj  d _|  nY| jkrjrj| nJ|  nF| jkrjrj  |  n5| jkrʈjrƈjj d _|  n| jkr݈j  j  |  n| j krd}nt!d|sd S t"# }|_$j%rj%j&j'rt()j%}|j*+| ,| d S d S d S )
Nzhandle sender defer: r5   c                     s*    j d } td|   j|  d S )NrI   zsend defer: )stater  rk  r   ru  )r~  deferrz   rE   rF   transition_stateU  s   
z8SendManager.send_request_defer.<locals>.transition_stateFT)r\  zunknown stater   )-r	  r  r~  r  rk  BEGIN	FLUSH_RUNr{  FLUSH_STATSFLUSH_PARTIAL_HISTORYFLUSH_TB	FLUSH_SUMFLUSH_DEBOUNCERrB  FLUSH_OUTPUT_output_raw_finish	FLUSH_JOB
_flush_job	FLUSH_DIRr   r   FLUSH_FPr   JOIN_FPr   FLUSH_FSr   r   FLUSH_FINALr   publish_finalpublish_footerENDAssertionErrorr   r/   r   r   r  mailbox_slotr#   rF  exit_resultCopyFromr,  )rz   r  r~  r  doner  r*  rE   r  rF   send_request_deferP  sz   


















zSendManager.send_request_deferc                 C   s   |j js
|j js
d S t|}| jrE| j \}}| j }|jj	}|j
|j_
|j|j_|j|j_|j|j_|j|j_|j|j_|j|j_| jrVd|jj	_|jj	j| j | | d S NT)r  rl  r  r#   rF  r   
get_statusfile_counts_by_categoryr  poll_exit_responseuploaded_bytespusher_statstotal_bytesdeduped_bytesr   file_countswandb_countmediamedia_countartifactartifact_countotherother_countr   r  r  r  r,  )rz   r  r*  _aliver   r  resprE   rE   rF   send_request_poll_exit  s$   









z"SendManager.send_request_poll_exitrunwandb_internal_pb2.ErrorInfoc              
   C   s  | j jsdS |jp| j}td||j|j | jj	||j|jd}|r+d|
ddvrG| j jdkrEt }tjjj|_d|j d	|_|S dS | j jd
krat }tjjj|_d|j d|_|S i }i }i }i }z]d}	d}
t|d }|rt|d }|
dd}
t|d }|rt|d }|
dd}	t|d pd}t|d pd}|
di 
dd}|dur|| j_|
dpg }W n; ttfy } z,tjd|d | j jdkrt }tjjj|_d|j|_|W  Y d}~S W Y d}~nd}~ww t|	|
| j_|
dd}|d }|dkr|d n|| j_|| j_|d | j_|d | j_|| j_|| j_ || j_!d | j_"td!| j dS )"zEQueries the backend for a run; fail if the settings are incompatible.Nz#checking resume status for %s/%s/%s)rg  project_namer   z"t":wandbConfig mustztYou provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming a run (z) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`.neverzuYou provided an invalid value for the `resume` argument. The value 'never' is not a valid option for resuming a run (zk) that already exists. Please check your inputs and try again with a valid value for the `resume` argument.r   historyTail_runtime
eventsTailrV   {}summaryMetrics_wandbru   rx   zunable to load resume tailsexc_infoz(resume='must' but could not resume ({}) _stephistoryLineCountrI   eventsLineCountlogLineCountTzconfigured resuming with: {})#r   r   rg  r   r  rk  projectrI  r   run_resume_statusr  r   	ErrorInfo	ErrorCodeUSAGEcodemessagerS   rT   r   rv   
IndexErrorr_   errorformatmaxru   rq   rr   rs   rt   rV   rw   rx   rp   )rz   r  rg  resume_statusr  rr   rs   rV   rw   	events_rt
history_rtnew_runtimerx   rO  	last_stephistory_line_countrE   rE   rF   _setup_resume  s   zSendManager._setup_resumec                    sN   | j dr| j j n| j dr| j j ndS t fddt D d}|S )z1Get telemetry data for internal config structure.imports_finishimports_initr  c                 3   s$    | ]\}}t  |d r|V  qdS )FN)r  )rM   rb   nimportsrE   rF   	<genexpr>&  s   " z7SendManager._telemetry_get_framework.<locals>.<genexpr>)r   HasFieldr  r  r   rG   )rz   	frameworkrE   r  rF   _telemetry_get_framework  s   

z$SendManager._telemetry_get_frameworkc                 C   s*   | j pt }|j| j|  | j| jdS )N)telemetry_recordr  start_time_millismetric_pbdicts)r   r   r   to_backend_dictr   r  r   r   )rz   rV   rE   rE   rF   r]  *  s   z SendManager._config_backend_dictr`  c                 C   s"   t j| jjd}t|| d S )Nzconfig.yaml)r   rK   r   r   r   r    save_config_file_from_dict)rz   r`  config_pathrE   rE   rF   r_  4  s   zSendManager._config_savec                 C   s   | j sdS z5tj}| jjd|dd d| jj| j j	| j j
| j j}tj|ddd |d	|d
dd W dS  tjyE   Y dS w )zSync this run with spell.N)r  	spell_urlSPELL_RUN_URL)rR   valz{}/{}/{}/runs/{}SPELL_API_URLzhttps://api.spell.runz
/wandb_urlWANDB_ACCESS_TOKEN)access_tokenurlr4   )rS   timeout)r   r   environr   publish_configr  r  r   app_urlrg  r  rI  requestsr  RequestException)rz   envr  rE   rE   rF   _sync_spell;  s$   
zSendManager._sync_spell
server_runc                 C   sf   | j jsJ | j jjdksJ | jsJ t| j jjd }|| j_|dd| j_	d| j_
|| j_d S )Nr  rI   r  r   T)r   	fork_frommetricr   r   valuer   rq   r  rr   forkedstarting_step)rz   r  
first_steprE   rE   rF   _setup_forkP  s   
zSendManager._setup_forkc                 C   sv   | j jsJ | jj|j|jpd |jpd | j jj| j jj| j j	p d d| _
| j
dd| j_t| j
dd| j_d S )N)run_namerg  r  metric_namemetric_valueprogram_pathr  r   rV   r  )r   resume_fromr   
rewind_runrI  rg  r  r  r  programr   r  r   rr   rS   rT   rV   )rz   r  rE   rE   rF   _load_rewind_stateZ  s   
zSendManager._load_rewind_statec                 C   s`   | j jsJ | j jjdksJ | jsJ | jsJ t| j jjd }|| j_d| j_	|| j_
d S )Nr  rI   T)r   r  r  r   r   r   r  r   rq   r  r  )rz   r  rE   rE   rF   _install_rewind_statei  s   

z!SendManager._install_rewind_stater  c                 C   sV   |j js|j jr"t|}|jj| |jj| | 	| d S t
d|j d S )NzGot error in async mode: %s)r  rl  r  r#   rF  
run_resultr  r  r  r,  r  r  )rz   r  r  r  r*  rE   rE   rF   _handle_errorx  s   
zSendManager._handle_errorfile_dirc              
   C   sx  |j }d }| jd u }t|j d | _|jr| j|j | j	j
r'd| jj_d }|jr<| j|j |  }| | | j	jd uoC|}| j	jd uoK|}t| j	j}	t|||	g}
|
dkrqt }tjjj|_d|_| ||| |r|jdkrt | j	j!|_|	r| "|}n|r| #|}|d ur| ||| d S | j$jd ur| j%t&'| j$j |  }| | |s|  }| | z| (||}W n( t)t*fy } zt+j,|dd t-.|}| ||| W Y d }~d S d }~ww | jsJ |r| /|}|d ur| ||| d S |j0j1s|j0j2r(t34|}|j5j 6| j | 7| |r2| 8| d S t+9d| jj: d S )NrT  TrI   zfMultiple resume options specified. Please specify only one of `fork_from`, `resume`, or `resume_from`.r  r  zupdated run: %s);r  r   r   
start_timeToMicrosecondsr   r%   r   	MergeFromr   r   featuresyncrV   r   update_from_protor]  r_  r  r  r   r   sumr   r  r  r  r  r  r  r  r   auto_project_namer   r  r  r   merge_resumed_configr    dict_strip_value_dict	_init_runr   r   r  r  r   from_exceptionr  r  rl  r  r#   rF  r  r  r,  _start_run_threadsrk  rI  )rz   r  r  r  r  is_wandb_initr`  do_fork	do_rewind	do_resumenum_resume_options_setr  rO  r*  rE   rE   rF   send_run  s   












zSendManager.send_runis_rewindinginsertedc                 C   s`   | j sJ | jjrd| j _| jjd ur| jj| j _d S d S |r%|   d S |s.d| jj_d S d S r  )	r   r   rp   rv   ru   r  r   r	  maybe_run_overwrite)rz   r  r  rE   rE   rF   _update_resume_state  s   
z SendManager._update_resume_stateconfig_dictc                 C   s  |j  d | jj }| jr| jjr|js|j| jj t| jj}|r2| j	s*J | j	}d }d}nE| j
j|j|jp;d |jp?d |jpCd |jpGd |jpKd |jpOd |jd d  pWd |pZd |jp^d |jpbd | jjpgd |jjpld |jjpqd d\}}}|jr~d| j_|pg | _|| _| jjr|rtd| || | jj | j_!| jj "t#|d  | jj$%| j&'| | jj(d ur| jj(%| j&)| jj( |*d}|r|| j_+|*d}	|	r| j
,|	 |*d}
|
r|
| j_|*d}|r.|*d}|r|| j_|| _-|| j.d< | j
/d| |*d	}|r.|*d}|r.|| j_|| _0|| j.d	< | j
/d	| |*d
}|r:|| j_t12drD| 3  |S )NrT  T)r   rg  r  groupjob_typedisplay_namenotesrx   rV   
sweep_namehostr  repocommitzcCannot attempt to rewind and resume a run - only one of `resume` or `resume_from` can be specified.idr   displayNamer  rg  	sweepNamer  )4r  r  r   ru   rx   extendr   r   r  r   r   r^  rI  rg  r  	run_groupr  r   r!  sweep_idr#  r   git
remote_urlr%  r   disabler   r   rp   r_   r  rq   r  rX  r   rV   r  r   _make_configrw   _make_summary_from_dictr  
storage_idset_current_run_idr   r   set_settingr   r   getenvr  )rz   r  r  r  r  r  server_messagesr  r1  r&  r   r  r  rg  entity_namer+  rE   rE   rF   r    s   














zSendManager._init_runc                 C   s  | j sJ tj| j| j j| j j d | jjpd| j	d| _
| j
dt  | j
dtj| jjd | j
dtj| jjd | j
dtj| jjd t| j }t| j}|| tjj|d	d
 | j
  t| j| j
| jd| _t| j| j|| _td| j j| j j d  d S )NrT  r   )r  r   zwandb-summary.jsonzwandb-history.jsonl)start_chunk_idzwandb-events.jsonlz
output.loginternal)rx   process_context)r   z"run started: %s with start time %s) r   r   FileStreamApir   rI  r  r  r   r   r   r   set_file_policySummaryFilePolicyJsonlFilePolicyr   rr   rs   CRDedupeFilePolicyrt   r'   r   updater   _sentryconfigure_scoper   r   r   r   r   r  rk  )rz   r  run_settingsr   rE   rE   rF   r  ]  sB   





zSendManager._start_run_threadshistory_dictc                 C   s$   | j r| j tjt| d S d S r   )r   pushr!   HISTORY_FNAMErS   dumps)rz   rC  rE   rE   rF   _save_history  s   zSendManager._save_historyc                 C   s    |j }t|j}| | d S r   )rr   r#   dict_from_proto_listitemrG  )rz   r  rr   rC  rE   rE   rF   send_history  s   zSendManager.send_historyrw   r1   c                 C   s   t |j}|| _|   d S r   )r#   rH  r?  r   rt  )rz   rw   summary_dictrE   rE   rF   _update_summary_record  s   z"SendManager._update_summary_recordc                 C      |  |j d S r   )rL  rw   ry  rE   rE   rF   send_summary  r%  zSendManager.send_summaryc                 C      |  |jjj d S r   )rL  r	  summary_recordrw   ry  rE   rE   rF   send_request_summary_record     z'SendManager.send_request_summary_recordc                 C   s   | j  }|dd  | jr| j|d< | j| t| j}| jr*| j	t
j| tj| jjt
j}t|d}|| W d    n1 sIw   Y  | tt
j d S )Nr  w)r   copyr^   r   r   r?  rS   rF  r   rD  r!   SUMMARY_FNAMEr   rK   r   r   r   r]   write
_save_filer   GlobStr)rz   rK  json_summarysummary_pathrb   rE   rE   rF   rt    s   

zSendManager._update_summaryc              	   C   s   |j }|jtjjjkrd S | jsd S | jsd S |j	 }| jj
	 }t }|jD ]}zt|j||j< W q( tjyF   td|j Y q(w t|d}| | d|d< |d |d< || d |d< | jtjt| d S )Nzerror decoding stats json: %s)systemTr  rT  
_timestampr  )stats
stats_typer   StatsRecord	StatsTypeSYSTEMr   r   	timestampr  r  r   rI  rS   rT   rU   rR   JSONDecodeErrorr  r  r1  rD  r!   EVENTS_FNAMErF  )rz   r  r]  now_usstart_usdrI  rowrE   rE   rF   
send_stats  s,   



zSendManager.send_statsc                 C   s   | j  D ])\}}|j  |jjdd |j r$td |j  |j	  | 
| qi | _ | jr?| j  d | _d S d S )Nr   )r  zprocessing output...)r   r   r   setr   r   is_aliver  rk  r   _output_raw_flushr   closerz   r   r
  rE   rE   rF   r    s   






zSendManager._output_raw_finishr   c              
   C   s   	 | j | }|j r|j rd S td q g }|j s,||j  |j r|j rQt	t
t|dkrQtd | | |D ]}| || qFd S z|jd| W n tyv } ztd|  W Y d }~nd }~ww q)NTg      ?i z6Terminal output too large. Logging without processing.r  z(problem writing to output_raw emulator: )r   r   emptyr   is_setr   sleeprc  r  r  r   lenr  rL  rl  r   rV  r   rK  )rz   r   r
  r   rc   rO  rE   rE   rF   r     s0   







z%SendManager._output_raw_writer_threadc                 C   sN   | j | }|j r|j s%| | tt |j r|j rd S d S r   )	r   r   rp  r   ro  rl  r   rq  _OUTPUT_MIN_CALLBACK_INTERVALrn  rE   rE   rF   r     s
   


z%SendManager._output_raw_reader_threadr   c              
   C   s   |d u r+| j | }z|j }W n ty* } ztd|  W Y d }~nd }~ww |rA| || | jrC| j|	d d S d S d S )Nz*problem reading from output_raw emulator: zutf-8)
r   r   readrK  r  rL  _send_output_liner   rV  encode)rz   r   r   r
  rO  rE   rE   rF   rl    s   
zSendManager._output_raw_flushc                 C   sz   dd l }ddlm} tdd |jjjD }t|j	| j
j|d}|d	| W d    d S 1 s6w   Y  d S )Nr   )REQUIREMENTS_FNAMEc                 s   s"    | ]}|j  d |j V  qdS )z==N)r   rW   )rM   rrE   rE   rF   r    s    
z;SendManager.send_request_python_packages.<locals>.<genexpr>rS  
)r   wandb.sdk.lib.filenamesrw  sortedr	  python_packagespackager]   rK   r   r   r   rV  )rz   r  r   rw  installed_packages_listrb   rE   rE   rF   send_request_python_packages  s   "z(SendManager.send_request_python_packagesc                 C   s>   | j sd S |j}d}|jtjjjkrd}|j}| || d S )Nr2   r3   )	r   rt   output_typer   OutputRecord
OutputTypeSTDERRrc   ru  )rz   r  outr   rc   rE   rE   rF   send_output	  s   zSendManager.send_outputc           	   
   C   s   | j sd S |j}d}|jtjjjkrd}|j}| j	|}|sgt
|| d}|| j|< | jsctj| jjtj}d }z
tt|d}W n ty] } ztd|  W Y d }~nd }~ww |rc|| _|  |j| d S )Nr2   r3   )r   r   wbz could not open output_raw_file: )r   r
  r  r   OutputRawRecordr  r  rc   r   r  r   r   r   rK   r   r   r   r!   OUTPUT_FNAMEr"   CRDedupedFiler]   OSErrorr  rL  r   r   r  )	rz   r  r  r   rc   r
  output_log_pathoutput_raw_filerO  rE   rE   rF   send_output_raw  s8   

zSendManager.send_output_rawrc   c                 C   s   d}|dkrd}| ds)| j|d |drd| j|< | j|  |7  < dS t }t| d }| j|d}| | | | }| j	rR| j	
tj| d| j|< dS )zyCombined writer for raw and non raw output lines.

        This is combined because they are both post emulator.
        r  r3   zERROR ry   N)endswithr   
setdefault
startswithr   r   utcfromtimestamp	isoformatr  r   rD  r!   r  )rz   r   rc   prependcur_timerb  prev_strrE   rE   rF   ru  3  s   


zSendManager._send_output_linec                 C   s
   d| _ d S r  )r   ry   rE   rE   rF   _update_configQ  r   zSendManager._update_configc                 C   s   | j |j |   d S r   )r   r  rV   r  ry  rE   rE   rF   send_configT  s   zSendManager.send_configc           	      C   s   |j }|jrtd d S | j|jt }|j	j
r!|| n|| || j|j< |}|jrS| j|j}|d urSt }|j | |j }|d |d |_t|}| j|j}|d uri|| j|< nt| j}| j| || j|j< |   d S )Nz(Seen metric with glob (shouldn't happen)step_metricrI   )r  	glob_namer  rL  r   r  r   r   MetricRecord_control	overwriter  r  r  r   r-   
ClearFieldstep_metric_indexr#   proto_encode_to_dictr   rr  rc  r  )	rz   r  r  
old_metricfind_step_idxrecmdfind_idxnext_idxrE   rE   rF   send_metricX  s8   






zSendManager.send_metricr%   c                 C   s   | j | |   d S r   )r   r  r  )rz   r%   rE   rE   rF   _update_telemetry_record  s   z$SendManager._update_telemetry_recordc                 C   rM  r   )r  r%   ry  rE   rE   rF   send_telemetry  r%  zSendManager.send_telemetryc                 C   rO  r   )r  r	  r  r%   ry  rE   rE   rF   send_request_telemetry_record  rR  z)SendManager.send_request_telemetry_recordendfnamepolicyzinterface.PolicyNamec                 C   s*   t d|| | jr| j|| d S d S )Nzsaving file %s with policy %s)r  rk  r   update_policy)rz   r  r  rE   rE   rF   rW    s   zSendManager._save_filec                 C   s2   |j }|j D ]}| t|jt|j qd S r   )r   rW  r   rX  rK   file_enum_to_policyr  )rz   r  r   r2  rE   rE   rF   
send_files  s   
zSendManager.send_filesc                 C   rx  r   rE   ry  rE   rE   rF   send_header  r   zSendManager.send_headerc                 C   rx  r   rE   ry  rE   rE   rF   send_footer  r   zSendManager.send_footerc                 C   rx  r   rE   ry  rE   rE   rF   send_tbrecord  r}  zSendManager.send_tbrecordc                 C   s$  |j js|j jstd|j t|}|jj}|j}|j	}|j
}|j}|j}|j}	|j}
td| d| d| d| d| d|
  |sI|r|r|r|rz| j||||||	|
 W n- ty } z!|
ph|}d| d	| d	| d
| |jj_td| W Y d }~nd }~ww | | d S )Nz3Expected either `req_resp` or `mailbox_slot`, got: z!link_artifact params - client_id=z, server_id=z, portfolio_name=z	, entity=z
, project=z, organization=zerror linking artifact to "/z
"; error: z(Failed to link artifact to portfolio: %s)r  rl  r  r_   r#   rF  r	  link_artifact	client_id	server_idportfolio_nameportfolio_entityportfolio_projectportfolio_aliasesportfolio_organizationr  r  r   rK  r  log_artifact_responseerror_messagerL  r,  )rz   r  r*  linkr  r  r  rg  r  aliasesorganizationrO  org_or_entityrE   rE   rF   send_request_link_artifact  sf   

	z&SendManager.send_request_link_artifactc                 C   sB   |j }|jdkr|jjsd| j_dS |jjr| j|j dS dS )z|Pretend to send a used artifact.

        This function doesn't actually send anything, it is just used internally.
        jobTN)use_artifacttypepartialjob_namer   r.  set_partial_source_idr&  )rz   r  userE   rE   rF   send_use_artifact  s   zSendManager.send_use_artifactc              
   C   s   |j jsJ t|}|jjj}|jjj}z!| ||}|s"J d|d |j	j
_td|j d|  W n" tyX } zd|j d|j d| |j	j
_W Y d }~nd }~ww | | d S )NzUnable to send artifactr&  zlogged artifact  - zerror logging artifact "r  z": )r  rl  r#   rF  r	  log_artifactr  history_step_send_artifactr  r  artifact_idr  rk  r   rK  r  r  r,  )rz   r  r*  r  r  resrO  rE   rE   rF   send_request_log_artifact  s   


z%SendManager.send_request_log_artifactc              
   C   sn   |j }z| |}td|j d|  W d S  ty6 } ztd|j|j| W Y d }~d S d }~ww )Nzsent artifact r  z.send_artifact: failed for artifact "{}/{}": {})	r  r  r  rk  r   rK  r  r  r  )rz   r  r  r  rO  rE   rE   rF   send_artifact  s   

zSendManager.send_artifactr  r*   r  c                 C   s"  ddl m} | jsJ t| j|jt|j| j|jd}|j	r5| 
 }|d u s.|||dk r5td d S |jr>t|jnd }|j|j|j|j|j||jpOd |jpSd |j|j|j|j	|j|j||jpdd d}| j|| |jjrt !t" t#$|jj W d    |S 1 sw   Y  |S )Nr   parse_version)apirh   manifest_jsonfile_pusheris_user_createdz0.10.16zlThis W&B Server doesn't support distributed artifacts, have your administrator install wandb/local >= 0.9.37)r  r   r  sequence_client_idmetadatattl_duration_secondsdescriptionr  rx   use_after_commitdistributed_idfinalizeincrementalr  base_id)%
wandb.utilr  r   r   r   rh   re   rH   user_createdr  _max_cli_versionr  rL  r  rS   rT   saver  r   r  r  r  r  r  rx   r  r  incremental_beta1r  r   _handle_server_artifactr[   
contextlibsuppressFileNotFoundErrorr   remove)rz   r  r  r  savermax_cli_versionr  r  rE   rE   rF   r    s\   


zSendManager._send_artifactc              
   C   s   ddl m} |j}|  }|d u s|||dk r td d S z| jj|j|j	|j
|jd W d S  tyP } ztd|jd|  W Y d }~d S d }~ww )Nr   r  z0.10.9z]This W&B server doesn't support alerts, have your administrator install wandb/local >= 0.9.31)titletextlevelwait_durationzsend_alert: failed for alert z: )r  r  alertr  r  rL  r   notify_scriptable_run_alertr  r  r  r  rK  r  )rz   r  r  r  r  rO  rE   rE   rF   
send_alert'  s(   &zSendManager.send_alertc                 C   sp   t d |   | jr| j  d | _| jr$| j  | j  d | _| jr1| j| j d | _t	j
  d S )Nzshutting down sender)r  rk  r  r   r   r   r   r   r   r   r@  end_sessionry   rE   rE   rF   r   >  s   



zSendManager.finishc                 C   s.   |   }|di dd }t|tsd S |S )NcliVersionInfor  )rj  r  r/  r   )rz   rp  r  rE   rE   rF   r  O  s   
zSendManager._max_cli_versionc                 C   s&   | j r| jrd S | j \| _| _ d S r   )r   r   r   viewer_server_infory   rE   rE   rF   get_viewer_server_infoX  s   z"SendManager.get_viewer_server_infoc                 C      | j s|   | j S r   )r   r  ry   rE   rE   rF   ri  ]     zSendManager.get_viewer_infoc                 C   r  r   )r   r  ry   rE   rE   rF   rj  b  r  zSendManager.get_server_infor,   c                 C   sd   t  }| jjrd|_|S d}|  }|di }|du r"d|_|S |dd|_|d||_|S )a  Queries the server to get the local version information.

        First, we perform an introspection, if it returns empty we deduce that the
        docker image is out-of-date. Otherwise, we use the returned values to deduce the
        state of the local server.
        FlatestlatestLocalVersionInfoN	outOfDateTlatestVersionString)r   r,   r   r7  out_of_daterj  r  rW   )rz   
local_infolatest_local_versionrp  latest_local_version_inforE   rE   rF   get_local_infog  s   zSendManager.get_local_infoc                 C   s   | j js| jjr
d S | j | j  | j }|	dd  | j 
| | j j| jd}|d urn| jd urp| j|}| jj|_| jj|_| jj|_|jd | j jD ]}|j| qTd|_d|_d|_| j| d S d S d S )Nr  )r  r  T)r   r.  r   r7  
set_configr   non_internal_configr   rT  r^   set_summarybuildr   r   r   _make_artifactrI  r  rg  r  rc  _aliasesr  r  r  _publish_artifact)rz   rK  r  proto_artifactaliasrE   rE   rF   r    s(   



zSendManager._flush_jobc                 C   s   | j jddS )NT)block)r   r  ry   rE   rE   rF   __next__  s   zSendManager.__next__)r5   r   )r  r-   r5   N)r  r-   r5   N)r*  r.   r5   N)Fr   )r  r0   )r  r-   r  r  r  r0   r5   Nr   )rw   r1   r5   N)r   r   r5   N)r  )r5   r,   )r5   r-   )xr   r   r   r   r   r   r   r   r   r   r   r   r   r%   r   r   ro   r   r	   r   r8  r   r"   r  r   r{   classmethodr   r   r   r   r   r   BaseException	tracebackTracebackExceptionr
   r   r   r   r   r$  r)  r,  r1  r  r  rE  rP  rR  r#  rB  rQ  rf  rq  rw  rz  r{  r|  r  r  r  r  r   BackendConfigDictr]  r_  r  r   r  r  r  r  r  r  r  r  rG  rJ  rL  rN  rQ  rt  ri  r  r   r   rl  r  r  r  ru  r  r  r  r  r  r  r   rX  rW  r  r  r  r  r  r  r  r  r  r  r   r  r  ri  rj  r  r  r  r   rE   rE   rE   rF   r      sn  
 


Q!








	








T
_


b
a(








 

'








'



3	r   )Y__doc__r  r\   rS   loggingr   r   r   r   r  collectionsr   r   r   typingr   r   r   r   r	   r
   r   r   r   r   r  r   r   wandb.errorsr   r   wandb.errors.utilr   wandb.filesync.dir_watcherr   wandb.protor   "wandb.sdk.artifacts.artifact_saverr   wandb.sdk.interfacer   #wandb.sdk.interface.interface_queuer   wandb.sdk.internalr   r   r   r   r   wandb.sdk.internal.file_pusherr   wandb.sdk.internal.job_builderr   "wandb.sdk.internal.settings_staticr   wandb.sdk.libr    r!   r"   r#   r$   r%   wandb.sdk.lib.mailboxr&   wandb.sdk.lib.proto_utilr'   wandb.proto.wandb_internal_pb2r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r   	getLoggerr   r  rs  r   rG   re   rL   ro   r   r   rE   rE   rE   rF   <module>   sT    0 0
 