o
    -wi={                     @   sZ  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZmZmZmZmZmZmZmZ ddlmZ ddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# d	d
l$m%Z% d	dl&m'Z'm(Z( ddl)m*Z*m+Z+m,Z, ddl-m.Z. erddlm/Z/ ee0ef Z1e2e3Z4dZ5dee0ef dee0 deddfddZ6G dd dZ7dS )zHandle Manager.    N)defaultdict)Queue)Event)
TYPE_CHECKINGAnyCallableDictIterableListOptionalSequenceTuplecast)url_registry)
HistoryRecordInternalMessagesMetricRecordRecordResult	RunRecordSampledHistoryItemSummaryItemSummaryRecordSummaryRecordRequest   )InterfaceQueue)handler_util
proto_util   )contextsample
tb_watcher)SettingsStatic)MetricSummaryFtargetkey_listvreturnc                 C   sR   |d d D ]}|  |i  | |}trttttf |}|} q|| |d < d S )N)
setdefaultgetr   r   r   strr   )r$   r%   r&   k
new_target r.   W/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/sdk/internal/handler.py_dict_nested_set@   s   
r0   c                   @   s  e Zd ZU eed< eeejf ed< eee	f ed< e
e ed< eed< ded< ded	< eed
< ded< eed< e
ej ed< eeef ed< eeef ed< eeedf ef ed< eeedf e	f ed< e
e ed< eed< e
e ed< ejed< dedddddedddedejddfd d!Zdefd"d#Zd$eddfd%d&Zd$eddfd'd(Zdd$ed*eddfd+d,Zd-eddfd.d/Zdd0d1Z d$eddfd2d3Z!d$eddfd4d5Z"d$eddfd6d7Z#d$eddfd8d9Z$d$eddfd:d;Z%d$eddfd<d=Z&d$eddfd>d?Z'd$eddfd@dAZ(d$eddfdBdCZ)d$eddfdDdEZ*d$eddfdFdGZ+d$eddfdHdIZ,d$eddfdJdKZ-ddLedMeddfdNdOZ.dPe/ddfdQdRZ0dSdTdUe1e dVdWdXedYe
e defdZd[Z2	ddUe1e dVe	d\e
e defd]d^Z3	ddUe1e dVe	d\e
e defd_d`Z4dVeee	f deee	f fdadbZ5dceee	f de1e fdddeZ6dPe/dceee	f ddfdfdgZ7dhede
e fdidjZ8dUe1e dVe	dceee	f dkeee	f ddf
dldmZ9dUe1e dVe	dceee	f dkeee	f ddf
dndoZ:dPe/dceee	f ddfdpdqZ;d$eddfdrdsZ<	ddte
e ddfdudvZ=d$eddfdwdxZ>d$eddfdydzZ?d$eddfd{d|Z@d$eddfd}d~ZAd$eddfddZBd$eddfddZCd$eddfddZDd$eddfddZEd$eddfddZFd$eddfddZGd$eddfddZHd$eddfddZId$eddfddZJd$eddfddZKd$eddfddZLd$eddfddZMd$eddfddZNd$eddfddZOd$eddfddZPd$eddfddZQd$eddfddZRd$eddfddZSd$eddfddZTd$eddfddZUd$eddfddZVd$eddfddZWd$eddfddZXd$eddfddZYd$eddfddZZd$eddfddZ[d$eddfddZ\dddZ]defddZ^e^Z_dPe/dceee	f ddfddZ`dS )HandleManager_consolidated_summary_sampled_history_partial_history
_run_proto	_settingszQueue[Record]	_record_qzQueue[Result]	_result_q_stopped	_writer_q
_interface_tb_watcher_metric_defines_metric_globs._metric_track_metric_copy_track_time_accumulate_time_run_start_time_context_keepersettingsrecord_qresult_qstoppedwriter_q	interfacecontext_keeperr'   Nc                 C   s   || _ || _|| _|| _|| _|| _|| _d | _d| _d | _	d| _
d | _t | _ttj| _d | _t | _tt| _tt| _t | _t | _t | _d| _d S )Nr   F)r6   r7   r8   r9   r:   r;   rD   r<   _steprA   rB   rC   dictr2   r   r    UniformSampleAccumulatorr3   r5   r4   r   r=   r>   r?   r@   r   _internal_messages_dropped_history)selfrE   rF   rG   rH   rI   rJ   rK   r.   r.   r/   __init__b   s,   



zHandleManager.__init__c                 C   s
   | j  S N)r7   qsizerQ   r.   r.   r/   __len__   s   
zHandleManager.__len__recordc                 C   sP   | j | |d}|sJ d| }t| |d }|s"J d| || d S )Nrecord_typehandle_unknown handle: )rD   add_from_record
WhichOneofgetattr)rQ   rW   rX   handler_strhandlerr.   r.   r/   handle   s   
zHandleManager.handlec                 C   s^   |j d}|s
J d| }t| |d }|dkr td|  |s)J d| || d S )Nrequest_typehandle_request_network_statuszhandle_request: rZ   )requestr\   r]   loggerdebug)rQ   rW   ra   r^   r_   r.   r.   r/   handle_request   s   zHandleManager.handle_requestFalways_sendc                 C   s   |rd|j _| j| d S )NT)controlrh   r:   put)rQ   rW   rh   r.   r.   r/   _dispatch_record   s   zHandleManager._dispatch_recordresultc                 C   s&   t |}| j| | j| d S rS   )r   context_id_from_resultrD   releaser8   rj   )rQ   rl   
context_idr.   r.   r/   _respond_result   s   
zHandleManager._respond_resultc                 C   s   d S rS   r.   rU   r.   r.   r/   debounce   s   zHandleManager.debouncec                 C      |  | d S rS   rk   rQ   rW   r.   r.   r/   handle_request_cancel      z#HandleManager.handle_request_cancelc                 C   s   |j j}|j}td|  ||jkr | jr| j  d | _n||jkr*| 	  n||j
kr7| j| jdd | j|dd d S )Nzhandle defer: T)flushrh   )rd   deferstatere   infoFLUSH_TBr<   finishFLUSH_PARTIAL_HISTORY_flush_partial_history	FLUSH_SUM_save_summaryr2   rk   )rQ   rW   ry   rz   r.   r.   r/   handle_request_defer   s   




z"HandleManager.handle_request_deferc                 C   rr   rS   rs   rt   r.   r.   r/   handle_request_python_packages   rv   z,HandleManager.handle_request_python_packagesc                 C   sB   | j jr|j| _t|}|jj|j | | | 	| d S rS   )
r6   _offlinerunr5   r   _result_from_record
run_resultCopyFromrp   rk   rQ   rW   rl   r.   r.   r/   
handle_run   s   

zHandleManager.handle_runc                 C   rr   rS   rs   rt   r.   r.   r/   handle_stats   rv   zHandleManager.handle_statsc                 C   rr   rS   rs   rt   r.   r.   r/   handle_config   rv   zHandleManager.handle_configc                 C   rr   rS   rs   rt   r.   r.   r/   handle_output   rv   zHandleManager.handle_outputc                 C   rr   rS   rs   rt   r.   r.   r/   handle_output_raw   rv   zHandleManager.handle_output_rawc                 C   rr   rS   rs   rt   r.   r.   r/   handle_files   rv   zHandleManager.handle_filesc                 C   rr   rS   rs   rt   r.   r.   r/   handle_request_link_artifact   rv   z*HandleManager.handle_request_link_artifactc                 C   rr   rS   rs   rt   r.   r.   r/   handle_use_artifact   rv   z!HandleManager.handle_use_artifactc                 C   rr   rS   rs   rt   r.   r.   r/   handle_artifact   rv   zHandleManager.handle_artifactc                 C   rr   rS   rs   rt   r.   r.   r/   handle_alert   rv   zHandleManager.handle_alertsummary_dictrw   c           
      C   s   t  }| D ]\}}|j }||_t||_q|r(t|d}| 	| d S | j
js?t|d}| jj|d}	| 	|	 d S d S )N)summary)summary_record)r   itemsupdateaddkeyjsondumps
value_jsonr   rk   r6   r   r   r;   _make_request)
rQ   r   rw   r   r,   r&   r   rW   r   request_recordr.   r.   r/   r      s   


zHandleManager._save_summaryhistoryc                 C   s>   |j D ]}|j}t|j}t|tjr| j| 	| qd S rS   )
itemr   r   loadsr   
isinstancenumbersRealr3   r   )rQ   r   r   r,   r&   r.   r.   r/   _save_history   s   
zHandleManager._save_historysr#   klr&   znumbers.Realfloat_vgoal_maxc                 C   s   d}d }|j r	dS |jrt|dkrt| j|| dS |jrAt|dg }| j|}	|	d u s3||	krA|| j|< t| j|| d}|j	rKt|dg }|j
sR|r|rt|dg }
| j|
}|d u sg||kr|| j|
< |j
rxt| j|
| d}|rt| j|| d}|js|r|st|dg }| j|}|d u s||k r|| j|< |jrt| j|| d}|rt| j|| d}|jrt|dg }t|d	g }t|d
g }| j|d}| j|d}||7 }|d7 }|| j|< || j|< t| j|||  d}|S )NFr   Tlastbestmaxmintotnummeang        r   )nonecopylenr0   r2   r   tupler?   r*   r   r   r   r   )rQ   r   r   r&   r   r   updatedbest_keylast_keyold_lastmax_keyold_maxmin_keyold_mintot_keynum_keyavg_keyr   r   r.   r.   r/   _update_summary_metrics  sh   




z%HandleManager._update_summary_metricsdc           	      C   s   |o| d}t|dkr5t|}| j|}|d u s||kr5|| j|< |r,|r5|jjr5|| j|d < dS |s9dS |s=dS t|t	j
sEdS t|rLdS t|}d }|jr[|j|jk}| j|j||||drhdS dS )Nr   r   r   TF)r   r&   r   r   )HasFieldr   r   r@   r*   r   r   r2   r   r   r   mathisnanfloatgoalGOAL_MAXIMIZEr   )	rQ   r   r&   r   has_summarycopy_keyold_copyr   r   r.   r.   r/   _update_summary_leafA  s4   

z"HandleManager._update_summary_leafc                 C   s   d dd |D }| j||}t|tr9t|s9d}| D ]\}}| j|d d  |g ||dr6d}q!|S t	rSt|trSt|rSd|v rSd|v rS|d |d< | j
|||d}|S )	N.c                 S      g | ]}| d dqS r   z\.replace.0r,   r.   r.   r/   
<listcomp>i      z6HandleManager._update_summary_list.<locals>.<listcomp>F)r   r&   r   T_latest_artifact_pathartifact_path)joinr=   r*   r   rM   r   metric_is_wandb_dictr   _update_summary_list$REPLACE_SUMMARY_ART_PATH_WITH_LATESTr   )rQ   r   r&   r   
metric_keyr   nknvr.   r.   r/   r   c  s&   z"HandleManager._update_summary_listc                 C   sR   |  D ]"\}}tr&t|tr&t|r&d|v r&d|v r&|d |d< |||< q|S )Nr   r   )r   r   r   rM   r   r   )rQ   r&   r   r   r.   r.   r/   _update_summary_media_objects~  s   z+HandleManager._update_summary_media_objectshistory_dictc                 C   s\   | j s| |}| j| t| S g }| D ]\}}| j|g|dr+|| q|S )N)r   r&   )	r=   r   r2   r   listkeysr   r   append)rQ   r   updated_keysr,   r&   r.   r.   r/   _update_summary  s   

zHandleManager._update_summaryc                 C   sr   | d}|j }d|_|r$|jj}||d< t||_|d | _	d S | j	|d< t| j	|_|  j	d7  _	d S )NsteprL   r   )
r   r   r   r   r   r   r   r   r   rL   )rQ   r   r   has_stepr   r   r.   r.   r/   _history_assign_step  s   


z"HandleManager._history_assign_stephkeyc                 C   sr   | drdS | j D ]*\}}|dr6| |dd r6t }|| |d d|j_||_	|  S qdS )zCCheck for hkey match in glob metrics and return the defined metric._N*r(   	glob_nameF)

startswithr>   r   endswithr   r   
ClearFieldoptionsdefinedname)rQ   r   r,   mglobmr.   r.   r/   _history_define_metric  s   



z$HandleManager._history_define_metricupdate_historyc           
      C   s   d dd |D }| j|}|s-| |}|sd S t }|j| d|j_| 	| |j
jrP|jrR|j|vrTt|jg}| j|}	|	d urV|	||j< d S d S d S d S d S )Nr   c                 S   r   r   r   r   r.   r.   r/   r     r   z6HandleManager._history_update_leaf.<locals>.<listcomp>T)r   r=   r*   r   r   metricr   ri   local_handle_defined_metricr   	step_syncstep_metricr   r@   )
rQ   r   r&   r   r   r   r   mrr   r   r.   r.   r/   _history_update_leaf  s&   


z"HandleManager._history_update_leafc                 C   sV   t |tr | D ]\}}| j|d d  |g |||d q	d S | j||||d d S )N)r   r&   r   r   )r   rM   r   _history_update_listr   )rQ   r   r&   r   r   r   r   r.   r.   r/   r     s   

z"HandleManager._history_update_listc           	      C   s   | dd u r| || i }| js| jr'| D ]\}}| |g||| q|rE|| | D ]\}}|j }||_	t
||_q2d S d S )NrL   )r*   r   r=   r>   r   r   r   r   r   r   r   r   r   )	rQ   r   r   r   r   hvalr,   r&   r   r.   r.   r/   _history_update  s   

zHandleManager._history_updatec                    s   t |jj}|d urd|vr |j|  |j|  |  |j  |}|r? fdd|D } 	| d S d S )N_runtimec                    s   i | ]}| j | qS r.   )r2   r   rU   r.   r/   
<dictcomp>  r   z0HandleManager.handle_history.<locals>.<dictcomp>)
r   dict_from_proto_listr   r   _history_assign_runtimer   rk   r   r   r   )rQ   rW   r   r   updated_itemsr.   rU   r/   handle_history  s   

zHandleManager.handle_historyr   c                 C   sj   | j sd S t }| j  D ]\}}|j }||_t||_q|d ur(||j	_
| t|d i | _ d S )N)r   )r4   r   r   r   r   r   r   r   r   r   r   r  r   )rQ   r   r   r,   r&   r   r.   r.   r/   r     s   

z$HandleManager._flush_partial_historyc                 C      | j |dd d S NTrx   rs   rt   r.   r.   r/   !handle_request_sender_mark_report     z/HandleManager.handle_request_sender_mark_reportc                 C   r  r  rs   rt   r.   r.   r/   handle_request_status_report"  r
  z*HandleManager.handle_request_status_reportc                 C   s   |j j}d }|dr|jj}d }|dr|jj}t|j	}|d ura|| j
k rT| js>dtd }| jj| d| _d| d| j
 d| d	}| jj| d S || j
kr`|   || _
n|d u rgd}| j| |rw| | j
 d S d S )
Nactionr   zoStep only supports monotonically increasing values, use define_metric to set a custom x axis. For details see: zdefine-metricTz(User provided step: z is less than current step: z. Dropping entry: z).)rd   partial_historyr   r  rw   r   r   r   r  r   rL   rP   r   urlrO   warningr   r   r4   r   )rQ   rW   r  rw   r   r   messager.   r.   r/   handle_request_partial_history%  sB   



z,HandleManager.handle_request_partial_historyc                 C   s   |j }|jD ]4}t|jdkr|jdksJ t|j}n|jf}| j}|d d D ]}|| }q)t|j	||d < q|j
D ]/}t|jdkrT|jdksNJ t|j}n|jf}| j}|d d D ]}|| }qa||d = q>| | j d S )Nr    r(   )r   r   r   
nested_keyr   r   r2   r   r   r   remover   )rQ   rW   r   r   r   r$   propr.   r.   r/   handle_summaryK  s(   



zHandleManager.handle_summaryc                 C   sB   | j d ur|  jt | j  7  _t| j|j_| j|dd d S r  )rA   rB   timeintexitruntimerk   rt   r.   r.   r/   handle_exitu  s   
zHandleManager.handle_exitc                 C   r  r  rs   rt   r.   r.   r/   handle_final{  r
  zHandleManager.handle_finalc                 C   rr   rS   rs   rt   r.   r.   r/   handle_preempting~  rv   zHandleManager.handle_preemptingc                 C   rr   rS   rs   rt   r.   r.   r/   handle_header  rv   zHandleManager.handle_headerc                 C   rr   rS   rs   rt   r.   r.   r/   handle_footer  rv   zHandleManager.handle_footerc                 C   rr   rS   rs   rt   r.   r.   r/   handle_metadata  rv   zHandleManager.handle_metadatac                 C   sF   t |}|jjj}|sJ | jsJ |jjj	| j | 
| d S rS   )r   r   rd   attach	attach_idr5   responseattach_responser   r   rp   )rQ   rW   rl   r"  r.   r.   r/   handle_request_attach  s   


z#HandleManager.handle_request_attachc                 C   rr   rS   rs   rt   r.   r.   r/   handle_request_log_artifact  rv   z)HandleManager.handle_request_log_artifactc                 C   rr   rS   rs   rt   r.   r.   r/   handle_telemetry  rv   zHandleManager.handle_telemetryc                 C   s   |j j}|sJ |jsJ |j| _|jj d | _t | _|jj	r-|jj
r-|jj
| _nd| _tj| j| j|jd| _|jj	sD|jjrI|jj| _t|}| | d S )Ng    .Ar   )rJ   	run_proto)rd   	run_startr   r5   
start_timeToMicrosecondsrC   r  rA   resumedr  rB   r!   	TBWatcherr6   r;   r<   forkedstarting_steprL   r   r   rp   )rQ   rW   r)  rl   r.   r.   r/   handle_request_run_start  s    



z&HandleManager.handle_request_run_startc                 C   s0   | j d ur|  jt | j  7  _t | _ d S rS   rA   rB   r  rt   r.   r.   r/   handle_request_resume  s   
z#HandleManager.handle_request_resumec                 C   s0   | j d ur|  jt | j  7  _d | _ d S d S rS   r1  rt   r.   r.   r/   handle_request_pause  s   

z"HandleManager.handle_request_pausec                 C   r  r  rs   rt   r.   r.   r/   handle_request_poll_exit  r
  z&HandleManager.handle_request_poll_exitc                 C   rr   rS   rs   rt   r.   r.   r/   handle_request_stop_status  rv   z(HandleManager.handle_request_stop_statusc                 C   rr   rS   rs   rt   r.   r.   r/   handle_request_network_status  rv   z+HandleManager.handle_request_network_statusc                 C   s4   t |}|jjj| j | j  | | d S rS   )	r   r   r#  internal_messages_responsemessagesr   rO   Clearrp   r   r.   r.   r/    handle_request_internal_messages  s   


z.HandleManager.handle_request_internal_messagesc                 C   s   t |}| | d S rS   )r   r   rp   r   r.   r.   r/   handle_request_status  s   
z#HandleManager.handle_request_statusc                 C   sT   t |}| j D ]\}}t }||_t||_|j	j
j| q
| | d S rS   )r   r   r2   r   r   r   r   r   r   r#  get_summary_responser   r   rp   )rQ   rW   rl   r   valuer   r.   r.   r/   handle_request_get_summary  s   
z(HandleManager.handle_request_get_summaryc                 C   s<   t d| | jr|j}| j|j|j|j | | d S )Nzhandling tbrecord: %s)	re   r{   r<   tbrecordr   log_dirsaveroot_dirrk   )rQ   rW   r?  r.   r.   r/   handle_tbrecord  s
   zHandleManager.handle_tbrecordc                 C   s   |j }|jjr| j|j | n	| j|j | | j|j }|jrG|j| jvrGt|jd}|| j|j< t	 }|j | d|j
_| | | | d S )N)r   T)r   _control	overwriter=   r   r   	MergeFromr   r   r   ri   r   rk   )rQ   rW   r   r   r   r.   r.   r/   r     s   
z$HandleManager._handle_defined_metricc                 C   sB   |j }|jjr| j|j | n	| j|j | | | d S rS   )r   rD  rE  r>   r   r   rF  rk   )rQ   rW   r   r.   r.   r/   _handle_glob_metric  s
   z!HandleManager._handle_glob_metricc                 C   s0   |j jr| | dS |j jr| | dS dS )a  Handle MetricRecord.

        Walkthrough of the life of a MetricRecord:

        Metric defined:
        - run.define_metric() parses arguments create wandb_metric.Metric
        - build MetricRecord publish to interface
        - handler (this function) keeps list of metrics published:
          - self._metric_defines: Fully defined metrics
          - self._metric_globs: metrics that have a wildcard
        - dispatch writer and sender thread
          - writer: records are saved to persistent store
          - sender: fully defined metrics get mapped into metadata for UI

        History logged:
        - handle_history
        - check if metric matches _metric_defines
        - if not, check if metric matches _metric_globs
        - if _metric globs match, generate defined metric and call _handle_metric

        Args:
            record (Record): Metric record to process
        N)r   r   r   r   rG  rt   r.   r.   r/   handle_metric  s
   zHandleManager.handle_metricc              	   C   s   t |}| j D ]@\}}t }||_| }tdd |D r3z|j	| W n t
y2   Y nw tdd |D rB|j	| |jjj| q
| | d S )Nc                 s       | ]	}t |tjV  qd S rS   )r   r   Integralr   ir.   r.   r/   	<genexpr>      z?HandleManager.handle_request_sampled_history.<locals>.<genexpr>c                 s   rI  rS   )r   r   r   rK  r.   r.   r/   rM  $  rN  )r   r   r3   r   r   r   r*   all
values_intextend
ValueErrorvalues_floatr#  sampled_history_responser   r   rp   )rQ   rW   rl   r   sampledr   valuesr.   r.   r/   handle_request_sampled_history  s   
z,HandleManager.handle_request_sampled_historyc                 C   s   dS )zmHandle a keepalive request.

        Keepalive is a noop, we just want to verify transport is alive.
        Nr.   rt   r.   r.   r/   handle_request_keepalive)  s    z&HandleManager.handle_request_keepalivec                 C   r  r  rs   rt   r.   r.   r/   handle_request_run_status/  r
  z'HandleManager.handle_request_run_statusc                 C   s"   t |}| | | j  d S rS   )r   r   rp   r9   setr   r.   r.   r/   handle_request_shutdown2  s   

z%HandleManager.handle_request_shutdownc                 C   s   |  t| dS )z.No-op. Not implemented for the legacy-service.N)rp   r   r   rt   r.   r.   r/   handle_request_operations8  s   z'HandleManager.handle_request_operationsc                 C   s"   t d | jr| j  d S d S )Nzshutting down handler)re   r{   r<   r}   rU   r.   r.   r/   r}   <  s   
zHandleManager.finishc                 C   s   | j jddS )NT)block)r7   r*   rU   r.   r.   r/   __next__B  rv   zHandleManager.__next__c                 C   sX   d|vrd S | j d u r|d | _ |d | j  |d< |j }d|_t||j |_d S )N
_timestampr  )rC   r   r   r   r   r   r   )rQ   r   r   r   r.   r.   r/   r  G  s   


z%HandleManager._history_assign_runtime)F)r'   NrS   )a__name__
__module____qualname__SummaryDict__annotations__r   r+   r    rN   r   r   r   r"   r   r   r!   r-  r   r   r   r   ContextKeeperrR   r  rV   r   r`   rg   boolrk   r   rp   rq   ru   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   r   r   r   r   r   r   r   r   r   r  r   r	  r  r  r  r  r  r  r  r  r   r%  r&  r'  r0  r2  r3  r4  r5  r6  r:  r;  r>  rC  r   rG  rH  rW  rX  rY  r[  r\  r}   r^  nextr  r.   r.   r.   r/   r1   M   sl  
 
	
&	



C
&
"










&*	

r1   )8__doc__r   loggingr   r   r  collectionsr   queuer   	threadingr   typingr   r   r   r   r	   r
   r   r   r   r   wandb.errors.linksr   wandb.proto.wandb_internal_pb2r   r   r   r   r   r   r   r   r   r   interface.interface_queuer   libr   r   r  r   r    r!   settings_staticr"   r#   r+   rc  	getLoggerr`  re   r   r0   r1   r.   r.   r.   r/   <module>   s.    00
&