o
    xiz                     @  sB  d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	 ddl
mZmZ ddlmZ ddlmZ ddlmZmZmZmZ dd	lmZ dd
lmZmZmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z- erddlm.Z. e/e0ef Z1e2e3Z4dZ5dddZ6G dd dZ7dS ) zHandle Manager.    )annotationsN)defaultdict)IterableSequence)Queue)Event)TYPE_CHECKINGAnyCallablecast)url_registry)
HistoryRecordInternalMessagesMetricRecordRecordResult	RunRecordSampledHistoryItemSummaryItemSummaryRecordSummaryRecordRequest   )InterfaceQueue)handler_util
proto_util   )contextsample
tb_watcher)SettingsStatic)MetricSummaryFtargetdict[str, Any]key_listSequence[str]vr	   returnNonec                 C  sR   |d d D ]}|  |i  | |}trttttf |}|} q|| |d < d S )N)
setdefaultgetr   r   dictstrr	   )r!   r#   r%   k
new_target r/   N/home/ubuntu/.local/lib/python3.10/site-packages/wandb/sdk/internal/handler.py_dict_nested_set8   s   
r1   c                   @  sb  e Zd ZU ded< ded< ded< ded< d	ed
< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded < ded!< d"ed#< dd-d.Zdd0d1Zdd4d5Zdd6d7Zddd;d<Zdd?d@Z	ddAdBZ
ddCdDZddEdFZddGdHZddIdJZddKdLZddMdNZddOdPZddQdRZddSdTZddUdVZddWdXZddYdZZdd[d\Zddd_d`ZddcddZddndoZ	pdddtduZ	pdddvdwZddxdyZdd{d|Zdd}d~ZdddZ dddZ!dddZ"dddZ#dddZ$	pddddZ%dddZ&dddZ'dddZ(dddZ)dddZ*dddZ+dddZ,dddZ-dddZ.dddZ/dddZ0dddZ1dddZ2dddZ3dddZ4dddZ5dddZ6dddZ7dddZ8dddZ9dddZ:dddZ;dddZ<dddZ=dddZ>dddÄZ?dddńZ@dddǄZAdddɄZBddd˄ZCddd̈́ZDdddτZEdddфZFeFZGdddӄZHdpS )HandleManagerSummaryDict_consolidated_summaryz*dict[str, sample.UniformSampleAccumulator]_sampled_historyr"   _partial_historyzRunRecord | None
_run_protor   	_settingsQueue[Record]	_record_qQueue[Result]	_result_qr   _stopped	_writer_qr   
_interfaceztb_watcher.TBWatcher | None_tb_watcherzdict[str, MetricRecord]_metric_defines_metric_globszdict[tuple[str, ...], float]_metric_trackzdict[tuple[str, ...], Any]_metric_copyzfloat | None_track_timefloat_accumulate_time_run_start_timecontext.ContextKeeper_context_keepersettingsrecord_qresult_qstoppedwriter_q	interfacecontext_keeperr&   r'   c                 C  s   || _ || _|| _|| _|| _|| _|| _d | _d| _d | _	d| _
d | _t | _ttj| _d | _t | _tt| _tt| _t | _t | _t | _d| _d S )Nr   F)r8   r:   r<   r=   r>   r?   rJ   r@   _steprE   rG   rH   r+   r4   r   r   UniformSampleAccumulatorr5   r7   r6   r   rA   rB   rC   rD   r   _internal_messages_dropped_history)selfrK   rL   rM   rN   rO   rP   rQ   r/   r/   r0   __init__Z   s,   



zHandleManager.__init__intc                 C  s
   | j  S N)r:   qsizerV   r/   r/   r0   __len__   s   
zHandleManager.__len__recordr   c                 C  sP   | j | |d}|sJ d| }t| |d }|s"J d| || d S )Nrecord_typehandle_unknown handle: )rJ   add_from_record
WhichOneofgetattr)rV   r]   r^   handler_strhandlerr/   r/   r0   handle   s   
zHandleManager.handlec                 C  s^   |j d}|s
J d| }t| |d }|dkr td|  |s)J d| || d S )Nrequest_typehandle_request_network_statuszhandle_request: r`   )requestrb   rc   loggerdebug)rV   r]   rg   rd   re   r/   r/   r0   handle_request   s   zHandleManager.handle_requestFalways_sendboolc                 C  s   |rd|j _| j| d S )NT)controlrn   r>   put)rV   r]   rn   r/   r/   r0   _dispatch_record   s   zHandleManager._dispatch_recordresultr   c                 C  s&   t |}| j| | j| d S rY   )r   context_id_from_resultrJ   releaser<   rq   )rV   rs   
context_idr/   r/   r0   _respond_result   s   
zHandleManager._respond_resultc                 C  s   d S rY   r/   r[   r/   r/   r0   debounce   s   zHandleManager.debouncec                 C     |  | d S rY   rr   rV   r]   r/   r/   r0   handle_request_cancel      z#HandleManager.handle_request_cancelc                 C  s   |j j}|j}td|  ||jkr | jr| j  d | _n||jkr*| 	  n||j
kr7| j| jdd | j|dd d S )Nzhandle defer: T)flushrn   )rj   deferstaterk   infoFLUSH_TBr@   finishFLUSH_PARTIAL_HISTORY_flush_partial_history	FLUSH_SUM_save_summaryr4   rr   )rV   r]   r   r   r/   r/   r0   handle_request_defer   s   




z"HandleManager.handle_request_deferc                 C  ry   rY   rz   r{   r/   r/   r0   handle_request_python_packages   r}   z,HandleManager.handle_request_python_packagesc                 C  sB   | j jr|j| _t|}|jj|j | | | 	| d S rY   )
r8   _offlinerunr7   r   _result_from_record
run_resultCopyFromrw   rr   rV   r]   rs   r/   r/   r0   
handle_run   s   

zHandleManager.handle_runc                 C  ry   rY   rz   r{   r/   r/   r0   handle_stats   r}   zHandleManager.handle_statsc                 C  ry   rY   rz   r{   r/   r/   r0   handle_config   r}   zHandleManager.handle_configc                 C  ry   rY   rz   r{   r/   r/   r0   handle_output   r}   zHandleManager.handle_outputc                 C  ry   rY   rz   r{   r/   r/   r0   handle_output_raw   r}   zHandleManager.handle_output_rawc                 C  ry   rY   rz   r{   r/   r/   r0   handle_files   r}   zHandleManager.handle_filesc                 C  ry   rY   rz   r{   r/   r/   r0   handle_request_link_artifact   r}   z*HandleManager.handle_request_link_artifactc                 C  ry   rY   rz   r{   r/   r/   r0   handle_use_artifact   r}   z!HandleManager.handle_use_artifactc                 C  ry   rY   rz   r{   r/   r/   r0   handle_artifact   r}   zHandleManager.handle_artifactc                 C  ry   rY   rz   r{   r/   r/   r0   handle_alert   r}   zHandleManager.handle_alertsummary_dictr~   c           
      C  s   t  }| D ]\}}|j }||_t||_q|r(t|d}| 	| d S | j
js?t|d}| jj|d}	| 	|	 d S d S )N)summary)summary_record)r   itemsupdateaddkeyjsondumps
value_jsonr   rr   r8   r   r   r?   _make_request)
rV   r   r~   r   r-   r%   r   r]   r   request_recordr/   r/   r0   r      s   


zHandleManager._save_summaryhistoryr   c                 C  s>   |j D ]}|j}t|j}t|tjr| j| 	| qd S rY   )
itemr   r   loadsr   
isinstancenumbersRealr5   r   )rV   r   r   r-   r%   r/   r/   r0   _save_history   s   
zHandleManager._save_historysr    kl	list[str]r%   numbers.Realfloat_vgoal_maxbool | Nonec                 C  s   d}d }|j r	dS |jrt|dkrt| j|| dS |jrAt|dg }| j|}	|	d u s3||	krA|| j|< t| j|| d}|j	rKt|dg }|j
sR|r|rt|dg }
| j|
}|d u sg||kr|| j|
< |j
rxt| j|
| d}|rt| j|| d}|js|r|st|dg }| j|}|d u s||k r|| j|< |jrt| j|| d}|rt| j|| d}|jrt|dg }t|d	g }t|d
g }| j|d}| j|d}||7 }|d7 }|| j|< || j|< t| j|||  d}|S )NFr   Tlastbestmaxmintotnummeang        r   )nonecopylenr1   r4   r   tuplerC   r*   r   r   r   r   )rV   r   r   r%   r   r   updatedbest_keylast_keyold_lastmax_keyold_maxmin_keyold_mintot_keynum_keyavg_keyr   r   r/   r/   r0   _update_summary_metrics   sf   




z%HandleManager._update_summary_metricsNr	   dMetricRecord | Nonec           	      C  s   |o| d}t|dkr5t|}| j|}|d u s||kr5|| j|< |r,|r5|jjr5|| j|d < dS |s9dS |s=dS t|t	j
sEdS t|rLdS t|}d }|jr[|j|jk}t| j|j||||dS )Nr   r   r   TF)r   r%   r   r   )HasFieldr   r   rD   r*   r   r   r4   r   r   r   mathisnanrF   goalGOAL_MAXIMIZEro   r   )	rV   r   r%   r   has_summarycopy_keyold_copyr   r   r/   r/   r0   _update_summary_leaf8  s4   

z"HandleManager._update_summary_leafc                 C  s   d dd |D }| j||}t|tr9t|s9d}| D ]\}}| j|d d  |g ||dr6d}q!|S t	rSt|trSt|rSd|v rSd|v rS|d |d< | j
|||d}|S )	N.c                 S     g | ]}| d dqS r   z\.replace.0r-   r/   r/   r0   
<listcomp>`      z6HandleManager._update_summary_list.<locals>.<listcomp>F)r   r%   r   T_latest_artifact_pathartifact_path)joinrA   r*   r   r+   r   metric_is_wandb_dictr   _update_summary_list$REPLACE_SUMMARY_ART_PATH_WITH_LATESTr   )rV   r   r%   r   
metric_keyr   nknvr/   r/   r0   r   Z  s&   z"HandleManager._update_summary_listc                 C  sR   |  D ]"\}}tr&t|tr&t|r&d|v r&d|v r&|d |d< |||< q|S )Nr   r   )r   r   r   r+   r   r   )rV   r%   r   r   r/   r/   r0   _update_summary_media_objectsu  s   z+HandleManager._update_summary_media_objectshistory_dictc                 C  s\   | j s| |}| j| t| S g }| D ]\}}| j|g|dr+|| q|S )N)r   r%   )	rA   r   r4   r   listkeysr   r   append)rV   r   updated_keysr-   r%   r/   r/   r0   _update_summary  s   

zHandleManager._update_summaryc                 C  sr   | d}|j }d|_|r$|jj}||d< t||_|d | _	d S | j	|d< t| j	|_|  j	d7  _	d S )NsteprR   r   )
r   r   r   r   r   r   r   r   r   rR   )rV   r   r   has_stepr   r   r/   r/   r0   _history_assign_step  s   


z"HandleManager._history_assign_stephkeyr,   c                 C  sr   | drdS | j D ]*\}}|dr6| |dd r6t }|| |d d|j_||_	|  S qdS )zCCheck for hkey match in glob metrics and return the defined metric._N*r(   	glob_nameF)

startswithrB   r   endswithr   r   
ClearFieldoptionsdefinedname)rV   r   r-   mglobmr/   r/   r0   _history_define_metric  s   


z$HandleManager._history_define_metricupdate_historyc           
      C  s   d dd |D }| j|}|s-| |}|sd S t }|j| d|j_| 	| |j
jrP|jrR|j|vrTt|jg}| j|}	|	d urV|	||j< d S d S d S d S d S )Nr   c                 S  r   r   r   r   r/   r/   r0   r     r   z6HandleManager._history_update_leaf.<locals>.<listcomp>T)r   rA   r*   r   r   metricr   rp   local_handle_defined_metricr   	step_syncstep_metricr   rD   )
rV   r   r%   r   r   r   r   mrr   r   r/   r/   r0   _history_update_leaf  s"   

z"HandleManager._history_update_leafc                 C  sV   t |tr | D ]\}}| j|d d  |g |||d q	d S | j||||d d S )N)r   r%   r   r   )r   r+   r   _history_update_listr  )rV   r   r%   r   r   r   r   r/   r/   r0   r    s   

z"HandleManager._history_update_listc           	      C  s   | dd u r| || i }| js| jr'| D ]\}}| |g||| q|rE|| | D ]\}}|j }||_	t
||_q2d S d S )NrR   )r*   r   rA   rB   r   r  r   r   r   r   r   r   r   )	rV   r   r   r   r   hvalr-   r%   r   r/   r/   r0   _history_update  s   

zHandleManager._history_updatec                   s   t |jj}|d urd|vr |j|  |j|  |  |j  |}|r? fdd|D } 	| d S d S )N_runtimec                   s   i | ]}| j | qS r/   )r4   r   r[   r/   r0   
<dictcomp>  r   z0HandleManager.handle_history.<locals>.<dictcomp>)
r   dict_from_proto_listr   r   _history_assign_runtimer
  rr   r   r   r   )rV   r]   r   r   updated_itemsr/   r[   r0   handle_history  s   

zHandleManager.handle_historyr   
int | Nonec                 C  sj   | j sd S t }| j  D ]\}}|j }||_t||_q|d ur(||j	_
| t|d i | _ d S )N)r   )r6   r   r   r   r   r   r   r   r   r   r   r  r   )rV   r   r   r-   r%   r   r/   r/   r0   r     s   

z$HandleManager._flush_partial_historyc                 C     | j |dd d S NTr   rz   r{   r/   r/   r0   !handle_request_sender_mark_report     z/HandleManager.handle_request_sender_mark_reportc                 C  r  r  rz   r{   r/   r/   r0   handle_request_status_report  r  z*HandleManager.handle_request_status_reportc                 C  s   |j j}d }|dr|jj}d }|dr|jj}t|j	}|d ura|| j
k rT| js>dtd }| jj| d| _d| d| j
 d| d	}| jj| d S || j
kr`|   || _
n|d u rgd}| j| |rw| | j
 d S d S )
Nactionr   zoStep only supports monotonically increasing values, use define_metric to set a custom x axis. For details see: zdefine-metricTz(User provided step: z is less than current step: z. Dropping entry: z).)rj   partial_historyr   r  r~   r   r   r   r  r   rR   rU   r   urlrT   warningr   r   r6   r   )rV   r]   r  r~   r   r   messager/   r/   r0   handle_request_partial_history  sB   



z,HandleManager.handle_request_partial_historyc                 C  s   |j }|jD ]4}t|jdkr|jdksJ t|j}n|jf}| j}|d d D ]}|| }q)t|j	||d < q|j
D ]/}t|jdkrT|jdksNJ t|j}n|jf}| j}|d d D ]}|| }qa||d = q>| | j d S )Nr    r(   )r   r   r   
nested_keyr   r   r4   r   r   r   remover   )rV   r]   r   r   r   r!   propr/   r/   r0   handle_summary?  s(   



zHandleManager.handle_summaryc                 C  sB   | j d ur|  jt | j  7  _t| j|j_| j|dd d S r  )rE   rG   timerX   exitruntimerr   r{   r/   r/   r0   handle_exiti  s   
zHandleManager.handle_exitc                 C  r  r  rz   r{   r/   r/   r0   handle_finalo  r  zHandleManager.handle_finalc                 C  ry   rY   rz   r{   r/   r/   r0   handle_preemptingr  r}   zHandleManager.handle_preemptingc                 C  ry   rY   rz   r{   r/   r/   r0   handle_headeru  r}   zHandleManager.handle_headerc                 C  ry   rY   rz   r{   r/   r/   r0   handle_footerx  r}   zHandleManager.handle_footerc                 C  ry   rY   rz   r{   r/   r/   r0   handle_metadata{  r}   zHandleManager.handle_metadatac                 C  sF   t |}|jjj}|sJ | jsJ |jjj	| j | 
| d S rY   )r   r   rj   attach	attach_idr7   responseattach_responser   r   rw   )rV   r]   rs   r,  r/   r/   r0   handle_request_attach~  s   


z#HandleManager.handle_request_attachc                 C  ry   rY   rz   r{   r/   r/   r0   handle_request_log_artifact  r}   z)HandleManager.handle_request_log_artifactc                 C  ry   rY   rz   r{   r/   r/   r0   handle_telemetry  r}   zHandleManager.handle_telemetryc                 C  s   |j j}|sJ |jsJ |j| _|jj d | _t | _|jj	r-|jj
r-|jj
| _nd| _tj| j| j|jd| _|jj	sD|jjrI|jj| _t|}| | d S )Ng    .Ar   )rP   	run_proto)rj   	run_startr   r7   
start_timeToMicrosecondsrH   r"  rE   resumedr$  rG   r   	TBWatcherr8   r?   r@   forkedstarting_steprR   r   r   rw   )rV   r]   r3  rs   r/   r/   r0   handle_request_run_start  s    



z&HandleManager.handle_request_run_startc                 C  s0   | j d ur|  jt | j  7  _t | _ d S rY   rE   rG   r"  r{   r/   r/   r0   handle_request_resume  s   
z#HandleManager.handle_request_resumec                 C  s0   | j d ur|  jt | j  7  _d | _ d S d S rY   r;  r{   r/   r/   r0   handle_request_pause  s   

z"HandleManager.handle_request_pausec                 C  r  r  rz   r{   r/   r/   r0   handle_request_poll_exit  r  z&HandleManager.handle_request_poll_exitc                 C  ry   rY   rz   r{   r/   r/   r0   handle_request_stop_status  r}   z(HandleManager.handle_request_stop_statusc                 C  ry   rY   rz   r{   r/   r/   r0   handle_request_network_status  r}   z+HandleManager.handle_request_network_statusc                 C  s4   t |}|jjj| j | j  | | d S rY   )	r   r   r-  internal_messages_responsemessagesr   rT   Clearrw   r   r/   r/   r0    handle_request_internal_messages  s   


z.HandleManager.handle_request_internal_messagesc                 C  s   t |}| | d S rY   )r   r   rw   r   r/   r/   r0   handle_request_status  s   
z#HandleManager.handle_request_statusc                 C  sT   t |}| j D ]\}}t }||_t||_|j	j
j| q
| | d S rY   )r   r   r4   r   r   r   r   r   r   r-  get_summary_responser   r   rw   )rV   r]   rs   r   valuer   r/   r/   r0   handle_request_get_summary  s   
z(HandleManager.handle_request_get_summaryc                 C  s<   t d| | jr|j}| j|j|j|j | | d S )Nzhandling tbrecord: %s)	rk   r   r@   tbrecordr   log_dirsaveroot_dirrr   )rV   r]   rI  r/   r/   r0   handle_tbrecord  s
   zHandleManager.handle_tbrecordc                 C  s   |j }|jjr| j|j | n	| j|j | | j|j }|jrG|j| jvrGt|jd}|| j|j< t	 }|j | d|j
_| | | | d S )N)r   T)r  _control	overwriterA   r   r   	MergeFromr  r   r   rp   r  rr   )rV   r]   r  r   r  r/   r/   r0   r    s   
z$HandleManager._handle_defined_metricc                 C  sB   |j }|jjr| j|j | n	| j|j | | | d S rY   )r  rN  rO  rB   r   r   rP  rr   )rV   r]   r  r/   r/   r0   _handle_glob_metric  s
   z!HandleManager._handle_glob_metricc                 C  s0   |j jr| | dS |j jr| | dS dS )a  Handle MetricRecord.

        Walkthrough of the life of a MetricRecord:

        Metric defined:
        - run.define_metric() parses arguments create wandb_metric.Metric
        - build MetricRecord publish to interface
        - handler (this function) keeps list of metrics published:
          - self._metric_defines: Fully defined metrics
          - self._metric_globs: metrics that have a wildcard
        - dispatch writer and sender thread
          - writer: records are saved to persistent store
          - sender: fully defined metrics get mapped into metadata for UI

        History logged:
        - handle_history
        - check if metric matches _metric_defines
        - if not, check if metric matches _metric_globs
        - if _metric globs match, generate defined metric and call _handle_metric

        Args:
            record (Record): Metric record to process
        N)r  r   r  r   rQ  r{   r/   r/   r0   handle_metric  s
   zHandleManager.handle_metricc              	   C  s   t |}| j D ]@\}}t }||_| }tdd |D r3z|j	| W n t
y2   Y nw tdd |D rB|j	| |jjj| q
| | d S )Nc                 s      | ]	}t |tjV  qd S rY   )r   r   Integralr   ir/   r/   r0   	<genexpr>      z?HandleManager.handle_request_sampled_history.<locals>.<genexpr>c                 s  rS  rY   )r   r   r   rU  r/   r/   r0   rW    rX  )r   r   r5   r   r   r   r*   all
values_intextend
ValueErrorvalues_floatr-  sampled_history_responser   r   rw   )rV   r]   rs   r   sampledr   valuesr/   r/   r0   handle_request_sampled_history  s   
z,HandleManager.handle_request_sampled_historyc                 C  s   dS )zmHandle a keepalive request.

        Keepalive is a noop, we just want to verify transport is alive.
        Nr/   r{   r/   r/   r0   handle_request_keepalive  s    z&HandleManager.handle_request_keepalivec                 C  r  r  rz   r{   r/   r/   r0   handle_request_run_status#  r  z'HandleManager.handle_request_run_statusc                 C  s"   t |}| | | j  d S rY   )r   r   rw   r=   setr   r/   r/   r0   handle_request_shutdown&  s   

z%HandleManager.handle_request_shutdownc                 C  s   |  t| dS )z.No-op. Not implemented for the legacy-service.N)rw   r   r   r{   r/   r/   r0   handle_request_operations,  s   z'HandleManager.handle_request_operationsc                 C  s"   t d | jr| j  d S d S )Nzshutting down handler)rk   r   r@   r   r[   r/   r/   r0   r   0  s   
zHandleManager.finishc                 C  s   | j jddS )NT)block)r:   r*   r[   r/   r/   r0   __next__6  r}   zHandleManager.__next__c                 C  sX   d|vrd S | j d u r|d | _ |d | j  |d< |j }d|_t||j |_d S )N
_timestampr  )rH   r   r   r   r   r   r   )rV   r   r   r   r/   r/   r0   r  ;  s   


z%HandleManager._history_assign_runtime)rK   r   rL   r9   rM   r;   rN   r   rO   r9   rP   r   rQ   rI   r&   r'   )r&   rX   )r]   r   r&   r'   )F)r]   r   rn   ro   r&   r'   )rs   r   r&   r'   )r&   r'   )r   r3   r~   ro   r&   r'   )r   r   r&   r'   )r   r    r   r   r%   r   r   rF   r   r   r&   ro   rY   )r   r   r%   r	   r   r   r&   ro   )r%   r"   r&   r"   )r   r"   r&   r   )r   r   r   r"   r&   r'   )r   r,   r&   r   )
r   r   r%   r	   r   r"   r   r"   r&   r'   )r   r  r&   r'   )r&   r   )I__name__
__module____qualname____annotations__rW   r\   rf   rm   rr   rw   rx   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r
  r  r   r  r  r  r!  r%  r&  r'  r(  r)  r*  r/  r0  r1  r:  r<  r=  r>  r?  r@  rD  rE  rH  rM  r  rQ  rR  ra  rb  rc  re  rf  r   rh  nextr  r/   r/   r/   r0   r2   E   s   
 

&

	

















B&











&
*

















	









r2   )r!   r"   r#   r$   r%   r	   r&   r'   )8__doc__
__future__r   r   loggingr   r   r"  collectionsr   collections.abcr   r   queuer   	threadingr   typingr   r	   r
   r   wandb.errors.linksr   wandb.proto.wandb_internal_pb2r   r   r   r   r   r   r   r   r   r   interface.interface_queuer   libr   r   r  r   r   r   settings_staticr   r    r+   r,   r3   	getLoggerrj  rk   r   r1   r2   r/   r/   r/   r0   <module>   s2    0

