o
    xilH                     @  sp  d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZmZ ddlZddlmZ ddlmZ ddlmZ dd	lmZ er~dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z! ddl"m#Z# e$e%ef Z&dZ'dZ(dZ)e*e+Z,d0ddZ-d1d$d%Z.G d&d' d'Z/G d(d) d)Z0G d*d+ d+Z1G d,d- d-Z2G d.d/ d/Z3dS )2ztensorboard watcher.    )annotationsN)TYPE_CHECKINGAny)util)CustomChart)
filesystem   )run)PriorityQueue)EventFileLoader)
ProtoEvent)	RunRecord)	FilesDict   )InterfaceQueue)SettingsStatic   z://pathstr	base_path	interfacer   settingsr   returnNonec                 C  s   |j }tj| |}tj| }tj||}ttj| tj	|r9|t
|kr9t| t|| ntj|sEt|| |ttt|dfgd d S )Nlivefiles)	files_dirosr   relpathabspathjoinr   mkdir_exists_okdirnameislinkreadlinkremovesymlinkexistspublish_filesdictGlobStrglobescape)r   r   r   r   r   	file_nameabs_path
wandb_path r1   Q/home/ubuntu/.local/lib/python3.10/site-packages/wandb/sdk/internal/tb_watcher.py_link_and_save_file+   s   
r3   hostname
str | None
start_timefloat | Noneboolc           
   	   C  s   | st dtj| }|drdS |d}z|d}W n
 t y)   Y dS w |durWt|dD ]!\}}z
||d |  }W n tyN   Y  dS w ||krV dS q5|durzz
t	||d  }	W n t tfyq   Y dS w |	t	|k rzdS d	S )
a  Check if a path is a tfevents file.

    Optionally checks that it was created by [hostname] after [start_time].

    tensorboard tfevents filename format:
        https://github.com/tensorflow/tensorboard/blob/f3f26b46981da5bd46a5bb93fcf02d9eb7608bc1/tensorboard/summary/writer/event_file_writer.py#L81
    tensorflow tfevents filename format:
        https://github.com/tensorflow/tensorflow/blob/8f597046dc30c14b5413813d02c0e0aed399c177/tensorflow/core/util/events_writer.cc#L68
    Path must be a nonempty string)z.profile-emptyz.sagemaker-uploadedF.tfeventsNr   r   T)

ValueErrorr   r   basenameendswithsplitindex	enumerate
IndexErrorint)
r   r4   r6   r=   fname_componentstfevents_idxipartfname_component_partcreated_timer1   r1   r2   is_tfevents_file_created_by@   s<   

rJ   c                   @  sJ   e Zd ZU ded< ded< 	dd ddZd!ddZd"ddZd#ddZdS )$	TBWatcherzdict[str, TBDirWatcher]_logdirsr
   _watcher_queueFr   r   	run_protor   r   r   forcer8   r   r   c                 C  s<   i | _ d | _|| _|| _|| _|| _t | _t	j
  d S N)rL   	_consumer	_settings
_interface
_run_proto_forcequeuer
   rM   wandbtensorboardreset_state)selfr   rN   r   rO   r1   r1   r2   __init__r   s   
zTBWatcher.__init__logdirr   rootdirr5   c                 C  s   t | j|g }tj|rtj|}nd}|dkrCttjtj	|}|
|d
|dd}t|dkrA|dvrAd }|S |
|d
|dd}|S )N /r   )train
validation)listrL   r   r   isfiler=   r   to_forward_slash_pathr#   commonprefixreplacestriplen)rZ   r\   r]   dirsfilename	namespacer1   r1   r2   _calculate_namespace   s   zTBWatcher._calculate_namespacesaveroot_dirc                 C  s   t |}t |}|| jv rd S | ||}| js*t| | j| j| j| _| j	  t
| |||| j| j}|| j|< |	  d S rP   )r   rd   rL   rl   rQ   TBEventConsumerrM   rT   rR   startTBDirWatcherrU   )rZ   r\   rm   rn   rk   tbdir_watcherr1   r1   r2   add   s   




zTBWatcher.addc                 C  sH   | j  D ]}|  q| j  D ]}|  q| jr"| j  d S d S rP   )rL   valuesshutdownfinishrQ   )rZ   tbdirwatcherr1   r1   r2   rv      s   

zTBWatcher.finishNF)
r   r   rN   r   r   r   rO   r8   r   r   )r\   r   r]   r   r   r5   )r\   r   rm   r8   rn   r   r   r   r   r   )__name__
__module____qualname____annotations__r[   rl   rs   rv   r1   r1   r1   r2   rK   n   s   
 

rK   c                   @  sz   e Zd Z	d*d+ddZd,ddZd-ddZ	d.d/ddZd*d0ddZd,ddZd,d d!Z	d1d$d%Z
d,d&d'Zd,d(d)ZdS )2rq   F	tbwatcherrK   r\   r   rm   r8   rk   r5   rV   r
   rO   r   r   c                 C  s   t jddd| _t jddd| _|| _| j|| ||| j| _t	j
| jd| _d | _t	 | _|| _d | _|| _|| _t | _|| _t	 | _d S )Nz6tensorboard.backend.event_processing.directory_watcher"Please install tensorboard package)requiredztensorboard.compattarget)r   
get_moduledirectory_watcher	tf_compat
_tbwatcherDirectoryWatcher_loader_is_our_tfevents_file
_generator	threadingThread_thread_except_body_thread_first_event_timestampEvent	_shutdown_queue_file_version
_namespace_logdirsocketgethostname	_hostnamerU   Lock_process_events_lock)rZ   r~   r\   rm   rk   rV   rO   r1   r1   r2   r[      s*   	

zTBDirWatcher.__init__c                 C  s   | j   d S rP   )r   rp   rZ   r1   r1   r2   rp      s   zTBDirWatcher.startr   c                 C  sB   |st d| jjj|}| jrt|ddS t|| j| jj	j
S )zECheck if a path has been modified since launch and contains tfevents.r9   N)r<   r   tfcompat
as_str_anyrU   rJ   r   r   rR   x_start_time)rZ   r   r1   r1   r2   r      s   z"TBDirWatcher._is_our_tfevents_fileTNr   c                   sX   | j j | j jzddlm} W n ty   tdw G  fddd|j}|S )zKIncredibly hacky class generator to optionally save / prefix tfevent files.r   )event_file_loaderr   c                      s&   e Zd Zd fddZ  ZS )z-TBDirWatcher._loader.<locals>.EventFileLoader	file_pathr   r   r   c                   s   t  | r>t|v rtd| d S tj|}ttj	|}r4|d kr4|
  tjj| }t||d d S d S )Nz&Not persisting remote tfevent file: %s)r   r   r   r   )superr[   REMOTE_FILE_TOKENloggerwarningr   r   r#   rb   r?   popr!   r3   )rZ   r   r\   parts)	__class___loader_interface_loader_settingsrk   rm   r1   r2   r[      s$   
z6TBDirWatcher._loader.<locals>.EventFileLoader.__init__)r   r   r   r   )rz   r{   r|   r[   __classcell__r1   r   r   rk   rm   )r   r2   r      s    r   )r   rS   rR   $tensorboard.backend.event_processingr   ImportError	Exceptionr   )rZ   rm   rk   r   r   r1   r   r2   r      s   zTBDirWatcher._loadershutdown_callc                 C  s   z%| j  | j D ]}| | q
W d    W d S 1 sw   Y  W d S  | jjtttfy^ } z't	
d| | j sL|sStt W Y d }~d S W Y d }~d S W Y d }~d S d }~ww )Nz3Encountered tensorboard directory watcher error: %s)r   r   Loadprocess_eventr   DirectoryDeletedErrorStopIterationRuntimeErrorOSErrorr   debugr   is_settimesleepERROR_DELAY)rZ   r   eventer1   r1   r2   _process_events  s$   &zTBDirWatcher._process_eventsc                 C  *   z|    W d S  ty   td  w )Nz(generic exception in TBDirWatcher thread_thread_bodyr   r   	exceptionr   r1   r1   r2   r   #     
z TBDirWatcher._thread_except_bodyc                 C  sF   d}	 |    | j rt }|s|t }n||krdS td q)z"Check for new events every second.NTr   )r   r   r   r   SHUTDOWN_DELAYr   )rZ   shutdown_timenowr1   r1   r2   r   *  s   


zTBDirWatcher._thread_bodyr   r   c                 C  sJ   | j d u r	|j| _ |dr|j| _|dr#| jt|| j d S d S )Nfile_versionsummary)	r   	wall_timeHasFieldr   r   r   putr   r   )rZ   r   r1   r1   r2   r   7  s   


zTBDirWatcher.process_eventc                 C  s   | j dd | j  d S )NT)r   )r   r   setr   r1   r1   r2   ru   B  s   zTBDirWatcher.shutdownc                 C  s   |    | j  d S rP   )ru   r   r!   r   r1   r1   r2   rv   F  s   zTBDirWatcher.finishrx   )r~   rK   r\   r   rm   r8   rk   r5   rV   r
   rO   r8   r   r   ry   )r   r   r   r8   )TN)rm   r8   rk   r5   r   r   )r   r8   r   r   )r   r   r   r   )rz   r{   r|   r[   rp   r   r   r   r   r   r   ru   rv   r1   r1   r1   r2   rq      s    
#
#



rq   c                   @  s$   e Zd ZdZdddZdddZdS )r   z-An event wrapper to enable priority queueing.r   r   rk   r5   c                 C  s   || _ || _t | _d S rP   )r   rk   r   
created_at)rZ   r   rk   r1   r1   r2   r[   N  s   zEvent.__init__otherr   r8   c                 C  s   | j j|j jk S rP   )r   r   )rZ   r   r1   r1   r2   __lt__S  s   zEvent.__lt__N)r   r   rk   r5   )r   r   r   r8   )rz   r{   r|   __doc__r[   r   r1   r1   r1   r2   r   K  s    
r   c                   @  s^   e Zd ZdZ	d$d%ddZd&ddZd&ddZd&ddZd&ddZ	d'd(ddZ	d)d"d#Z
dS )*ro   zConsume tfevents from a priority queue.

    There should always only be one of these per run_manager.  We wait for 10 seconds of
    queued events to reduce the chance of multiple tfevent files triggering out of order
    steps.
    
   r~   rK   rV   r
   rN   r   r   r   delayrC   r   r   c                   sf   | _ | _tj jd _t  _t  _	| _
d fdd}t||| _ j j j d S )	Nr   fnamefilesystem.GlobStrr   r   c                   s"   t | dfgd} jj| d S )Nr   r   )r*   r   rS   r)   )r   r   r   r1   r2   datatypes_cbq  s   z.TBEventConsumer.__init__.<locals>.datatypes_cb)r   r   r   r   )r   r   r   r   r   r   r   r   	TBHistory
tb_history_delayinternal_runInternalRun_internal_run_set_internal_run_interfacerS   )rZ   r~   rV   rN   r   r   r   r1   r   r2   r[   _  s   
zTBEventConsumer.__init__c                 C  s   t   | _| j  d S rP   )r   _start_timer   rp   r   r1   r1   r2   rp   y  s   
zTBEventConsumer.startc                 C  sv   d| _ | j  | j  | j s9| jdd}|r2| j|| j	d | j	
 }|D ]}| | q*| j rd S d S )Nr   Tr   history)r   r   r   r   r!   r   emptyget_handle_eventr   _get_and_reset	_save_rowrZ   r   itemsitemr1   r1   r2   rv   }  s   



zTBEventConsumer.finishc                 C  r   )Nz+generic exception in TBEventConsumer threadr   r   r1   r1   r2   r     r   z#TBEventConsumer._thread_except_bodyc                 C  s   	 z%| j dd}t | j| j k r%| j s%| j | td W q W n t	j
y9   d }| j r7Y nY nw |rS| j|| jd | j }|D ]}| | qKq| j  | j }|D ]}| | q`d S )NTr   g?r   )r   r   r   r   r   r   r   r   r   rV   Emptyr   r   r   r   _flushr   r1   r1   r2   r     s:   




zTBEventConsumer._thread_bodyNr   r   r   TBHistory | Nonec                 C  s    t jj|j|jj|j|d d S )N)steprk   r   )rW   rX   _logr   r   rk   )rZ   r   r   r1   r1   r2   r     s   
zTBEventConsumer._handle_eventrowHistoryDictc                 C  s   t  }| D ] \}}t|tr'|| || | jjj|j	j
|j	jd q|D ]}||}t|tr=|j||j	j< q*| jjj| j|dd d S )N)keyvalF)publish_step)r   r   
isinstancer   rs   set_keyr   rS   publish_configspec
config_keyconfig_valuer   table	table_keypublish_historyr   )rZ   r   
chart_keyskvchartr1   r1   r2   r     s(   





zTBEventConsumer._save_row)r   )r~   rK   rV   r
   rN   r   r   r   r   rC   r   r   ry   rP   )r   r   r   r   r   r   )r   r   r   r   )rz   r{   r|   r   r[   rp   rv   r   r   r   r   r1   r1   r1   r2   ro   W  s    




ro   c                   @  sZ   e Zd ZU ded< ded< dddZdd	d
ZdddZdddZdddZdddZ	dS )r   r   _datalist[HistoryDict]_addedr   r   c                 C  s   d| _ d| _t | _g | _d S )Nr   )_step
_step_sizer*   r   r  r   r1   r1   r2   r[     s   
zTBHistory.__init__c                 C  s   | j sd S | jtjkrZdd | j  D }|jdd dd d}g }|D ]\}}| j| tjd k r4 n||7 }|| | j |= q$td	| j	 d
t
| d tdd|  | j	| j d< | j| j  |  j	d7  _	d| _d S )Nc                 S  s   g | ]\}}|t |fqS r1   )sys	getsizeof).0r   r   r1   r1   r2   
<listcomp>  s    z$TBHistory._flush.<locals>.<listcomp>c                 S  s   | d S )Nr   r1   )tr1   r1   r2   <lambda>  s    z"TBHistory._flush.<locals>.<lambda>T)r   reverser   i zStep z" exceeds max data limit, dropping z of the largest keys:	z
	r  r   )r   r  r   MAX_LINE_BYTESr   sortappendrW   termwarnr  rh   printr!   r  )rZ   metricsbaddropped_keysr   r   r1   r1   r2   r     s*   


zTBHistory._flushdc                 C  s&   |    t | _| j| | d S rP   )r   r*   r   update_track_history_dictrZ   r  r1   r1   r2   rs     s   zTBHistory.addc                 C  s6   i }|D ]}|| ||< |  j t|| 7  _ q|S rP   )r  r  r  )rZ   r  r   r   r1   r1   r2   r    s
   zTBHistory._track_history_dictc                 C  s   | j | | d S rP   )r   r  r  r  r1   r1   r2   _row_update  s   zTBHistory._row_updatec                 C  s   | j d d  }g | _ |S rP   )r  )rZ   addedr1   r1   r2   r     s   zTBHistory._get_and_resetNry   )r  r   r   r   )r  r   r   r   )r   r  )
rz   r{   r|   r}   r[   r   rs   r  r  r   r1   r1   r1   r2   r     s   
 




r   )
r   r   r   r   r   r   r   r   r   r   )r   r   r4   r5   r6   r7   r   r8   )4r   
__future__r   r,   loggingr   rV   r   r  r   r   typingr   r   rW   r   
wandb.plotr   wandb.sdk.libr   r^   r	   r   r
   6tensorboard.backend.event_processing.event_file_loaderr   "tensorboard.compat.proto.event_pb2r   wandb.proto.wandb_internal_pb2r   wandb.sdk.lib.filesystemr   interface.interface_queuer   settings_staticr   r*   r   r   r   r   r   	getLoggerrz   r   r3   rJ   rK   rq   r   ro   r   r1   r1   r1   r2   <module>   sJ    


.O y