o
    niI                  
   @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m
Z
mZmZmZmZ ddlZddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ e
rdd
lmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddl"m#Z# ddl$m%Z% ee&ef Z'dZ(dZ)dZ*e+e,Z-de&de&ddddddf
ddZ.de&dee& dee/ de0fdd Z1G d!d" d"Z2G d#d$ d$Z3G d%d& d&Z4G d'd( d(Z5G d)d* d*Z6dS )+ztensorboard watcher.    N)TYPE_CHECKINGAnyDictListOptional)util)CustomChart)GlobStr)
filesystem   )run)PriorityQueue)EventFileLoader)
ProtoEvent)	RunRecord)	FilesDict   )InterfaceQueue)SettingsStatic   z://path	base_path	interfacer   settingsr   returnc                 C   s   |j }tj| |}tj| }tj||}ttj| tj	|r9|t
|kr9t| t|| ntj|sEt|| |ttt|dfgd d S )Nlivefiles)	files_dirosr   relpathabspathjoinr
   mkdir_exists_okdirnameislinkreadlinkremovesymlinkexistspublish_filesdictr	   globescape)r   r   r   r   r   	file_nameabs_path
wandb_path r1   [/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/internal/tb_watcher.py_link_and_save_file*   s   
$r3   hostname
start_timec           
   	   C   s   | st dtj| }|drdS |d}z|d}W n
 t y)   Y dS w |durWt|dD ]!\}}z
||d |  }W n tyN   Y  dS w ||krV dS q5|durzz
t	||d  }	W n t tfyq   Y dS w |	t	|k rzdS d	S )
a  Check if a path is a tfevents file.

    Optionally checks that it was created by [hostname] after [start_time].

    tensorboard tfevents filename format:
        https://github.com/tensorflow/tensorboard/blob/f3f26b46981da5bd46a5bb93fcf02d9eb7608bc1/tensorboard/summary/writer/event_file_writer.py#L81
    tensorflow tfevents filename format:
        https://github.com/tensorflow/tensorflow/blob/8f597046dc30c14b5413813d02c0e0aed399c177/tensorflow/core/util/events_writer.cc#L68
    Path must be a nonempty string)z.profile-emptyz.sagemaker-uploadedF.tfeventsNr   r   T)

ValueErrorr   r   basenameendswithsplitindex	enumerate
IndexErrorint)
r   r4   r5   r:   fname_componentstfevents_idxipartfname_component_partcreated_timer1   r1   r2   is_tfevents_file_created_by=   s<   

rG   c                   @   s~   e Zd ZU ded< ded< 	ddddd	d
ddeddf
ddZdededee fddZdedededdfddZ	dddZ
dS )	TBWatcherzDict[str, TBDirWatcher]_logdirsr   _watcher_queueFr   r   	run_protor   r   r   forcer   Nc                 C   s<   i | _ d | _|| _|| _|| _|| _t | _t	j
  d S N)rI   	_consumer	_settings
_interface
_run_proto_forcequeuer   rJ   wandbtensorboardreset_state)selfr   rK   r   rL   r1   r1   r2   __init__o   s   
zTBWatcher.__init__logdirrootdirc                 C   s   t | j|g }tj|rtj|}nd}|dkrCttjtj	|}|
|d
|dd}t|dkrA|dvrAd }|S |
|d
|dd}|S )N /r   )train
validation)listrI   r   r   isfiler:   r   to_forward_slash_pathr$   commonprefixreplacestriplen)rW   rY   rZ   dirsfilename	namespacer1   r1   r2   _calculate_namespace   s   zTBWatcher._calculate_namespacesaveroot_dirc                 C   s   t |}t |}|| jv rd S | ||}| js*t| | j| j| j| _| j	  t
| |||| j| j}|| j|< |	  d S rM   )r   ra   rI   ri   rN   TBEventConsumerrJ   rQ   rO   startTBDirWatcherrR   )rW   rY   rj   rk   rh   tbdir_watcherr1   r1   r2   add   s   




zTBWatcher.addc                 C   sH   | j  D ]}|  q| j  D ]}|  q| jr"| j  d S d S rM   )rI   valuesshutdownfinishrN   )rW   tbdirwatcherr1   r1   r2   rs      s   

zTBWatcher.finishFr   N)__name__
__module____qualname____annotations__boolrX   strr   ri   rp   rs   r1   r1   r1   r2   rH   k   s$   
 
rH   c                   @   s   e Zd Z	d&dddededee ddd	ed
dfddZd'ddZded
efddZ	d(dedee d
dfddZ	d&ded
dfddZ
d'ddZd'ddZd)d d!Zd'd"d#Zd'd$d%ZdS )*rn   F	tbwatcherrH   rY   rj   rh   rS   r   rL   r   Nc                 C   s   t jddd| _t jddd| _|| _| j|| ||| j| _t	j
| jd| _d | _t	 | _|| _d | _|| _|| _t | _|| _t	 | _d S )Nz6tensorboard.backend.event_processing.directory_watcher"Please install tensorboard package)requiredztensorboard.compattarget)r   
get_moduledirectory_watcher	tf_compat
_tbwatcherDirectoryWatcher_loader_is_our_tfevents_file
_generator	threadingThread_thread_except_body_thread_first_event_timestampEvent	_shutdown_queue_file_version
_namespace_logdirsocketgethostname	_hostnamerR   Lock_process_events_lock)rW   r}   rY   rj   rh   rS   rL   r1   r1   r2   rX      s*   	

zTBDirWatcher.__init__c                 C   s   | j   d S rM   )r   rm   rW   r1   r1   r2   rm      s   zTBDirWatcher.startr   c                 C   sB   |st d| jjj|}| jrt|ddS t|| j| jj	j
S )zECheck if a path has been modified since launch and contains tfevents.r6   N)r9   r   tfcompat
as_str_anyrR   rG   r   r   rO   x_start_time)rW   r   r1   r1   r2   r      s   z"TBDirWatcher._is_our_tfevents_fileTr   c                    sX   | j j | j jzddlm} W n ty   tdw G  fddd|j}|S )zKIncredibly hacky class generator to optionally save / prefix tfevent files.r   )event_file_loaderr~   c                       s.   e Zd Zdeddf fddZ  ZS )z-TBDirWatcher._loader.<locals>.EventFileLoader	file_pathr   Nc                    s   t  | r>t|v rtd| d S tj|}ttj	|}r4|d kr4|
  tjj| }t||d d S d S )Nz&Not persisting remote tfevent file: %s)r   r   r   r   )superrX   REMOTE_FILE_TOKENloggerwarningr   r   r$   r_   r<   popr"   r3   )rW   r   rY   parts)	__class___loader_interface_loader_settingsrh   rj   r1   r2   rX      s$   
z6TBDirWatcher._loader.<locals>.EventFileLoader.__init__)rw   rx   ry   r|   rX   __classcell__r1   r   r   rh   rj   )r   r2   r      s    &r   )r   rP   rO   $tensorboard.backend.event_processingr   ImportError	Exceptionr   )rW   rj   rh   r   r   r1   r   r2   r      s   zTBDirWatcher._loadershutdown_callc                 C   s   z%| j  | j D ]}| | q
W d    W d S 1 sw   Y  W d S  | jjtttfy^ } z't	
d| | j sL|sStt W Y d }~d S W Y d }~d S W Y d }~d S d }~ww )Nz3Encountered tensorboard directory watcher error: %s)r   r   Loadprocess_eventr   DirectoryDeletedErrorStopIterationRuntimeErrorOSErrorr   debugr   is_settimesleepERROR_DELAY)rW   r   eventer1   r1   r2   _process_events  s$   &zTBDirWatcher._process_eventsc              
   C   6   z|    W d S  ty } ztd |d }~ww )Nz(generic exception in TBDirWatcher thread_thread_bodyr   r   	exceptionrW   r   r1   r1   r2   r         
z TBDirWatcher._thread_except_bodyc                 C   sF   d}	 |    | j rt }|s|t }n||krdS td q)z"Check for new events every second.NTr   )r   r   r   r   SHUTDOWN_DELAYr   )rW   shutdown_timenowr1   r1   r2   r   '  s   


zTBDirWatcher._thread_bodyr   r   c                 C   sJ   | j d u r	|j| _ |dr|j| _|dr#| jt|| j d S d S )Nfile_versionsummary)	r   	wall_timeHasFieldr   r   r   putr   r   )rW   r   r1   r1   r2   r   4  s   


zTBDirWatcher.process_eventc                 C   s   | j dd | j  d S )NT)r   )r   r   setr   r1   r1   r2   rr   ?  s   zTBDirWatcher.shutdownc                 C   s   |    | j  d S rM   )rr   r   r"   r   r1   r1   r2   rs   C  s   zTBDirWatcher.finishru   rv   )TN)r   r   r   N)rw   rx   ry   r|   r{   r   rX   rm   r   r   r   r   r   r   rr   rs   r1   r1   r1   r2   rn      sB    

#
#



rn   c                   @   s8   e Zd ZdZdddee fddZdd defd	d
ZdS )r   z-An event wrapper to enable priority queueing.r   r   rh   c                 C   s   || _ || _t | _d S rM   )r   rh   r   
created_at)rW   r   rh   r1   r1   r2   rX   K  s   zEvent.__init__otherr   c                 C   s   | j j|j jk r
dS dS )NTF)r   r   )rW   r   r1   r1   r2   __lt__P  s   zEvent.__lt__N)	rw   rx   ry   __doc__r   r|   rX   r{   r   r1   r1   r1   r2   r   H  s    r   c                   @   s   e Zd ZdZ	d!dedddddd	d
eddfddZd"ddZd"ddZd"ddZ	d"ddZ
	d#ddded ddfddZd$dd ZdS )%rl   zConsume tfevents from a priority queue.

    There should always only be one of these per run_manager.  We wait for 10 seconds of
    queued events to reduce the chance of multiple tfevent files triggering out of order
    steps.
    
   r}   rS   r   rK   r   r   r   delayr   Nc                    sn   | _ | _tj jd _t  _t  _	| _
dtdd f fdd}t||| _ j j j d S )Nr   fnamer   c                    s"   t | dfgd} jj| d S )Nr   r   )r+   r   rP   r*   )r   r   r   r1   r2   datatypes_cbp  s   z.TBEventConsumer.__init__.<locals>.datatypes_cb)r   r   r   r   r   r   r   r   	TBHistory
tb_history_delayr	   internal_runInternalRun_internal_run_set_internal_run_interfacerP   )rW   r}   rS   rK   r   r   r   r1   r   r2   rX   ^  s   
zTBEventConsumer.__init__c                 C   s   t   | _| j  d S rM   )r   _start_timer   rm   r   r1   r1   r2   rm   x  s   
zTBEventConsumer.startc                 C   sv   d| _ | j  | j  | j s9| jdd}|r2| j|| j	d | j	
 }|D ]}| | q*| j rd S d S )Nr   Tr   history)r   r   r   r   r"   r   emptyget_handle_eventr   _get_and_reset	_save_rowrW   r   itemsitemr1   r1   r2   rs   |  s   



zTBEventConsumer.finishc              
   C   r   )Nz+generic exception in TBEventConsumer threadr   r   r1   r1   r2   r     r   z#TBEventConsumer._thread_except_bodyc                 C   s   	 z%| j dd}t | j| j k r%| j s%| j | td W q W n t	j
y9   d }| j r7Y nY nw |rS| j|| jd | j }|D ]}| | qKq| j  | j }|D ]}| | q`d S )NTr   g?r   )r   r   r   r   r   r   r   r   r   rS   Emptyr   r   r   r   _flushr   r1   r1   r2   r     s:   




zTBEventConsumer._thread_bodyr   r   r   r   c                 C   s    t jj|j|jj|j|d d S )N)steprh   r   )rT   rU   _logr   r   rh   )rW   r   r   r1   r1   r2   r     s   
zTBEventConsumer._handle_eventrowHistoryDictc                 C   s   t  }| D ] \}}t|tr'|| || | jjj|j	j
|j	jd q|D ]}||}t|tr=|j||j	j< q*| jjj|| jdd d S )N)keyvalF)r   publish_step)r   r   
isinstancer   rp   set_keyr   rP   publish_configspec
config_keyconfig_valuer   table	table_keypublish_historyr   )rW   r   
chart_keyskvchartr1   r1   r2   r     s$   





zTBEventConsumer._save_row)r   rv   rM   )r   r   r   N)rw   rx   ry   r   rH   r@   rX   rm   rs   r   r   r   r   r   r1   r1   r1   r2   rl   V  s:    






rl   c                   @   sZ   e Zd ZU ded< ded< dddZdd	d
ZdddZdddZdddZdddZ	dS )r   r   _dataList[HistoryDict]_addedr   Nc                 C   s   d| _ d| _t | _g | _d S )Nr   )_step
_step_sizer+   r   r   r   r1   r1   r2   rX     s   
zTBHistory.__init__c                 C   s   | j sd S | jtjkrWdd | j  D }|jdd dd d}g }|D ]\}}| j| tjd k r4 n||7 }|| | j |= q$td		| j
t| td
d|  | j
| j d< | j| j  |  j
d7  _
d| _d S )Nc                 S   s   g | ]\}}|t |fqS r1   )sys	getsizeof).0r   r   r1   r1   r2   
<listcomp>  s    z$TBHistory._flush.<locals>.<listcomp>c                 S   s   | d S )Nr   r1   )tr1   r1   r2   <lambda>  s    z"TBHistory._flush.<locals>.<lambda>T)r   reverser   i z@Step {} exceeds max data limit, dropping {} of the largest keys:	z
	r  r   )r   r  r   MAX_LINE_BYTESr   sortappendrT   termwarnformatr  re   printr"   r   )rW   metricsbaddropped_keysr   r   r1   r1   r2   r     s.   



zTBHistory._flushdc                 C   s&   |    t | _| j| | d S rM   )r   r+   r   update_track_history_dictrW   r  r1   r1   r2   rp     s   zTBHistory.addc                 C   s:   i }|  D ]}|| ||< |  jt|| 7  _q|S rM   )keysr  r  r  )rW   r  r   r   r1   r1   r2   r    s
   zTBHistory._track_history_dictc                 C   s   | j | | d S rM   )r   r  r  r  r1   r1   r2   _row_update  s   zTBHistory._row_updatec                 C   s   | j d d  }g | _ |S rM   )r   )rW   addedr1   r1   r2   r     s   zTBHistory._get_and_resetrv   )r  r   r   N)r  r   r   r   )r   r   )
rw   rx   ry   rz   rX   r   rp   r  r  r   r1   r1   r1   r2   r     s   
 




r   )7r   r,   loggingr   rS   r   r  r   r   typingr   r   r   r   r   rT   r   
wandb.plotr   wandb.sdk.interface.interfacer	   wandb.sdk.libr
   r[   r   r   r   6tensorboard.backend.event_processing.event_file_loaderr   "tensorboard.compat.proto.event_pb2r   wandb.proto.wandb_internal_pb2r   r   interface.interface_queuer   settings_staticr   r|   r   r   r   r   	getLoggerrw   r   r3   floatr{   rG   rH   rn   r   rl   r   r1   r1   r1   r2   <module>   sn    


.O w