o
    -wi?                     @   sv  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	m
Z
mZmZmZ d dlmZ d dlmZ d dlmZ erd dlm  m  m  mZ d dlm  m  m  mZ d dlm  m  m  mZ  d dlm!Z! d dl"m#Z# d dl$m%Z% n
e&d	Ze&d
Z e'Z(e)e*Z+G dd de j,Z-G dd de-Z.G dd de-Z/G dd de-Z0G dd dZ1dS )    N)TYPE_CHECKINGAnyMappingMutableMapping
MutableSetOptional)util)GlobStr)LogicalPath)
PolicyName)
FilePusher)SettingsStaticz wandb_watchdog.observers.pollingzwandb_watchdog.eventsc                   @   s   e Zd Zdededddededdfd	d
Zeej	dddZ
ej	ddeddfddZej	dddZdededdfddZdS )FileEventHandler	file_path	save_namefile_pusherr   argskwargsreturnNc                 O   s    || _ t|| _|| _d | _d S N)r   r
   r   _file_pusher
_last_sync)selfr   r   r   r   r    r   W/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/filesync/dir_watcher.py__init__    s   

zFileEventHandler.__init__r   c                 C      t r   NotImplementedErrorr   r   r   r   policy.   s   zFileEventHandler.policyFforcec                 C   r   r   r   r   r!   r   r   r   on_modified3      zFileEventHandler.on_modifiedc                 C   r   r   r   r   r   r   r   finish7   r$   zFileEventHandler.finishnew_pathnew_namec                 C   s   || _ || _|   d S r   )r   r   r#   )r   r&   r'   r   r   r   
on_renamed;   s   zFileEventHandler.on_renamedr   r   Fr   N)__name__
__module____qualname__PathStrr
   r   r   propertyabcabstractmethodr    boolr#   r%   r(   r   r   r   r   r      s,    
r   c                   @   <   e Zd ZdZddeddfddZddd	ZedddZdS )	PolicyNowz#This policy only uploads files now.Fr!   r   Nc                 C   s8   | j d u s|r| j| j| j tj| j| _ d S d S r   )r   r   file_changedr   r   ospathgetmtimer"   r   r   r   r#   D   s   zPolicyNow.on_modifiedc                 C      d S r   r   r   r   r   r   r%   J      zPolicyNow.finishr   c                 C      dS )Nnowr   r   r   r   r   r    M   r$   zPolicyNow.policyr*   r+   r)   	r,   r-   r.   __doc__r3   r#   r%   r0   r    r   r   r   r   r5   A   s    
r5   c                   @   r4   )	PolicyEndz/This policy only updates at the end of the run.Fr!   r   Nc                 C   r:   r   r   r"   r   r   r   r#   U   r;   zPolicyEnd.on_modifiedc                 C   s*   t j| j| _| jj| j| jdd d S )NF)copy)r7   r8   r9   r   r   r   r6   r   r   r   r   r   r%   Y   s   zPolicyEnd.finishr   c                 C   r<   )Nendr   r   r   r   r   r    _   r$   zPolicyEnd.policyr*   r+   r)   r>   r   r   r   r   r@   R   s    
r@   c                       s   e Zd ZdZdZeejZdZ		d"de
deddd	ed
 dededdf fddZedefddZededefddZdefddZd#deddfddZd$ddZd$ddZed%d d!Z  ZS )&
PolicyLivezEvent handler that uploads respecting throttling.

    Uploads files every RATE_LIMIT_SECONDS, which changes as the size increases to deal
    with throttling.
       g333333?Nr   r   r   r   settingsr   r   r   r   c                    sZ   t  j|||g|R i | d | _d| _|d ur(|jd ur"|j| _|j| _d S d | _d S Nr   )superr   _last_uploaded_time_last_uploaded_sizex_live_policy_rate_limitRATE_LIMIT_SECONDSx_live_policy_wait_time_min_wait_time)r   r   r   r   rE   r   r   	__class__r   r   r   p   s   	

zPolicyLive.__init__c                 C   s   t j| jS r   )r7   r8   getsizer   r   r   r   r   current_size   s   zPolicyLive.current_sizesizec                 C   sB   |d| j d  k rdS |d| j d  k rdS || j d k rdS dS )	N
   MB<   d   i,  GBiX  i  )	unit_dict)clsrR   r   r   r   min_wait_for_size   s   zPolicyLive.min_wait_for_sizec                 C   sl   | j d ur4t | j  }|| jk rdS t| jdkr)| jt| j }|| jk r)dS || jp2| | jkS dS )NFr   T)	rH   timerK   floatrI   rQ   RATE_LIMIT_SIZE_INCREASErM   rZ   )r   time_elapsedsize_increaser   r   r   should_update   s   


zPolicyLive.should_updateFr!   c                 C   sB   | j dkrd S | jtj| jkrd S |s|  r|   d S d S rF   )rQ   r   r7   r8   r9   r   r`   	save_filer"   r   r   r   r#      s   
zPolicyLive.on_modifiedc                 C   s8   t j| j| _t | _| j| _| j	
| j| j d S r   )r7   r8   r9   r   r   r[   rH   rQ   rI   r   r6   r   r   r   r   r   ra      s   
zPolicyLive.save_filec                 C   s   | j dd d S )NTr!   )r#   r   r   r   r   r%      s   zPolicyLive.finishr   c                 C   r<   )Nliver   r   r   r   r   r       r$   zPolicyLive.policyr   r*   r+   r)   )r,   r-   r.   r?   rK   dictr   POW_10_BYTESrX   r]   r/   r
   r   r   r   r0   intrQ   classmethodr\   rZ   r3   r`   r#   ra   r%   r    __classcell__r   r   rN   r   rC   d   s>    



rC   c                	   @   s   e Zd Z	d%dddddee ddfdd	Zeded
 fddZdeddddfddZ	d&ddZ
d'ddZd(ddZd)ddZded edefd!d"Zd*d#d$ZdS )+
DirWatcherNrE   r   r   r   file_dirr   c                 C   s|   d| _ |p|j| _|| _i | _t t t d| _|| _i | _t	
 | _| jj|  | jdd | j  td|j d S )Nr   )rB   rc   r=   T)	recursivezwatching files in: %s)_file_count	files_dir_dir	_settings_savename_file_policiesset_user_file_policiesr   _file_event_handlers
wd_pollingPollingObserver_file_observerschedule_per_file_event_handlerstartloggerinfo)r   rE   r   rj   r   r   r   r      s    

zDirWatcher.__init__zwd_api.EventEmitterc                 C   s(   z	t t| jjW S  ty   Y d S w r   )nextiterrv   emittersStopIterationr   r   r   r   emitter   s
   zDirWatcher.emitterr8   r    r   c              	   C   s   t tjtj| j|| j}|drn|t|kr#|| j	|< n| j
| | ttj| j|D ]3}t tj|| j}| ||}|j|krcz| j|= W n	 ty\   Y nw | ||}|jdd q6d S )Nmedia/Trb   )r
   r7   r8   relpathjoinrn   
startswithglobescaperp   rr   add_get_file_event_handlerr    rs   KeyErrorr#   )r   r8   r    r   src_pathfehr   r   r   update_policy   s(   

zDirWatcher.update_policy wd_events.FileSystemEventHandlerc                 C   s   t  }| j|_| j|_| j|_tj	
| jtj	dg|_dddtj	
| jdtj	
| jdg|_| jjD ]}|jtj	
| j| q6|S )zOCreate a Watchdog file event handler that does different things for every file.*z*.tmpz*.wandbzwandb-summary.jsonz.*z*/.*)	wd_eventsPatternMatchingEventHandler_on_file_created
on_created_on_file_modifiedr#   _on_file_movedon_movedr7   r8   r   rn   normpath	_patterns_ignore_patternsro   ignore_globsappend)r   file_event_handlerglbr   r   r   rx      s   z"DirWatcher._per_file_event_handlereventwd_events.FileCreatedEventc                 C   s   t d|j tj|jrd S |  jd7  _| jd dkr-| j}|r-t| jd d |_	t
tj|j| j}| |j|  d S )Nzfile/dir created: %s   rV   r   )rz   r{   r   r7   r8   isdirrl   r   rf   _timeoutr
   r   rn   r   r#   )r   r   r   r   r   r   r   r     s   zDirWatcher._on_file_createdwd_events.FileModifiedEventc                 C   sP   t d|j  tj|jrd S ttj|j| j}| 	|j|
  d S )Nzfile/dir modified: )rz   r{   r   r7   r8   r   r
   r   rn   r   r#   )r   r   r   r   r   r   r      s
   zDirWatcher._on_file_modifiedwd_events.FileMovedEventc                 C   s   t d|j d|j  tj|jrd S ttj|j| j	}ttj|j| j	}| 
|j|}|| j|< | j|= ||j| d S )Nzfile/dir moved: z -> )rz   r{   r   	dest_pathr7   r8   r   r
   r   rn   r   rs   r(   )r   r   old_save_namenew_save_namehandlerr   r   r   r   '  s   
zDirWatcher._on_file_movedr   r   c           	         s2    drt| | j| jS  | jvrd v sd v r(t| | j| j| j < nl | jv rM| j  }|dkr8tn|dkr>tnt}|| | j| j| j < nGt}| j	 D ]3\}}|dkr]qTt
|D ]%}ttj| j|}t fdd|D r|dkrt}qa|dkrt}qaqT|| | j| j| j < | j  S )	zGet or create an event handler for a particular file.

        file_path: the file's actual path
        save_name: its path relative to the run directory (aka the watch directory)
        r   tfeventszgraph.pbtxtrc   r=   rB   c                 3   s    | ]} |v V  qd S r   r   ).0pr   r   r   	<genexpr>\  s    z5DirWatcher._get_file_event_handler.<locals>.<genexpr>)r   r5   r   ro   rs   rC   rp   r@   rr   itemslistr   r7   r8   r   rn   any)	r   r   r   policy_namemake_handlerr    globsgpathsr   r   r   r   6  sF   
	




z"DirWatcher._get_file_event_handlerc           	      C   sX  t d z9| j r=d| j_| jj  | j  | j	d 	 z| j
| jjd W n
 tjy6   Y nw q!| j  W n tyG   Y n	 tyO   Y nw t d| j t| jD ]L\}}}|D ]D}tj||}ttj|| j}d}| jjD ]}tt|g|dkrd}t d||  nq}|rqdt d|| | ||  qdq]d S )Nzshutting down directory watcherr   Tzscan: %sFzignored: %s matching glob %szscan save: %s %s)rz   r{   rv   is_aliver   _stopped_eventrq   r   r   queue_eventsdispatch_eventsevent_queuequeueEmptystop	TypeErrorSystemErrorrn   r7   walkr8   r
   r   ro   r   lenfnmatchfilterr   r%   )	r   dirpath_	filenamesfnamer   r   ignoredr   r   r   r   r%   f  sR   



zDirWatcher.finishr   )r   r   )r   r   r   N)r   r   r   N)r   r   r   Nr+   )r,   r-   r.   r   r/   r   r0   r   r	   r   rx   r   r   r   r
   r   r   r%   r   r   r   r   ri      s4    





0ri   )2r1   r   r   loggingr7   r   r[   typingr   r   r   r   r   r   wandbr   wandb.sdk.interface.interfacer	   wandb.sdk.lib.pathsr
   )wandb.vendor.watchdog_0_9_0.observers.apivendorwatchdog_0_9_0	observersapiwd_api-wandb.vendor.watchdog_0_9_0.observers.pollingpollingrt   +wandb.vendor.watchdog_0_9_0.watchdog.eventswatchdogeventsr   r   wandb.sdk.internal.file_pusherr   "wandb.sdk.internal.settings_staticr   vendor_importstrr/   	getLoggerr,   rz   ABCr   r5   r@   rC   ri   r   r   r   r   <module>   s6     


"X