o
    x¡·iÔ?  ã                   @  s†  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
mZmZ d dlmZmZ d dlmZ d dlmZ d dlmZ erˆd dlm  m  m  mZ d dlm  m  m  mZ d dlm  m  m   m!Z" d dl#m$Z$ d d	l%m&Z& d d
lm'Z' n
e (d¡Ze (d¡Z"e)Z*e +e,¡Z-G dd„ dej.ƒZ/G dd„ de/ƒZ0G dd„ de/ƒZ1G dd„ de/ƒZ2G dd„ dƒZ3dS )é    )ÚannotationsN)ÚMappingÚMutableMappingÚ
MutableSet)ÚTYPE_CHECKINGÚAny)Úutil)ÚGlobStr)ÚLogicalPath)Ú
FilePusher)ÚSettingsStatic)Ú
PolicyNamez wandb_watchdog.observers.pollingzwandb_watchdog.eventsc                   @  sV   e Zd Zddd„Zeejddd„ƒƒZejdd dd„ƒZejd!dd„ƒZ	d"dd„Z
dS )#ÚFileEventHandlerÚ	file_pathÚPathStrÚ	save_namer
   Úfile_pusherr   Úargsr   ÚkwargsÚreturnÚNonec                 O  s    || _ t|ƒ| _|| _d | _d S ©N)r   r
   r   Ú_file_pusherÚ
_last_sync)Úselfr   r   r   r   r   © r   úN/home/ubuntu/.local/lib/python3.10/site-packages/wandb/filesync/dir_watcher.pyÚ__init__#   s   

zFileEventHandler.__init__r   c                 C  ó   t ‚r   ©ÚNotImplementedError©r   r   r   r   Úpolicy1   s   zFileEventHandler.policyFÚforceÚboolc                 C  r   r   r   ©r   r#   r   r   r   Úon_modified6   ó   zFileEventHandler.on_modifiedc                 C  r   r   r   r!   r   r   r   Úfinish:   r'   zFileEventHandler.finishÚnew_pathÚnew_namec                 C  s   || _ || _|  ¡  d S r   )r   r   r&   )r   r)   r*   r   r   r   Ú
on_renamed>   s   zFileEventHandler.on_renamedN)r   r   r   r
   r   r   r   r   r   r   r   r   ©r   r   ©F©r#   r$   r   r   ©r   r   )r)   r   r*   r
   r   r   )Ú__name__Ú
__module__Ú__qualname__r   ÚpropertyÚabcÚabstractmethodr"   r&   r(   r+   r   r   r   r   r   "   s    
r   c                   @  ó4   e Zd ZdZdddd„Zdd	d
„Zeddd„ƒZdS )Ú	PolicyNowz#This policy only uploads files now.Fr#   r$   r   r   c                 C  s8   | j d u s|r| j | j| j¡ tj | j¡| _ d S d S r   )r   r   Úfile_changedr   r   ÚosÚpathÚgetmtimer%   r   r   r   r&   G   s   þzPolicyNow.on_modifiedc                 C  ó   d S r   r   r!   r   r   r   r(   M   ó   zPolicyNow.finishr   c                 C  ó   dS )NÚnowr   r!   r   r   r   r"   P   r'   zPolicyNow.policyNr-   r.   r/   r,   ©r0   r1   r2   Ú__doc__r&   r(   r3   r"   r   r   r   r   r7   D   s    
r7   c                   @  r6   )Ú	PolicyEndz/This policy only updates at the end of the run.Fr#   r$   r   r   c                 C  r<   r   r   r%   r   r   r   r&   X   r=   zPolicyEnd.on_modifiedc                 C  s*   t j | j¡| _| jj| j| jdd d S )NF)Úcopy)r9   r:   r;   r   r   r   r8   r   r!   r   r   r   r(   \   s   zPolicyEnd.finishr   c                 C  r>   )NÚendr   r!   r   r   r   r"   b   r'   zPolicyEnd.policyNr-   r.   r/   r,   r@   r   r   r   r   rB   U   s    
rB   c                      sŒ   e Zd ZdZdZeejƒZdZ		d)d*‡ fdd„Z
ed+dd„ƒZed,dd„ƒZd-dd„Zd.d/d d!„Zd0d"d#„Zd0d$d%„Zed1d'd(„ƒZ‡  ZS )2Ú
PolicyLivez¤Event handler that uploads respecting throttling.

    Uploads files every RATE_LIMIT_SECONDS, which changes as the size increases to deal
    with throttling.
    é   g333333ó?Nr   r   r   r
   r   r   ÚsettingsúSettingsStatic | Noner   r   r   r   r   c                   sZ   t ƒ j|||g|¢R i |¤Ž d | _d| _|d ur(|jd ur"|j| _|j| _d S d | _d S ©Nr   )Úsuperr   Ú_last_uploaded_timeÚ_last_uploaded_sizeÚx_live_policy_rate_limitÚRATE_LIMIT_SECONDSÚx_live_policy_wait_timeÚ_min_wait_time)r   r   r   r   rG   r   r   ©Ú	__class__r   r   r   s   s   	

zPolicyLive.__init__Úintc                 C  s   t j | j¡S r   )r9   r:   Úgetsizer   r!   r   r   r   Úcurrent_size†   s   zPolicyLive.current_sizeÚsizeÚfloatc                 C  sB   |d| j d  k rdS |d| j d  k rdS || j d k rdS dS )	Né
   ÚMBé<   éd   i,  ÚGBiX  i°  )Ú	unit_dict)ÚclsrV   r   r   r   Úmin_wait_for_sizeŠ   s   zPolicyLive.min_wait_for_sizer$   c                 C  sl   | j d ur4t ¡ | j  }|| jk rdS t| jƒdkr)| jt| jƒ }|| jk r)dS || jp2|  | j¡kS dS )NFr   T)	rK   ÚtimerN   rW   rL   rU   ÚRATE_LIMIT_SIZE_INCREASErP   r_   )r   Útime_elapsedÚsize_increaser   r   r   Úshould_update•   s   


ÿzPolicyLive.should_updateFr#   c                 C  sB   | j dkrd S | jtj | j¡krd S |s|  ¡ r|  ¡  d S d S rI   )rU   r   r9   r:   r;   r   rd   Ú	save_filer%   r   r   r   r&   ©   s   
ÿzPolicyLive.on_modifiedc                 C  s8   t j | j¡| _t ¡ | _| j| _| j	 
| j| j¡ d S r   )r9   r:   r;   r   r   r`   rK   rU   rL   r   r8   r   r!   r   r   r   re   ±   s   
zPolicyLive.save_filec                 C  s   | j dd d S )NT©r#   )r&   r!   r   r   r   r(   ·   s   zPolicyLive.finishr   c                 C  r>   )NÚliver   r!   r   r   r   r"   º   r'   zPolicyLive.policyr   )r   r   r   r
   r   r   rG   rH   r   r   r   r   r   r   )r   rS   )rV   rS   r   rW   )r   r$   r-   r.   r/   r,   )r0   r1   r2   rA   rN   Údictr   ÚPOW_10_BYTESr]   ra   r   r3   rU   Úclassmethodr_   rd   r&   re   r(   r"   Ú__classcell__r   r   rQ   r   rE   g   s"    
û



rE   c                   @  sn   e Zd Z	d+d,d
d„Zed-dd„ƒZd.dd„Zd/dd„Zd0dd„Zd1dd„Z	d2d d!„Z
d3d'd(„Zd4d)d*„ZdS )5Ú
DirWatcherNrG   r   r   r   Úfile_dirúPathStr | Noner   r   c                 C  s|   d| _ |p|j| _|| _i | _tƒ tƒ tƒ dœ| _|| _i | _t	 
¡ | _| jj|  ¡ | jdd | j ¡  t d|j¡ d S )Nr   )rD   rg   r?   T)Ú	recursivezwatching files in: %s)Ú_file_countÚ	files_dirÚ_dirÚ	_settingsÚ_savename_file_policiesÚsetÚ_user_file_policiesr   Ú_file_event_handlersÚ
wd_pollingÚPollingObserverÚ_file_observerÚscheduleÚ_per_file_event_handlerÚstartÚloggerÚinfo)r   rG   r   rm   r   r   r   r   À   s    ý
ÿ
zDirWatcher.__init__úwd_api.EventEmitter | Nonec                 C  s(   z	t t| jjƒƒW S  ty   Y d S w r   )ÚnextÚiterrz   ÚemittersÚStopIterationr!   r   r   r   ÚemitterØ   s
   ÿzDirWatcher.emitterr:   r	   r"   r   c              	   C  sØ   t tj tj | j|¡| j¡ƒ}| d¡rn|t |¡kr#|| j	|< n| j
|  |¡ t tj | j|¡¡D ]3}t tj || j¡ƒ}|  ||¡}|j|krcz| j|= W n	 ty\   Y nw |  ||¡}|jdd q6d S )Númedia/Trf   )r
   r9   r:   ÚrelpathÚjoinrr   Ú
startswithÚglobÚescapert   rv   ÚaddÚ_get_file_event_handlerr"   rw   ÚKeyErrorr&   )r   r:   r"   r   Úsrc_pathÚfehr   r   r   Úupdate_policyß   s(   ÿ

þõzDirWatcher.update_policyú wd_events.FileSystemEventHandlerc                 C  sŽ   t  ¡ }| j|_| j|_| j|_tj	 
| jtj	 d¡¡g|_dddtj	 
| jd¡tj	 
| jd¡g|_| jjD ]}|j tj	 
| j|¡¡ q6|S )zOCreate a Watchdog file event handler that does different things for every file.Ú*z*.tmpz*.wandbzwandb-summary.jsonz.*z*/.*)Ú	wd_eventsÚPatternMatchingEventHandlerÚ_on_file_createdÚ
on_createdÚ_on_file_modifiedr&   Ú_on_file_movedÚon_movedr9   r:   rˆ   rr   ÚnormpathÚ	_patternsÚ_ignore_patternsrs   Úignore_globsÚappend)r   Úfile_event_handlerÚglbr   r   r   r|   ý   s   ûz"DirWatcher._per_file_event_handlerÚeventúwd_events.FileCreatedEventc                 C  s†   t  d|j¡ tj |j¡rd S |  jd7  _| jd dkr-| j}|r-t| jd ƒd |_	t
tj |j| j¡ƒ}|  |j|¡ ¡  d S )Nzfile/dir created: %sé   r[   r   )r~   r   r   r9   r:   Úisdirrp   r…   rS   Ú_timeoutr
   r‡   rr   r   r&   )r   r¢   r…   r   r   r   r   r–     s   zDirWatcher._on_file_createdúwd_events.FileModifiedEventc                 C  sP   t  d|j› ¡ tj |j¡rd S ttj |j| j¡ƒ}|  	|j|¡ 
¡  d S )Nzfile/dir modified: )r~   r   r   r9   r:   r¥   r
   r‡   rr   r   r&   )r   r¢   r   r   r   r   r˜   #  s
   zDirWatcher._on_file_modifiedúwd_events.FileMovedEventc                 C  sŠ   t  d|j› d|j› ¡ tj |j¡rd S ttj |j| j	¡ƒ}ttj |j| j	¡ƒ}|  
|j|¡}|| j|< | j|= | |j|¡ d S )Nzfile/dir moved: z -> )r~   r   r   Ú	dest_pathr9   r:   r¥   r
   r‡   rr   r   rw   r+   )r   r¢   Úold_save_nameÚnew_save_nameÚhandlerr   r   r   r™   *  s   
zDirWatcher._on_file_movedr   r   r   r
   r   c           	        s2  ˆ   d¡rt|ˆ | j| jƒS ˆ | jvr”dˆ v sdˆ v r(t|ˆ | j| jƒ| jˆ < nlˆ | jv rM| jˆ  }|dkr8tn|dkr>tnt}||ˆ | j| jƒ| jˆ < nGt}| j 	¡ D ]3\}}|dkr]qTt
|ƒD ]%}t tj | j|¡¡}t‡ fdd„|D ƒƒr†|dkr€t}qa|dkr†t}qaqT||ˆ | j| jƒ| jˆ < | jˆ  S )	z½Get or create an event handler for a particular file.

        file_path: the file's actual path
        save_name: its path relative to the run directory (aka the watch directory)
        r†   Útfeventszgraph.pbtxtrg   r?   rD   c                 3  s    | ]}ˆ |v V  qd S r   r   )Ú.0Úp©r   r   r   Ú	<genexpr>_  s   € z5DirWatcher._get_file_event_handler.<locals>.<genexpr>)r‰   r7   r   rs   rw   rE   rt   rB   rv   ÚitemsÚlistrŠ   r9   r:   rˆ   rr   Úany)	r   r   r   Úpolicy_nameÚmake_handlerr"   ÚglobsÚgÚpathsr   r°   r   r   9  sF   
	
ÿ

ÿÿûÿ€ú
ÿ
z"DirWatcher._get_file_event_handlerc           	      C  sX  t  d¡ z9| j ¡ r=d| j_| jj ¡  | j ¡  | j 	d¡ 	 z| j 
| jjd¡ W n
 tjy6   Y nw q!| j ¡  W n tyG   Y n	 tyO   Y nw t  d| j¡ t | j¡D ]L\}}}|D ]D}tj ||¡}ttj || j¡ƒ}d}| jjD ]}tt |g|¡ƒdkr•d}t  d||¡  nq}|r™qdt  d||¡ |  ||¡ ¡  qdq]d S )Nzshutting down directory watcherr   Tzscan: %sFzignored: %s matching glob %szscan save: %s %s)r~   r   rz   Úis_aliver¦   Ú_stopped_eventru   rˆ   r…   Úqueue_eventsÚdispatch_eventsÚevent_queueÚqueueÚEmptyÚstopÚ	TypeErrorÚSystemErrorrr   r9   Úwalkr:   r
   r‡   rs   rž   ÚlenÚfnmatchÚfilterr   r(   )	r   ÚdirpathÚ_Ú	filenamesÚfnamer   r   Úignoredr¡   r   r   r   r(   i  sR   


ÿÿû
€ÿýôÿzDirWatcher.finishr   )rG   r   r   r   rm   rn   r   r   )r   r€   )r:   r	   r"   r   r   r   )r   r’   )r¢   r£   r   r   )r¢   r§   r   r   )r¢   r¨   r   r   )r   r   r   r
   r   r   r/   )r0   r1   r2   r   r3   r…   r‘   r|   r–   r˜   r™   r   r(   r   r   r   r   rl   ¿   s    ü





0rl   )4Ú
__future__r   r4   rÆ   rŠ   Úloggingr9   r¿   r`   Úcollections.abcr   r   r   Útypingr   r   Úwandbr   Úwandb.sdk.lib.filesystemr	   Úwandb.sdk.lib.pathsr
   Ú)wandb.vendor.watchdog_0_9_0.observers.apiÚvendorÚwatchdog_0_9_0Ú	observersÚapiÚwd_apiÚ-wandb.vendor.watchdog_0_9_0.observers.pollingÚpollingrx   Ú+wandb.vendor.watchdog_0_9_0.watchdog.eventsÚwatchdogÚeventsr”   Úwandb.sdk.internal.file_pusherr   Ú"wandb.sdk.internal.settings_staticr   r   Úvendor_importÚstrr   Ú	getLoggerr0   r~   ÚABCr   r7   rB   rE   rl   r   r   r   r   Ú<module>   s:    


"X