o
    xi>                     @  s"  d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZ ddlZddlmZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZ dZ dZ!dZ"G dd dZ#G dd de
j$Z%G dd dZ&						d"d#ddZ'd d! Z(dS )$zsync.    )annotationsN)quote)wandb_internal_pb2)InterfaceQueue)context	datastorehandlersender
tb_watcher)SettingsStatic)
filesystem)check_and_warn_old.wandbz.syncedz
.tfevents.c                   @  s   e Zd ZdddZdd ZdS )	_LocalRunNc                 C  sP   || _ || _tj |d| _tjtj |dd dd d| _d S )Nzoffline-run-   -r   z%Y%m%d_%H%M%S)	pathsyncedosbasename
startswithofflinedatetimestrptimesplit)selfr   r    r   C/home/ubuntu/.local/lib/python3.10/site-packages/wandb/sync/sync.py__init__   s    
z_LocalRun.__init__c                 C  s   | j S Nr   r   r   r   r   __str__&   s   z_LocalRun.__str__r    )__name__
__module____qualname__r   r#   r   r   r   r   r      s    
r   c                   @  sb   e Zd Z													dddZdddZdd Zdd	 Zd
d Zdd Zdd Z	dS )
SyncThreadNc                 C  s   t j|  || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _|| _|| _|| _|p0i | _t | _t| jj d S r    )	threadingThreadr   
_sync_list_project_entity_run_id	_job_type_view_verbose_mark_synced_app_url_sync_tensorboard	_log_path_append_skip_console_replace_tagstempfileTemporaryDirectory_tmp_diratexitregistercleanup)r   	sync_listprojectentityrun_idjob_typeviewverbosemark_syncedapp_urlsync_tensorboardlog_pathappendskip_consolereplace_tagsr   r   r   r   +   s"   

zSyncThread.__init__c                   s$  t  }|| |d} jr$ jrtd| ntd| ||dfS |dkrh jr0 j|j_	 j
r8 j
|j_ jr@ j|j_ jrH j|j_ jrc fdd|jjD }|jd |jj| d|j_n%|dv rt jrt||dfS |d	kr|}||dfS |d
kr|sJ d|}d }||dfS )Nrecord_typezRecord:Trunc                   s   g | ]	} j ||qS r   )r7   get).0tagr"   r   r   
<listcomp>d   s    z(SyncThread._parse_pb.<locals>.<listcomp>tags)output
output_rawexitfinalzfinal seen without exitF)r   RecordParseFromString
WhichOneofr/   r0   printr-   rM   rA   r+   r?   r,   r@   r.   rB   r7   rR   
ClearFieldextendcontrolreq_respr6   )r   dataexit_pbpbrL   new_tagsr   r"   r   	_parse_pbO   s@   











zSyncThread._parse_pbc                 C  s   d}g }d }| j rqtj|r\g }t|D ]\}}}|D ]}	t|	v r-|tj||	 qq|D ]}
|d7 }tjtj	|
}||vrJ|| q1t
|dkr[tjtj|}nt|v rqtjtj	|}|| d}|||fS )Nr   r   )r3   r   r   isdirwalkTFEVENT_SUBSTRINGrI   joindirnameabspathlencommonprefix)r   	sync_itemtb_event_files
tb_logdirstb_rootfilesdirpath__filesftfeventtb_dirr   r   r   _find_tfevent_filess   s4   


zSyncThread._find_tfevent_filesc                 C  sf   |dur1|dkr| trtd d	S td| d|  t|dkr/tdt| d dS d	S )
z;Return true if this sync item can be synced as tensorboard.Nr   z5Found .wandb file, not streaming tensorboard metrics.zFound z tfevent files in    z directories containing tfevent files. If these represent multiple experiments, sync them individually or pass a list of paths.TF)endswithWANDB_SUFFIXwandbtermwarnrZ   rj   )r   ro   rn   rm   rl   r   r   r   _setup_tensorboard   s   

zSyncThread._setup_tensorboardc              	   C  sb  | j d u r|j \}}|d| _ t }| jptj	 |_
| jp'tjd |_| j |_d|jj_| j dt|j dt|j dt|j
 }td| d tj  t }t }	t|}
t }tj|j|	t |
|d}|j j!|d}tj"| j#j$|j
t%% d	}t&t'|}t(j)||d d
|	|
|d}t*+|j, |j-||j,d t./|||
d}|D ]}|0|d| tj  q|1  d}g d}d}t2|dkrt3|}|4| t2|dkrt3|}|5| t2|dks||d  | }tj6|d
dd |d7 }t2|dkst2|dkr"t3|}|5| t2|dkstj  |1  |1  d S )Nr@   T//runs/	Syncing: z ...)settingsrecord_qresult_q	interfacecontext_keeper)rM   )root_dirrA   x_start_timeF)r   r   r   stoppedwriter_qr   r   )file_dirr   )r   \|r~   z Uploading data to wandb   )newlineprefixr   )7r,   _apiviewer_server_inforN   r   	RunRecordr-   r{   utilgenerate_idrA   r+   auto_project_namer?   r@   	telemetryfeaturesync_tfeventsr2   	url_quoterZ   sysstdoutflushqueueQueuer   r   ContextKeeperr	   SendManager	_settings
_interface_make_recordSettingsr:   nametimer   dictr   HandleManagerr   mkdir_exists_ok	files_dirsend_runr
   	TBWatcheraddfinishrj   nexthandlesendtermlog)r   ro   rn   send_managerviewerrr   	proto_runurlr   sender_record_qnew_interfacer   recordr   settings_statichandle_managerwatchertbprogress_stepspinner_statesliner_   
print_liner   r   r   _send_tensorboard   s   







zSyncThread._send_tensorboardc              
   C  sN   z|  W S  ty& } z| r!td| d W Y d}~dS  d}~ww )z0Attempt to scan data, handling incomplete files.z.wandb file is incomplete (z4), be sure to sync this run again once it's finishedN)	scan_dataAssertionErrorin_last_blockr{   r|   )r   dser   r   r   _robust_scan   s   

zSyncThread._robust_scanc                 C  s  | j d urtd| j   | jD ]K}| |\}}}tj|rTt|}tt	dd |}|d u rEt
|s=t|dkrEtd|  qt|dkrTtj||d }| ||||}|rb| jjntj|}| jrmdnd }	tjj||	d}
|r| |||
 qt }z|| W n ty } ztd	| d
|  W Y d }~qd }~ww d }d}d}	 | |}|d u rns| ||\}}}|d urd}|rq|
| |
j s|
jjdd}|
| |
j r|j j!r,|
j"jdd}|#d}|s,|dkr,|j$j%}| j& dt'|j( dt'|j) dt'|j* }td| ddd t+j,-  d}q|
.  | j/rX| j0sX|rX| t1 }t2|d W d    n	1 sSw   Y  td qd S )NzFind logs at: c                 S  s
   |  tS r    )ry   rz   )rt   r   r   r   <lambda>     
 z SyncThread.run.<locals>.<lambda>r   zSkipping directory: r   allow)resumez.wandb file is empty (z), skipping: FT)blockresult_type
run_resultr~   r   r   z ...  )endwzdone.)3r4   rZ   r*   rw   r   r   rd   listdirlistfilterr   rj   rg   r}   r:   r   rh   r5   r	   r   setupr   r   	DataStoreopen_for_scanr   r   rc   r   	_record_qemptyrN   r]   r^   	_result_qrY   r   rM   r2   r   r@   r?   rA   r   r   r   r   r1   r/   SYNCED_SUFFIXopen)r   rl   rm   rn   ro   rp   filtered_filessync_tbr   r   smr   r   r`   finishedshownr_   ra   contresultr   rr   synced_filer   r   r   rM     s   










zSyncThread.runNNNNNNNNNNNNNr    )
r$   r%   r&   r   rc   rw   r}   r   r   rM   r   r   r   r   r'   *   s*    

$$Xr'   c                   @  sX   e Zd Z													dddZdd Zdd Zdd	 Zd
d Zdd ZdS )SyncManagerNc                 C  sb   g | _ d | _|| _|| _|| _|| _|| _|| _|| _|| _	|	| _
|
| _|| _|| _|p-i | _d S r    )r*   _threadr+   r,   r-   r.   r1   r2   r/   r0   r3   r4   r5   r6   r7   )r   r?   r@   rA   rB   rE   rF   rC   rD   rG   rH   rI   rJ   rK   r   r   r   r   R  s   zSyncManager.__init__c                 C  s   d S r    r   r"   r   r   r   statusr  s   zSyncManager.statusc                 C  s   | j tjt| d S r    )r*   rI   r   r   ri   str)r   pr   r   r   r   u  s   zSyncManager.addc                 C  sP   t | j| j| j| j| j| j| j| j| j	| j
| j| j| j| jd| _| j  d S )N)r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   )r'   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r   startr"   r   r   r   r   x  s"   zSyncManager.startc                 C  s   | j   S r    )r   is_aliver"   r   r   r   is_done  s   zSyncManager.is_donec                 C  s   t d dS )Nr   F)r   sleepr"   r   r   r   poll  s   
zSyncManager.pollr   )	r$   r%   r&   r   r   r   r   r   r   r   r   r   r   r   Q  s(    
 r   TFinclude_offlineboolinclude_onlineinclude_syncedinclude_unsyncedexclude_globslist[str] | Noneinclude_globsc              	   C  s  t jdrdnd}t j|sdS t |}g }| r$|tdd |7 }|r/|tdd |7 }g }	|  |D ]Q}
t t j||
}|r\t|}|D ]}|tt|| }qKt	|}|rst }|D ]}|
t||}qct	|}|D ]}|tr|	t j||
| quq7g }|	D ]0}t j|}t j| t st j|dr|r|t|d q|r|t|d	 qt|S )
Nr   r{   r   c                 S  
   |  dS )Nzoffline-run-r   _dr   r   r   r     r   zget_runs.<locals>.<lambda>c                 S  r   )Nr   r   r   r   r   r   r     r   r   TF)r   r   existsr   r   sortrg   setfnmatchr   unionry   rz   rI   rh   r   r   r   r   tuple)r   r   r   r   r   r   baseall_dirsdirsfnamesdpathsg	new_pathsrt   filtereddnamer   r   r   get_runs  sR   	

 r  c                 C  s   t | S r    )r   r!   r   r   r   get_run_from_path  s   r  )TTFTNN)r   r   r   r   r   r   r   r   r   r   r   r   ))__doc__
__future__r   r;   r   r  r   r   r   r8   r(   r   urllib.parser   r   r{   wandb.protor   #wandb.sdk.interface.interface_queuer   wandb.sdk.internalr   r   r   r	   r
   "wandb.sdk.internal.settings_staticr   wandb.sdk.libr   
wandb.utilr   rz   r   rf   r   r)   r'   r   r  r  r   r   r   r   <module>   sF      )D3