o
    -wi`=                     @   sN  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZmZ ddlmZ ddlZddlmZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lm Z  dZ!dZ"dZ#G dd dZ$G dd dej%Z&G dd dZ'						d de(de(de(de(deee)  deee)  fddZ*dd Z+dS )!zsync.    N)ListOptional)quote)wandb_internal_pb2)InterfaceQueue)context	datastorehandlersender
tb_watcher)SettingsStatic)
filesystem)check_and_warn_old.wandbz.syncedz
.tfevents.c                   @   s   e Zd ZdddZdd ZdS )	_LocalRunNc                 C   sP   || _ || _tj |d| _tjtj |dd dd d| _d S )Nzoffline-run-   -r   z%Y%m%d_%H%M%S)	pathsyncedosbasename
startswithofflinedatetimestrptimesplit)selfr   r    r   L/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/sync/sync.py__init__   s    
z_LocalRun.__init__c                 C   s   | j S Nr   r   r   r   r   __str__%   s   z_LocalRun.__str__r!   )__name__
__module____qualname__r    r$   r   r   r   r   r      s    
r   c                   @   s`   e Zd Z												dddZdddZdd Zdd	 Zd
d Zdd Zdd Z	dS )
SyncThreadNc                 C   s   t j|  tjdd || _|| _|| _|| _|| _	|| _
|| _|| _|	| _|
| _|| _|| _|| _t | _t| jj d S )NT)disable)	threadingThreadr    wandb_set_internal_process
_sync_list_project_entity_run_id	_job_type_view_verbose_mark_synced_app_url_sync_tensorboard	_log_path_append_skip_consoletempfileTemporaryDirectory_tmp_diratexitregistercleanup)r   	sync_listprojectentityrun_idjob_typeviewverbosemark_syncedapp_urlsync_tensorboardlog_pathappendskip_consoler   r   r   r    *   s"   
zSyncThread.__init__c                 C   s   t  }|| |d}| jr$| jrtd| ntd| ||dfS |dkrM| jr0| j|j_	| j
r8| j
|j_| jr@| j|j_| jrH| j|j_d|j_n%|dv rY| jrY||dfS |dkrd|}||dfS |dkrr|snJ d|}d }||d	fS )
Nrecord_typezRecord:Trun)output
output_rawexitfinalzfinal seen without exitF)r   RecordParseFromString
WhichOneofr3   r4   printr1   rO   rD   r/   rB   r0   rC   r2   rE   controlreq_respr:   )r   dataexit_pbpbrN   r   r   r   	_parse_pbN   s8   











zSyncThread._parse_pbc                 C   s   d}g }d }| j rqtj|r\g }t|D ]\}}}|D ]}	t|	v r-|tj||	 qq|D ]}
|d7 }tjtj	|
}||vrJ|| q1t
|dkr[tjtj|}nt|v rqtjtj	|}|| d}|||fS )Nr   r   )r7   r   r   isdirwalkTFEVENT_SUBSTRINGrL   joindirnameabspathlencommonprefix)r   	sync_itemtb_event_files
tb_logdirstb_rootfilesdirpath__filesftfeventtb_dirr   r   r   _find_tfevent_filesm   s4   


zSyncThread._find_tfevent_filesc                 C   sf   |dur1|dkr| trtd d	S td| d|  t|dkr/tdt| d dS d	S )
z;Return true if this sync item can be synced as tensorboard.Nr   z5Found .wandb file, not streaming tensorboard metrics.zFound z tfevent files in    z directories containing tfevent files. If these represent multiple experiments, sync them individually or pass a list of paths.TF)endswithWANDB_SUFFIXr,   termwarnrW   rd   )r   ri   rh   rg   rf   r   r   r   _setup_tensorboard   s   

zSyncThread._setup_tensorboardc              	   C   sb  | j d u r|j \}}|d| _ t }| jptj	 |_
| jp'tjd |_| j |_d|jj_| j dt|j dt|j dt|j
 }td| d tj  t }t }	t|}
t }tj|j|	t |
|d}|j j!|d}tj"| j#j$|j
t%% d	}t&|' }t(j)||d d
|	|
|d}t*+|j, |j-||j,d t./|||
d}|D ]}|0|d| tj  q|1  d}g d}d}t2|dkrt3|}|4| t2|dkrt3|}|5| t2|dks||d  | }tj6|d
dd |d7 }t2|dkst2|dkr"t3|}|5| t2|dkstj  |1  |1  d S )NrC   T//runs/	Syncing: z ...)settingsrecord_qresult_q	interfacecontext_keeper)rO   )root_dirrD   x_start_timeF)rz   r{   r|   stoppedwriter_qr}   r~   )file_dirr   )r   \|rw   z Uploading data to wandb   )newlineprefixr   )7r0   _apiviewer_server_infogetr   	RunRecordr1   r,   utilgenerate_idrD   r/   auto_project_namerB   rC   	telemetryfeaturesync_tfeventsr6   	url_quoterW   sysstdoutflushqueueQueuer   r   ContextKeeperr
   SendManager	_settings
_interface_make_recordSettingsr=   nametimer   to_protor	   HandleManagerr   mkdir_exists_ok	files_dirsend_runr   	TBWatcheraddfinishrd   nexthandlesendtermlog)r   ri   rh   send_managerviewerrl   	proto_runurlr{   sender_record_qnew_interfacer~   recordrz   settings_statichandle_managerwatchertbprogress_stepspinner_stateslinerZ   
print_liner   r   r   _send_tensorboard   s   







zSyncThread._send_tensorboardc              
   C   sN   z|  W S  ty& } z| r!td| d W Y d}~dS  d}~ww )z0Attempt to scan data, handling incomplete files.z.wandb file is incomplete (z4), be sure to sync this run again once it's finishedN)	scan_dataAssertionErrorin_last_blockr,   ru   )r   dser   r   r   _robust_scan   s   

zSyncThread._robust_scanc                 C   s  | j d urtd| j   | jD ]K}| |\}}}tj|rTt|}tt	dd |}|d u rEt
|s=t|dkrEtd|  qt|dkrTtj||d }| ||||}|rb| jjntj|}| jrmdnd }	tjj||	d}
|r| |||
 qt }z|| W n ty } ztd	| d
|  W Y d }~qd }~ww d }d}d}	 | |}|d u rns| ||\}}}|d urd}|rq|
| |
j s|
jjdd}|
| |
j r|j j!r,|
j"jdd}|#d}|s,|dkr,|j$j%}| j& dt'|j( dt'|j) dt'|j* }td| ddd t+j,-  d}q|
.  | j/rX| j0sX|rX| t1 }t2|d W d    n	1 sSw   Y  td qd S )NzFind logs at: c                 S   s
   |  tS r!   )rs   rt   )rn   r   r   r   <lambda>      
 z SyncThread.run.<locals>.<lambda>r   zSkipping directory: r   allow)resumez.wandb file is empty (z), skipping: FT)blockresult_type
run_resultrw   rx   ry   z ...  )endwzdone.)3r8   rW   r.   rq   r   r   r^   listdirlistfilterr   rd   ra   rv   r=   r   rb   r9   r
   r   setupr   r   	DataStoreopen_for_scanr   r   r]   r   	_record_qemptyr   rX   rY   	_result_qrV   r   rO   r6   r   rC   rB   rD   r   r   r   r   r5   r3   SYNCED_SUFFIXopen)r   rf   rg   rh   ri   rj   filtered_filessync_tbr   r   smr   r   r[   finishedshownrZ   r\   contresultr   rr   synced_filer   r   r   rO      s   










zSyncThread.runNNNNNNNNNNNNr!   )
r%   r&   r'   r    r]   rq   rv   r   r   rO   r   r   r   r   r(   )   s(    

$Sr(   c                   @   sV   e Zd Z												dddZdd Zdd Zdd	 Zd
d Zdd ZdS )SyncManagerNc                 C   sX   g | _ d | _|| _|| _|| _|| _|| _|| _|| _|| _	|	| _
|
| _|| _|| _d S r!   )r.   _threadr/   r0   r1   r2   r5   r6   r3   r4   r7   r8   r9   r:   )r   rB   rC   rD   rE   rH   rI   rF   rG   rJ   rK   rL   rM   r   r   r   r    G  s   
zSyncManager.__init__c                 C   s   d S r!   r   r#   r   r   r   statuse  s   zSyncManager.statusc                 C   s   | j tjt| d S r!   )r.   rL   r   r   rc   str)r   pr   r   r   r   h  s   zSyncManager.addc                 C   sL   t | j| j| j| j| j| j| j| j| j	| j
| j| j| jd| _| j  d S )N)rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   )r(   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r   startr#   r   r   r   r   k  s    zSyncManager.startc                 C   s   | j   S r!   )r   is_aliver#   r   r   r   is_done~  s   zSyncManager.is_donec                 C   s   t d dS )Nr   F)r   sleepr#   r   r   r   poll  s   
zSyncManager.pollr   )	r%   r&   r'   r    r   r   r   r   r   r   r   r   r   r   F  s&    
r   TFinclude_offlineinclude_onlineinclude_syncedinclude_unsyncedexclude_globsinclude_globsc              	   C   s  t jdrdnd}t j|sdS t |}g }| r$|tdd |7 }|r/|tdd |7 }g }	|  |D ]Q}
t t j||
}|r\t|}|D ]}|tt|| }qKt	|}|rst }|D ]}|
t||}qct	|}|D ]}|tr|	t j||
| quq7g }|	D ]0}t j|}t j| t st j|dr|r|t|d q|r|t|d	 qt|S )
Nr   r,   r   c                 S   
   |  dS )Nzoffline-run-r   _dr   r   r   r     r   zget_runs.<locals>.<lambda>c                 S   r   )Nr   r   r   r   r   r   r     r   r   TF)r   r   existsr   r   sortra   setfnmatchr   unionrs   rt   rL   rb   r   r   r   r   tuple)r   r   r   r   r   r   baseall_dirsdirsfnamesdpathsg	new_pathsrn   filtereddnamer   r   r   get_runs  sR   	

 r  c                 C   s   t | S r!   )r   r"   r   r   r   get_run_from_path  s   r  )TTFTNN),__doc__r>   r   r   r   r   r   r;   r*   r   typingr   r   urllib.parser   r   r,   wandb.protor   #wandb.sdk.interface.interface_queuer   wandb.sdk.internalr   r   r	   r
   r   "wandb.sdk.internal.settings_staticr   wandb.sdk.libr   
wandb.utilr   rt   r   r`   r   r+   r(   r   boolr   r  r  r   r   r   r   <module>   s^      A


3