o
    wiV2                     @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlZd dlmZmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z'm(Z( ddl)m*Z* dZ+e,e-Z.e./ej0 e1drd dl2m3Z3 e.4e3ddd ne5 Z6e6/ej0 e7dZ8e69e8 e.4e6 eedddeddZ:G dd de#Z;e
ddG d d! d!Z<e
G d"d# d#Z=	d8d9d*d+Z>dddddd,d:d6d7Z?dS );    )annotationsN)Iterable)	dataclass)Path)Any)retrystop_after_attemptwait_random_exponential)Artifact)wandb_internal_pb2)wandb_telemetry_pb2)file_policy_to_enum)InterfaceQueue)context)SendManager)SettingsStatic)coalescerecursive_cast_dictlike_to_dict   )ImporterRunz./wandb-importer"WANDB_IMPORTER_ENABLE_RICH_LOGGING)RichHandlerT)rich_tracebackstracebacks_show_localsz)%(asctime)s - %(levelname)s - %(message)s
   )
multipliermax   )waitstopc                      s   e Zd Z fddZ  ZS )AlternateSendManagerc                   s"   t  j|i | t| j| _d S N)super__init__	exp_retry_send_artifact)selfargskwargs	__class__ [/home/ubuntu/.local/lib/python3.10/site-packages/wandb/apis/importers/internals/internal.pyr#   6   s   zAlternateSendManager.__init__)__name__
__module____qualname__r#   __classcell__r+   r+   r)   r,   r    5   s    r    )frozenc                   @  s~   e Zd ZU dZdZded< dZded< dZded< dZded< dZ	ded< dZ
ded	< dZded
< dZded< dZded< dS )SendManagerConfigz4Configure which parts of SendManager tooling to use.Fbooluse_artifactslog_artifactsmetadatafilesmediacodehistorysummaryterminal_outputN)r-   r.   r/   __doc__r4   __annotations__r5   r6   r7   r8   r9   r:   r;   r<   r+   r+   r+   r,   r2   ;   s   
 r2   c                   @  s   e Zd ZU ded< e Zded< ed,ddZ				d-d.ddZd/ddZ	d0ddZ
d0ddZd0ddZd1ddZd2d!d"Z	#d3d4d&d'Zd0d(d)Zd,d*d+Zd	S )5RecordMakerr   runr   	interfacereturnstrc                 C  s<   t t d| j  d}|jddd t d| j  S )N/z/wandbT)parentsexist_ok)r   ROOT_DIRr@   run_idmkdir)r&   pr+   r+   r,   run_dirO   s   zRecordMaker.run_dirN	artifactsIterable[Artifact] | Noneused_artifactsIterable[pb.Record]c                 c  sL    |   V  |r|D ]
}| j|ddV  q
|r"|D ]
}| |V  qdS dS )zsOnly make records required to upload artifacts.

        Escape hatch for adding extra artifacts to a run.
        Tuse_artifactN)_make_run_record_make_artifact_record)r&   rL   rN   artr+   r+   r,   make_artifacts_only_recordsU   s   
	z'RecordMaker.make_artifacts_only_recordsconfigr2   c                 c  s    |   V  |  V  |jp|j}| ||j|j|jV  |jr6| j	  }dur6|D ]
}| j
|ddV  q+|jrM| j  }durM|D ]}| 
|V  qD|jrW|  E dH  |jr_|  V  |jrv| j  }durx|D ]}| |V  qmdS dS dS )z+Make all the records that constitute a run.NTrP   )rR   _make_telem_recordr5   r4   _make_files_recordr7   r8   r9   r@   rN   rS   rL   r:   _make_history_recordsr;   _make_summary_recordr<   logs_make_output_record)r&   rV   include_artifactsrN   artifactrL   linesliner+   r+   r,   make_recordsh   s0   


zRecordMaker.make_records	pb.Recordc                 C  s0  t  }| j |_| j |_| j |_t| j |_t| j d|_|j	
t| j	 g  |j| j  | j }|d urH||_| j }|d urT||_| j }|d ur`||_| j }d|vrmi |d< | j |d d< | j |d d< | j |d d< | jj||jd | jj|dS )N _wandb	code_pathpython_versioncli_version)dataobj)r@   )pb	RunRecordr@   rH   entityprojectr   display_namenotestagsextend
start_timeFromMillisecondshostruntime	run_grouprV   re   rf   rg   rA   _make_config_make_record)r&   r@   rt   ru   rv   rV   r+   r+   r,   rR      s8   



zRecordMaker._make_run_recordc                 C  s(   t  }t jjj|_||_| jj|dS )N)
output_raw)rj   OutputRawRecord
OutputTypeSTDOUToutput_typer`   rA   rx   )r&   r`   ry   r+   r+   r,   r\      s   zRecordMaker._make_output_recordc                 C  s>   i | j  d| j  i}t|}| j|}| jj|dS )N_runtime)r;   )r@   r;   ru   r   rA   _make_summary_from_dictrx   )r&   dr;   r+   r+   r,   rZ      s   
z RecordMaker._make_summary_recordc                 c  s    | j  D ]E}t }| D ]0\}}|j }||_t|t	r&t
|s*|dkr-tj}t|tr8|d}nt|}||_q| jj|d}|V  qd S )NNaNzutf-8)r:   )r@   metricsrj   HistoryRecorditemsitemaddkey
isinstancefloatmathisnannpr   bytesdecodejsondumps
value_jsonrA   rx   )r&   r   r:   kvr   recr+   r+   r,   rY      s   


z!RecordMaker._make_history_recordsr3   r7   r8   r9   c                 C  s   | j  }| j d}|r|d u r|  }|dfg}t }|D ]<\}}	|s,|dr,q |s4|dr4q |s<|dr<q d|v rOt|}
t|
	| j d}|j
 }||_t|	|_q | jj|dS )	Nz/files/wandb-metadata.jsonendz	artifact/zmedia/zcode/r8   /files)r7   )r@   r7   rK   _make_metadata_filerj   FilesRecord
startswithr   rC   relative_tor   pathr   policyrA   rx   )r&   rL   r7   r8   r9   	run_filesmetadata_fnamefiles_recordr   r   rJ   fr+   r+   r,   rX      s(   


zRecordMaker._make_files_recordFr^   r
   c                 C  s   | j |}t| j |_t| j |_t| j |_||_||_d|_	|j
}|ddg7 }|D ]}|j| q2| j j|dS )NTlatestimported)r^   )rA   _make_artifactrC   r@   rH   rm   rl   user_createduse_after_commitfinalize_aliasesaliasesappendrx   )r&   r^   rQ   protor   aliasr+   r+   r,   rS      s   z!RecordMaker._make_artifact_recordc                 C  sX   t  }t  }d|_|j| | j }|r||_| j }|r%||_| j	j
|dS )NT)	telemetry)telem_pbTelemetryRecordFeatureimporter_mlflowfeatureCopyFromr@   rg   rf   rA   rx   )r&   telemr   rg   rf   r+   r+   r,   rW     s   

zRecordMaker._make_telem_recordc           	      C  sz  d}| j  d}tj|dd i }t| j ||d< t| j ||d< t| j ||d< t| j ||d< t| j	 ||d	< t| j
 ||d
< t| j ||d< | j }|d urpt||d< tt||d< | j }|d urt| j |d< | j }|d urtd| j i|d< | d}t|d}|t| W d    |S 1 sw   Y  |S )NzThis data was not capturedr   T)rF   ospythonprogramcudart   username
executablegpu_devices	gpu_count	cpu_counttotalmemoryz/wandb-metadata.jsonw)rK   r   makedirsr   r@   
os_versionrf   r   cuda_versionrt   r   r   	gpus_usedr   r   len	cpus_usedmemory_usedopenwrite)	r&   missing_text	files_dirr   r   r   mem_usedfnamer   r+   r+   r,   r     s8   




zRecordMaker._make_metadata_file)rB   rC   )NN)rL   rM   rN   rM   rB   rO   )rV   r2   rB   rO   )rB   rb   )rB   rO   )
rL   r3   r7   r3   r8   r3   r9   r3   rB   rb   )F)r^   r
   rB   rb   )r-   r.   r/   r>   r   rA   propertyrK   rU   ra   rR   r\   rZ   rY   rX   rS   rW   r   r+   r+   r+   r,   r?   J   s$   
 

"
%




r?   root_dirrC   settings_overridedict[str, Any] | NonerB   r   c                 C  s8   t |i }ttj| d| dd g ddddddd|S )	Nr7   neverTr      iX  <   )x_files_dirr   resumer   ignore_globsdisable_job_creationx_start_timex_syncx_live_policy_rate_limitx_live_policy_wait_timex_file_stream_timeout_seconds)r   r   r   r   join)r   r   _settings_overrider+   r+   r,   _make_settings5  s"   
r   )
extra_artsextra_used_artsrV   	overridesr   r@   r   r   rM   r   rV   SendManagerConfig | Noner   Nonec                C  s   |d u rt  }|r| D ]\}}t| ||fdd qt| }|j}	t|	|}
t }t }t|d}t	
 }t|
||||}|sE|rL|||}n||}|D ]}td| || qS|  d S )Nc                 S  s   | S r!   r+   )r   r+   r+   r,   <lambda>]  s    zsend_run.<locals>.<lambda>)record_qz
Sending r=)r2   r   setattrr?   rK   r   queueQueuer   r   ContextKeeperr    rU   ra   loggerdebugsendfinish)r@   r   r   rV   r   r   r   r   rmr   settingssm_record_qresult_qrA   context_keepersmrecordsrr+   r+   r,   send_runL  s,   	



r   r!   )r   rC   r   r   rB   r   )r@   r   r   rM   r   rM   rV   r   r   r   r   r   rB   r   )@
__future__r   r   loggingr   r   r   collections.abcr   dataclassesr   pathlibr   typingr   numpyr   tenacityr   r   r	   wandbr
   wandb.protor   rj   r   r   wandb.sdk.interface.interfacer   #wandb.sdk.interface.interface_queuer   wandb.sdk.internalr   wandb.sdk.internal.senderr   "wandb.sdk.internal.settings_staticr   
wandb.utilr   r   	protocolsr   rG   	getLoggerr-   r   setLevelINFOgetenvrich.loggingr   
addHandlerStreamHandlerconsole_handler	Formatter	formattersetFormatterr$   r    r2   r?   r   r   r+   r+   r+   r,   <module>   sd    




 l