o
    ni/                     @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m	Z	 ddl
mZmZmZmZ ddlZddlZddlmZ ddlmZmZmZmZmZ errdd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlm Z  e!e"Z#		d&dddddddee$ dee$ ddfddZ%	d'de&de$dee& ddfddZ'G dd dejZ(G d d! d!ejZ)G d"d# d#ejZ*G d$d% d%Z+dS )(a  Internal process.

This module implements the entrypoint for the internal process. The internal process
is responsible for handling "record" requests, and responding with "results". Data is
passed to the process over multiprocessing queues.

Threads:
    HandlerThread -- read from record queue and call handlers
    SenderThread -- send to network
    WriterThread -- write to disk

    N)datetime)TYPE_CHECKINGAnyListOptional   )InterfaceQueue   )contexthandlerinternal_utilsenderwriter)Queue)Event)RecordResult)RecordLoopThread)SettingsStaticsettingsr   record_qQueue[Record]result_qQueue[Result]portuser_pidreturnc              	   C   s  t   t }t jjdt| d tjddd}| }|jr&t	|j|j
 |p+t }t }td	|t| t|d
}	t }
g }t }t }t }t||||
|	d|d}|| t||||
|	||d}|| t||||
||	|d}|| t||d}|D ]}|  qd}|
 szAz|
 st d |! rt"d |
#  |
 rW n t$y   |d7 }t%d|  Y nw W |dkrt"d |
#  n|dkrt"d |
#  w w |
 r|D ]}|&  qddd}|D ]?}|' }|r:tj"d|j( d|d t)d|j( dt*j+d t,j-|  t j.| t /d | j0s5t1d t*2d q|  dS ) zInternal process function entrypoint.

    Read from record queue and dispatch work to various threads.

    Args:
        settings: settings object
        record_q: records to be handled
        result_q: for sending results back

    internal)process_contexttagsargsr   r   Nc                  W   s   t d d S )NzInternal process exited)loggerinfo)r     r#   Y/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/internal/internal.pyhandle_exitG   s   z#wandb_internal.<locals>.handle_exitz6W&B internal server running at pid: %s, started at: %s)r     )r   r   r   stopped	interfacedebounce_interval_mscontext_keeper)r   r   r   r'   r(   sender_qr*   r   r   r   r'   writer_qr(   r*   )r   r   r   r	   zInternal process shutdown.zInternal process interrupt: r   zInternal process interrupted.c                  S   s4   t d} | jd d  D ]}|  | | qd S )Nwandb)logging	getLoggerhandlerscloseremoveHandler)root_handlerr#   r#   r$   close_internal_log   s
   
z*wandb_internal.<locals>.close_internal_logzThread :)exc_info)filez.Internal wandb error: file data was not synced)r    r   r   Nr   N)3r.   _set_internal_processtime_sentryconfigure_scopedictatexitregisterlog_internalconfigure_loggingx_log_levelosgetppidgetpidr!   r"   r   fromtimestampr   	threadingr   r
   ContextKeeperqueuer   SenderThreadappendWriterThreadHandlerThreadProcessCheckstartis_setsleepis_deaderrorsetKeyboardInterruptwarningjoinget_exceptionnameprintsysstderr	tracebackprint_exception	exception	termerrorx_disable_service_exitexit)r   r   r   r   r   startedr%   	_settingspidpublish_interfacer'   threadsr*   send_record_qwrite_record_qrecord_sender_threadrecord_writer_threadrecord_handler_threadprocess_checkthreadinterrupt_countr6   r8   r#   r#   r$   wandb_internal.   s   

	
	
	












rt   	log_fname	log_levelrun_idc                    s   t | }|| G  fdddt j} rt d}nt d}||  r/||  t d}d|_|t j	 |
| d S )Nc                       s"   e Zd Zdddef fddZdS )z#configure_logging.<locals>.WBFilterrecordr   r   c                    s
    |_ dS )NTrw   selfrx   ry   r#   r$   filter   s   z*configure_logging.<locals>.WBFilter.filterN)__name__
__module____qualname__boolr|   r#   ry   r#   r$   WBFilter   s    r   zy%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d [%(run_id)s:%(filename)s:%(funcName)s():%(lineno)s] %(message)szn%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d [%(filename)s:%(funcName)s():%(lineno)s] %(message)sr.   F)r/   FileHandlersetLevelFilter	FormattersetFormatter	addFilterr0   	propagateDEBUG
addHandler)ru   rv   rw   log_handlerr   	formatterr4   r#   ry   r$   rD      s"   



rD   c                       s   e Zd ZU dZded< ded< ded< ejed< 		d#d
ddddddddddddejddddf fddZd$ddZd%ddZ	d$dd Z
d$d!d"Z  ZS )&rP   z9Read records from queue and dispatch to handler routines.r   	_record_qr   	_result_qr   _stopped_context_keeper  r   r   r   r   r'   r-   r(   r   r*   r)   floatr   Nc	           	         sH   t  j||||d d| _|| _|| _|| _|| _|| _|| _|| _	d S )Ninput_record_qr   r'   r)   rP   )
super__init__r\   rh   r   r   r   	_writer_q
_interfacer   )	r{   r   r   r   r'   r-   r(   r*   r)   	__class__r#   r$   r      s   
zHandlerThread.__init__c              	   C   s,   t j| j| j| j| j| j| j| jd| _	d S )Nr,   )
r   HandleManagerrh   r   r   r   r   r   r   _hmr{   r#   r#   r$   _setup   s   zHandlerThread._setuprx   r   c                 C      | j | d S N)r   handlerz   r#   r#   r$   _process     zHandlerThread._processc                 C      | j   d S r   )r   finishr   r#   r#   r$   _finish
     zHandlerThread._finishc                 C   r   r   )r   debouncer   r#   r#   r$   	_debounce  r   zHandlerThread._debouncer   r;   rx   r   r   Nr}   r~   r   __doc____annotations__r
   rK   r   r   r   r   r   __classcell__r#   r#   r   r$   rP      s<   
 
	



rP   c                       s   e Zd ZU dZded< ded< ejed< 	d!dd	d
ddddddddejddddf fddZd"ddZd#ddZ	d"ddZ
d"dd Z  ZS )$rM   z8Read records from queue and dispatch to sender routines.r   r   r   r   r   r&   r   r   r   r   r'   r   r(   r   r*   r)   r   r   Nc                    s<   t  j||||d d| _|| _|| _|| _|| _|| _d S )Nr   rM   )r   r   r\   rh   r   r   r   r   )r{   r   r   r   r'   r(   r*   r)   r   r#   r$   r     s   

zSenderThread.__init__c                 C   s$   t j| j| j| j| j| jd| _d S )N)r   r   r   r(   r*   )r   SendManagerrh   r   r   r   r   _smr   r#   r#   r$   r   /  s   zSenderThread._setuprx   r   c                 C   r   r   )r   sendrz   r#   r#   r$   r   8  r   zSenderThread._processc                 C   r   r   )r   r   r   r#   r#   r$   r   ;  r   zSenderThread._finishc                 C   r   r   )r   r   r   r#   r#   r$   r   >  r   zSenderThread._debounce)r&   r;   r   r   r#   r#   r   r$   rM     s6   
 

	

	
rM   c                       s   e Zd ZU dZded< ded< ejed< 	d"dd	d
ddddddddddejddddf fddZd#ddZd$ddZ	d#ddZ
d#d d!Z  ZS )%rO   z8Read records from queue and dispatch to writer routines.r   r   r   r   r   r   r   r   r   r   r'   r   r(   r   r+   r*   r)   r   r   Nc	           	         sB   t  j||||d d| _|| _|| _|| _|| _|| _|| _d S )Nr   rO   )	r   r   r\   rh   r   r   	_sender_qr   r   )	r{   r   r   r   r'   r(   r+   r*   r)   r   r#   r$   r   I  s   
zWriterThread.__init__c                 C   s(   t j| j| j| j| j| j| jd| _d S )N)r   r   r   r+   r(   r*   )	r   WriteManagerrh   r   r   r   r   r   _wmr   r#   r#   r$   r   b  s   zWriterThread._setuprx   r   c                 C   r   r   )r   writerz   r#   r#   r$   r   l  r   zWriterThread._processc                 C   r   r   )r   r   r   r#   r#   r$   r   o  r   zWriterThread._finishc                 C   r   r   )r   r   r   r#   r#   r$   r   r  r   zWriterThread._debouncer   r;   r   r   r#   r#   r   r$   rO   B  s:   
 
	




rO   c                   @   sF   e Zd ZU dZee ed< dddee ddfdd	Zde	fd
dZ
dS )rQ   z;Class to help watch a process id to detect when it is dead.check_process_lastr   r   r   r   Nc                 C   s   || _ || _d | _|j| _d S r   )r   ri   r   x_internal_check_processcheck_process_interval)r{   r   r   r#   r#   r$   r   {  s   zProcessCheck.__init__c                 C   sd   | j r| jsdS t }| jr|| j| j  k rdS || _t| j}|s0td| j d dS dS )NFz%Internal process exiting, parent pid z disappearedT)r   ri   r=   r   psutil
pid_existsr!   rY   )r{   time_nowexistsr#   r#   r$   rU     s   zProcessCheck.is_dead)r}   r~   r   r   r   r   r   intr   r   rU   r#   r#   r#   r$   rQ   v  s
   
 rQ   )NNr   ),r   rA   r/   rF   rL   r^   rJ   r=   r`   r   typingr   r   r   r   r   r.   interface.interface_queuer    r
   r   r   r   r   r   r   wandb.proto.wandb_internal_pb2r   r   r   settings_staticr   r0   r}   r!   r   rt   strrD   rP   rM   rO   rQ   r#   r#   r#   r$   <module>   sh   

 	
%714