o
    پi                     @  s   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
mZ d dlZd dlmZ eeZG dd dZG dd	 d	eZG d
d deZG dd dZdS )    )annotationsN)contextmanager)CallableOptional)pyspy_dump_schedulersc                   @  s4   e Zd Ze		ddddZdd Zedd ZdS )WatchdogFr   
debug_namestrwatchdog_timeoutOptional[float]softbooltest_stuck_timefloatreturnc                 C  s.   |d u r|dksJ dt  S t| |||dS )Nr   z=stuck tester can be enabled only if soft watchdog is enabled.r   r
   r   r   )_WatchdogNoop_WatchdogRealr    r   M/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/utils/watchdog.pycreate   s   
zWatchdog.createc                 C  s   d S Nr   selfr   r   r   feed&   s   zWatchdog.feedc                 c  s    d V  d S r   r   r   r   r   r   disable)   s   
zWatchdog.disableNFr   )
r   r	   r
   r   r   r   r   r   r   r   )__name__
__module____qualname__staticmethodr   r   r   r   r   r   r   r   r      s    r   c                   @  s0   e Zd Z		ddd
dZdd Zedd ZdS )r   Fr   r   r	   r
   r   r   r   r   c                   s   d _ d _| _d _t| fdd fdd||d _td jj d	  jdkr>td jj d
|d d S d S )Nr   TFc                         j S r   )_counterr   r   r   r   <lambda><       z(_WatchdogReal.__init__.<locals>.<lambda>c                     r!   r   _activer   r   r   r   r#   =   r$   )r   get_counter	is_activer
   r   	Watchdog z initialized.z& is configured to use test_stuck_time=.)	r"   r&   _test_stuck_time_test_stuck_triggeredWatchdogRaw_rawloggerinfor   )r   r   r
   r   r   r   r   r   __init__/   s"   


z_WatchdogReal.__init__c                 C  sp   | j dkr/| js/d| _td| jj d| j  d t| j  td| jj d| j  d |  jd7  _d S )Nr   Tr)   z start deliberately stuck for sz end deliberately stuck for    )	r+   r,   r/   r0   r.   r   timesleepr"   r   r   r   r   r   G   s   z_WatchdogReal.feedc                 c  sB    | j sJ d| _ zd V  W | j rJ d| _ d S | j rJ d| _ w )NFTr%   r   r   r   r   r   V   s   



z_WatchdogReal.disableNr   )r   r	   r
   r   r   r   r   r   )r   r   r   r1   r   r   r   r   r   r   r   r   .   s    r   c                   @  s   e Zd ZdS )r   N)r   r   r   r   r   r   r   r   a   s    r   c                   @  s,   e Zd Z		ddddZdd Zdd ZdS )r-   FNr   r	   r'   Callable[[], int]r(   Callable[[], bool]r
   r   r   r   	dump_infoOptional[Callable[[], str]]c                 C  sN   || _ || _|| _|| _|| _|| _t  | _	t
j| jdd}|  d S )NT)targetdaemon)r   r'   r(   r
   r   r8   psutilProcessparentparent_process	threadingThread_watchdog_threadstart)r   r   r'   r(   r
   r   r8   tr   r   r   r1   f   s   	zWatchdogRaw.__init__c              
   C  sN   z	 |    q ty& } ztj| j d| dd W Y d }~d S d }~ww )NTz watchdog thread crashed: )exc_info)_watchdog_once	Exceptionr/   errorr   )r   er   r   r   rB   z   s   zWatchdogRaw._watchdog_threadc                 C  s   d}t  }	 t  }|  r$|  }||kr ||| j krnn|}|}t | jd  q| jd urC|   }rCt| j	 d|  t
  t| j	 d| jd| jd ttjdd ttjdd | jsvt d	 | jtj d S d S )
Nr   T   z debug info:
z) watchdog timeout (self.watchdog_timeout=z, self.soft=))fileflush   )r4   perf_counterr(   r'   r
   r5   r8   r/   rH   r   r   r   printsysstderrstdoutr?   send_signalsignalSIGQUIT)r   watchdog_last_counterwatchdog_last_timecurrentcurrent_counterinfo_msgr   r   r   rF      s<   
zWatchdogRaw._watchdog_once)FN)r   r	   r'   r6   r(   r7   r
   r   r   r   r8   r9   )r   r   r   r1   rB   rF   r   r   r   r   r-   e   s    	r-   )
__future__r   loggingrU   rQ   r@   r4   
contextlibr   typingr   r   r<   sglang.srt.utils.commonr   	getLoggerr   r/   r   r   r   r-   r   r   r   r   <module>   s    
3