o
    ۷i                     @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlmZmZmZmZ d dlZd dlZd dlmZ d dlmZmZ d dlmZ d d	lmZ d d
l m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4 erd dl5m4Z4 d dl6m7Z7 G dd dZ8G dd dZ9G dd dZ:G dd dZ	d d!ddZ;dS )"    )annotationsN)defaultdict)Queue)TYPE_CHECKINGAnyLiteralcast)run_sync)route_utilsroutes)PredictBodyInternal)Error)TrackedIterable)	EstimationMessageEventMessage
LogMessageProcessCompletedMessageProcessGeneratingMessageProcessStartsMessageProgressMessageProgressUnitServerMessage)LRUCacheerror_payloadrun_coro_in_backgroundsafe_aclose_iteratorsafe_get_lockset_task_name   )BlockFunction)Blocksc                   @  s.   e Zd Zddd	Zed
d Zedd ZdS )Eventsession_hash
str | Nonefnr   requestfastapi.Requestusernamec                 C  sd   t  j| _|p
| j| _|| _|| _|| _|j| _d | _	d | _
d| _d| _d| _d| _t | _d S )NFTr   )uuiduuid4hex_idr"   r$   r%   r'   concurrency_iddataprogressprogress_pendingaliven_callsrun_timeasyncior!   signal)selfr"   r$   r%   r'    r6   E/home/ubuntu/vllm_env/lib/python3.10/site-packages/gradio/queueing.py__init__5   s   zEvent.__init__c                 C  s   | j jdkS )Nstream)r$   
connectionr5   r6   r6   r7   	streamingJ   s   zEvent.streamingc                 C  s,   | j std| jjd u rdS | j| jjkS )Nz6Cannot access if_finished during a non-streaming eventF)r<   
ValueErrorr$   
time_limitr2   r;   r6   r6   r7   is_finishedN   s
   zEvent.is_finishedN)r"   r#   r$   r   r%   r&   r'   r#   )__name__
__module____qualname__r8   propertyr<   r?   r6   r6   r6   r7   r!   4   s    

r!   c                   @  s   e Zd ZdddZdS )	
EventQueuer,   strconcurrency_limit
int | Nonec                 C  s&   g | _ || _|| _d| _tt| _d S Nr   )queuer,   rF   current_concurrencyr   setstart_times_per_fn)r5   r,   rF   r6   r6   r7   r8   X   s   
zEventQueue.__init__N)r,   rE   rF   rG   )r@   rA   rB   r8   r6   r6   r6   r7   rD   W   s    rD   c                   @  s   e Zd Zdd ZdddZdS )	ProcessTimec                 C  s   d| _ d| _d| _d S rH   process_timecountavg_timer;   r6   r6   r7   r8   c   s   
zProcessTime.__init__timefloatc                 C  s.   |  j |7  _ |  jd7  _| j | j | _d S )Nr   rN   )r5   rR   r6   r6   r7   addh   s   zProcessTime.addN)rR   rS   )r@   rA   rB   r8   rT   r6   r6   r6   r7   rM   b   s    rM   c                   @  sB  e Zd Z	dtduddZedd Zdd Zdd ZdvddZdd Z	dwd d!Z
dxd#d$Zd%d& Zdyd.d/Zd0d1 Zdzd4d5Zd{d6d7Zd|d9d:Zd}d<d=Zd}d>d?Zd~dDdEZ	F	GdddOdPZdQdQdRddTdUZd}dVdWZ	QdddZd[Zdd]d^Zedd_d`ZeddbdcZeddedfZeddldmZddpdqZddrdsZdQS )r   not_setlive_updatesboolconcurrency_countintupdate_intervalsrS   max_sizerG   blocksr    default_concurrency_limitint | None | Literal['not_set']c                 C  s   t d| _i | _i | _t | _i | _d| _|| _|| _	g | _
t | _d | _tt| _|| _d| _d| _|| _|| _g | _| || _i | _di i| _d| _ttdd| _d S )	Ni  Fg?g?	functionsr    GRADIO_ANALYTICS_CACHE_FREQUENCY1)r   pending_messages_per_sessionpending_event_ids_sessionevent_ids_to_eventsr   pending_message_lockevent_queue_per_concurrency_idstoppedmax_thread_countrZ   active_jobsdelete_lock
server_appr   rM   process_time_per_fnrV   sleep_when_freeprogress_update_sleep_when_freer[   r\   _asyncio_tasks_resolve_concurrency_limitr]   event_analyticscached_event_analytics_summaryevent_count_at_last_cacherY   osgetenvANAYLTICS_CACHE_FREQUENCY)r5   rV   rX   rZ   r[   r\   r]   r6   r6   r7   r8   o   s<   



zQueue.__init__c              
   C  s   dd l }z-|dd |t|  jtjdjdd	 W  d    W S 1 s*w   Y  W d S  t	y] } z dt
|v rW|t|  jtjdjddW  Y d }~S |d }~ww )Nr   zfuture.no_silent_downcastingT)valueF)copyzNo such keys(s))pandasoption_context	DataFramelistvaluesfillnanpnaninfer_objects	ExceptionrE   )rq   pder6   r6   r7   _get_df   s*   (zQueue._get_dfc                 C  s   t || j | jkrj| |}t || _|d}di i}|D ]G\}}|d j}t|dk}t|dk}	||	 }
|
dkrB||
 nd }t|d jg d}||d |d	 |d
 d|j	d d|d |< q|| _
| j
S )Nfunctionr_   statussuccessfailedr   rO   )2   Z   c   r      )50th90th99th)success_rateprocess_time_percentilestotal_requests)lenrs   rv   r   groupbyr}   r   sum
percentileshaperr   )r5   rq   dfgroupedmetricsfn_namefn_dfr   r   failuretotalr   percentilesr6   r6   r7   compute_analytics_summary   s.   



	zQueue.compute_analytics_summaryc                 C  s:   d g| j  | _t| j t| j | jst| j d S d S N)rh   ri   r   start_processingstart_progress_updatesrV   notify_clientsr;   r6   r6   r7   start   s   

zQueue.startblock_fnr   c                 C  st   |j }|jdkr| j}n|j}|| jvrt||| j|< d S |d ur6| j| }|jd u s1||jk r8||_d S d S d S )Ndefault)r,   rF   r]   rf   rD   )r5   r   r,   rF   existing_event_queuer6   r6   r7   create_event_queue_for_fn   s   





zQueue.create_event_queue_for_fnc                 C  s
   d| _ d S )NT)rg   r;   r6   r6   r7   close      
zQueue.closeeventr!   event_messager   c                 C  s,   |j sd S |j|_| j|j }|| d S r   )r0   r+   event_idrb   r"   
put_nowait)r5   r   r   messagesr6   r6   r7   send_message   s
   zQueue.send_messagereturnc                 C  s8   |dkr|S t jd }r| dkrdS t|S dS )a  
        Handles the logic of resolving the default_concurrency_limit as this can be specified via a combination
        of the `default_concurrency_limit` parameter of the `Blocks.queue()` or the `GRADIO_DEFAULT_CONCURRENCY_LIMIT`
        environment variable. The parameter in `Blocks.queue()` takes precedence over the environment variable.
        Parameters:
            default_concurrency_limit: The default concurrency limit, as specified by a user in `Blocks.queu()`.
        rU    GRADIO_DEFAULT_CONCURRENCY_LIMITnoneNr   )rt   environgetlowerrY   )r5   r]   default_concurrency_limit_envr6   r6   r7   rp      s   
z Queue._resolve_concurrency_limitc                 C  s&   d}| j  D ]	}|t|j7 }q|S rH   )rf   r}   r   rI   )r5   	total_lenevent_queuer6   r6   r7   __len__	  s   zQueue.__len__bodyr   r%   r&   r'   r#   etuple[bool, str | list[dict[str, Any]], Literal['success', 'error', 'queue_full', 'validator_error']]c              
     sb  |j d u rdS | jd ur#t| | jkr#dd| j dt|  ddfS |jr5| jj|j }|jj|j  }n| jj|j  }t	| jd |}| 
| |jd urtj|||d d}|jd us_J tj|jd}tj|j|| jjjd	}td)i d
|jddd|jdd dd d|jd|jd|jddd|jdg ddd|jd|jdd ddd|jd|jd|j}	t|j|	||}
z+tj| jj|||	|dI d H }|d }|d urt||\}}|du rd|d!fW S W n  t y } zt!t"| dt"|d"fW  Y d }~S d }~ww t|j|||}
||
_#|jd u r$|
j|_| j$4 I d H ' |j| j%vr:t& | j%|j< |j| j'vrHt( | j'|j< W d   I d H  n1 I d H sYw   Y  | j'|j )|
j* |
| j+|
j*< z| j,|
j- }W n t.y } zt.d#|d }~ww |j/0|
 t11 d$d |j2|jd%| j3|
j*< | 4|
j-t|j/d&  d'|
j*d(fS )*N)FzNo function index provided.errorFzQueue is full. Max size is z and size is .
queue_fullr   r$   r'   r%   r%   r%   
route_path	root_pathr$   api_namebatchr,   rF   inputsoutputs
preprocesspostprocessinputs_as_dicttargetsr+   max_batch_sizetracks_progressjsshow_progresshiddenshow_progress_oncancelscollects_event_dataappr   
gr_requestr$   r   r-   validator_errorr   zEvent not found in queue. If you are deploying this Gradio app with multiple replicas, please enable stickiness to ensure that all requests from the same user are routed to the same instance.queued)rR   r   rO   r   r"   r   Tr   r6   )5fn_indexr[   r   r"   r\   state_holderblocks_configfnsr
   get_fnr   	validatorcompile_gr_requestr%   get_api_call_pathget_root_urlr   r   r   r   r   r   r   r   r   r   r   r   r!   call_process_apir   process_validation_responser   printrE   r-   re   rb   ThreadQueuerc   rK   rT   r+   rd   rf   r,   KeyErrorrI   appendrR   r   rq   broadcast_estimations)r5   r   r%   r'   session_stater$   r   api_route_pathr   validator_fnr   responsevalidation_responseis_validvalidation_datar   r   r6   r6   r7   push  s  

	
	*z
Queue.pushc                 C  s   | j D ]}|  qg | _ d S r   )ro   cancel)r5   taskr6   r6   r7   _cancel_asyncio_tasks  s   


zQueue._cancel_asyncio_tasksr   
routes.Appc                 C  s
   || _ d S r   )rk   )r5   r   r6   r6   r7   set_server_app  r   zQueue.set_server_appc                 C  s$   d}| j D ]
}|d ur|d7 }q|S )Nr   r   )ri   )r5   rP   workerr6   r6   r7   get_active_worker_count  s   
zQueue.get_active_worker_count$tuple[list[Event], bool, str] | Nonec                   s   t | j }t| |D ]P}| j| }t|jr^|jd u s%|j|jk r^|jd   j	} g}|j
}|rL| fdd|jdd  D d |jd  7 }|D ]}|j| qN|||f  S qd S )Nr   c                   s   g | ]
}|j  j kr|qS r6   )r$   .0r   first_eventr6   r7   
<listcomp>  s
    z$Queue.get_events.<locals>.<listcomp>r   )r|   rf   keysrandomshuffler   rI   rF   rJ   r$   r   r   remove)r5   concurrency_idsr,   r   r   eventsr   r   r6   r   r7   
get_events  s,   




zQueue.get_eventsNonec           	   	     s  z| j st| dkrt| jI d H  qd | jvr$t| jI d H  q| j4 I d H  |  }W d   I d H  n1 I d H s@w   Y  |r|\}}}|| j| jd < | j	| }| j
d7  _
t }|j|d j | |D ]
}d| j|j d< qrt| j|||}t||d j|d jj|d j| | j| | jr| | n	t| jI d H  | j rW d| _ |   d S W d| _ |   d S d| _ |   w )Nr   r   
processingr   T)rg   r   r3   sleeprm   ri   rj   r  indexrf   rJ   rR   rL   r$   rT   rq   r+   r   process_eventsr   r"   ro   r   rV   r   r   )	r5   event_batchr  r   r,   r   
start_timer   process_event_taskr6   r6   r7   r     sX   

(




''
zQueue.start_processingc                   s   | j s?dd | jD }t|dkrt| jI dH  q|D ]}|jr0|jr0d|_| ||j qt| jI dH  | j rdS dS )ah  
        Because progress updates can be very frequent, we do not necessarily want to send a message per update.
        Rather, we check for progress updates at regular intervals, and send a message if there is a pending update.
        Consecutive progress updates between sends will overwrite each other so only the most recent update will be sent.
        c                 S  "   g | ]}|d ur|D ]}|q
qS r   r6   r   jobevtr6   r6   r7   r       " z0Queue.start_progress_updates.<locals>.<listcomp>r   NF)	rg   ri   r   r3   r  rn   r/   r.   r   )r5   r  r   r6   r6   r7   r     s   zQueue.start_progress_updatesr   rE   	iterableslist[TrackedIterable] | Nonec              
   C  s   |d u rd S | j D ]4}|d u rq	|D ]*}|j|kr<g }|D ]}t|j|j|j|j|jd}|| qt	|d|_d|_
qq	d S )N)r  lengthunitr.   desc)progress_dataT)ri   r+   r   r  r  r  r.   r  r   r   r/   )r5   r   r  r  r  r  iterableprogress_unitr6   r6   r7   set_progress  s,   

zQueue.set_progress
   Tlogtitlelevel%Literal['info', 'warning', 'success']durationfloat | Nonevisiblec           
      C  sF   dd | j D }|D ]}|j|kr t|||||d}	| ||	 q
d S )Nc                 S  r  r   r6   r  r6   r6   r7   r    r  z%Queue.log_message.<locals>.<listcomp>)r   r"  r$  r&  r!  )ri   r+   r   r   )
r5   r   r   r!  r"  r$  r&  r  r   log_messager6   r6   r7   r'    s   	
zQueue.log_messageN)r"   r   r"   c             	     s   | j D ]}|r|D ]}|j|ks|j|krd|_q
q| j4 I d H : g }| j D ]}|jD ]}|j|ks;|j|kr@|| q/q*|D ]}| j|j	 j
| qDW d   I d H  d S 1 I d H sbw   Y  d S )NF)ri   r"   r+   r0   rj   rf   r}   rI   r   r,   r  )r5   r"   r   job_setr  events_to_remover   r   r6   r6   r7   clean_events$  s*   


.zQueue.clean_eventsc                   sJ   | j s#t| jI dH  t| dkr| jD ]}| | q| j rdS dS )zQ
        Notify clients about events statuses in the queue periodically.
        Nr   )rg   r3   r  rZ   r   rf   r   )r5   r,   r6   r6   r7   r   9  s   
zQueue.notify_clientsr,   afterc              	     s  d}| j | }d}|j|jkrWg }|j D ])\}}|| jvr#d } n|jdkr.|jp,d n| j| j | fdd|D 7 }q|d urWt	|dkrWt
|}	t|	t  d}t|jD ]c\}
}|j| jv rm| j|j jnd }|d ur|d ur|d ur|| | nd }|d u s|
|kr| |t|
|t	|jd |jd u rd}q\|d ur|d ur||j }|jr|pd|j }||7 }q\d }q\d S )Nr   r9   c                   s   g | ]}|  qS r6   r6   )r   r  rO   r6   r7   r  T  s    z/Queue.broadcast_estimations.<locals>.<listcomp>)rankrank_eta
queue_size)rf   rJ   rF   rL   itemsrl   r:   r>   rQ   r   minmaxrR   	enumeraterI   r$   r   r   r<   )r5   r,   r+  wait_so_farr   time_till_available_workerexpected_end_timesr$   start_timestime_of_first_completionr-  r   process_time_for_fnr.  deltar6   r,  r7   r   C  sb   






zQueue.broadcast_estimationsr   c                 C  s   t t| dS )N)r/  )r   r   r;   r6   r6   r7   
get_status  s   zQueue.get_statusc                   s   | j  I d H  dS )Nr4   )r4   wait)r   r6   r6   r7   wait_for_event     zQueue.wait_for_eventtimeoutc                   s   t | I d H  dS )Nr?  )r3   r  )r?  r6   r6   r7   r?    r>  zQueue.timeoutLiteral['signal', 'timeout']c                   sj   t t| }t t|}t j||gt jdI d H \}}dd |D }| j  t	t
d |d S )Nreturn_whenc                 S     g | ]}|  qS r6   resultr   dr6   r6   r7   r        z3Queue.wait_for_event_or_timeout.<locals>.<listcomp>)r4   r?  r   )r3   create_taskr   r=  r?  r<  FIRST_COMPLETEDr4   clearr   r   )r   r?  t1t2done_r6   r6   r7   wait_for_event_or_timeout  s   
zQueue.wait_for_event_or_timeoutr  list[Event]timeoutslist[float]tuple[list[Event], list[Event]]c           
   	     s   g }t | |ddD ]\}}|tt|| q
tj|tjdI d H \}}dd |D }g }g }t || ddD ]\}	}|	dkrH|| q:|| q:||fS )NF)strictrA  c                 S  rC  r6   rD  rF  r6   r6   r7   r    rH  z(Queue.wait_for_batch.<locals>.<listcomp>r4   )zipr   r3   rI  r   rP  r<  ALL_COMPLETED)
r  rR  tasksr   r?  rN  rO  awake_eventsclosed_eventsrE  r6   r6   r7   wait_for_batch  s$   zQueue.wait_for_batchr   
begin_timec                   s	  g }|d j }d}zz|D ]}|jr.| |t|| jv r$| j| jnd d || q|sW W | j|d j }| j	d8  _	|j
| }	||	v rP|	| zd | j| j|< W n	 tyd   Y nw |D ].}| |jI d H  ||v r|rzdnd| j|j d< nd| j|j d< t| j| jI d H  qgd S |}|d j}
|
d u rtd	|d j}|s|d jnd |
_z|d j|
_W n	 ty   Y nw |rttd
d |D ddi|
_|d j|
_d|
_| j}|d u rtdtj|
||d d}|
jd usJ tj|
jd}tj |
j||j!d}d}z6t"# }tj$||
|||dI d H }t"# }|| }d }|D ]}| j%|| 7  _%|j&rC|j' |d< q.W nL ty } z?t(|t)rY|j*r]t+,  d }|}|D ]$}t-||. j/}| |t0||1dddd t| j| jI d H  qcW Y d }~nd }~ww |r-|1ddr-|}|}|r|1ddrt"# }|}|}|D ]+}| |t2|j&st3j4nt3j5||d u|j6sd n|j&rt7t8|j6| nd d qdd |D }|sWW W | j|d j }| j	d8  _	|j
| }	||	v r|	| zd | j| j|< W n
 ty    Y nw |D ]1}| |jI d H  ||v rA|r8dnd| j|j d< nd| j|j d< t| j| jI d H  q#d S z~t"# }|d j&rt9:|t7t;|j6pld| gt<| I d H \}}|D ]}| |t0|dd q}|sW nut7t=|d j}
|rttdd |D ddi|
_tj$||
|||dI d H }t"# }|D ]}| j%|| 7  _%|j&r|j' |d< qW n% ty } zt(|t)r|j*rt+,  d }|}W Y d }~nd }~ww |r|1dds|rd}|}nd}|p|}t-||. j/}|D ]}| |t0||d qn8|ret>?|}t@|D ]+\}}|rUd|v rUtt|1dddi| |d< |d u}| |t0||d q9t"" }|d ur|d j&sx|| n|}| j|d j  A| |D ]}|| j|j d< qW n! ty } zt(|t)r|j*rt+,  W Y d }~nd }~ww W | j|d j }| j	d8  _	|j
| }	||	v r|	| zd | j| j|< W n
 ty   Y nw |D ]1}| |jI d H  ||v r|rdnd| j|j d< nd| j|j d< t| j| jI d H  qd S | j|d j }| j	d8  _	|j
| }	||	v r?|	| zd | j| j|< W n
 tyT   Y nw |D ]1}| |jI d H  ||v ru|rldnd| j|j d< nd| j|j d< t| j| jI d H  qWw )Nr   F)etar   r   r   r   	cancelledzNo event datac                 S     g | ]	}|j r|j j qS r6   r-   r   r6   r6   r7   r    s    z(Queue.process_events.<locals>.<listcomp>rU  TServer app has not been set.r   r   r   r   is_generatingr!  r   )outputr!  r   )msgrc  r   r>   c                 S  s   g | ]}|j r|qS r6   )r0   r   r6   r6   r7   r  -  s       )rc  r   c                 S  r_  r6   r`  r   r6   r6   r7   r  I  s    r-   rO   )Br$   r0   r   r   rl   rQ   r   rf   r,   rJ   rL   r  ri   r  r=   reset_iteratorsr+   rq   r	   r   r-   r'   r   r%   r|   rV  batchedrk   r   r
   r   r   r   r   rR   	monotonicr   r2   r<   r?   
isinstancer   print_exception	traceback	print_excr   
get_blocks
show_errorr   r   r   r   process_generatingprocess_streamingr>   r   rY   r   r[  rS   r   r   rx   deepcopyr3  rT   )r5   r  r   r\  rY  r$   r   r   r   r7  r   r'   r   r   r   r   first_iterationr   r   enderrr   contentold_responseold_errrZ  closed_eventrc  r   end_timer$  r6   r6   r7   r    s>  


 A


 



]



	
J










zQueue.process_eventsc              	     s   | j }|d u rtd||jvrd S |j4 I d H , zt|j| I d H  W n	 ty0   Y nw |j|= |j| W d   I d H  d S 1 I d H sLw   Y  d S )Nra  )rk   r   	iteratorslockr   iterators_to_resetrT   )r5   r   r   r6   r6   r7   rf    s$   
zQueue.reset_iterators)rU   )rV   rW   rX   rY   rZ   rS   r[   rG   r\   r    r]   r^   )r   r   )r   r!   r   r   )r]   r^   r   rG   )r   r   r%   r&   r'   r#   r   r   )r   r   )r   rY   )r   r   )r   r	  )r   rE   r  r  )r  T)r   rE   r   rE   r!  rE   r"  r#  r$  r%  r&  rW   )r"   r#   r   r#   r   r	  r   )r,   rE   r+  rG   r   r	  )r   r   )r   r!   r   rE   )r?  rS   r   rE   )r   r!   r?  rS   r   r@  )r  rQ  rR  rS  r   rT  )r  rQ  r   rW   r\  rS   r   r	  )r   rE   ) r@   rA   rB   r8   staticmethodr   r   r   r   r   r   rp   r   r   r   r   r   r  r   r   r  r'  r*  r   r   r;  r=  r?  rP  r[  r  rf  r6   r6   r6   r7   r   n   sP    )









,


B
 pr   r   %list[dict[str, Any]] | dict[str, Any]r$   BlockFunction | Noner   !tuple[bool, list[dict[str, Any]]]c           	      C  s   g }g }|r|j rt|j }t|j }t| trWt| D ]6\}}t|trM|	dd dkrM|t
|k r:|| nd| }i |d|i}|| q|ddd qnt|trj|	dd d	u rj||  n|ddd td
d |D |fS )N__type__validate
parameter_parameter_nameT )r   messager   Fc                 s  s     | ]}| d ddu V  qdS )r   NT)r   )r   xr6   r6   r7   	<genexpr>  s    
z.process_validation_response.<locals>.<genexpr>)r$   inspect	signaturer|   
parametersr  ri  r3  dictr   r   r   all)	r   r$   r   param_namessigir-   
param_namedata_with_namer6   r6   r7   r     s4   

r   r   )r   r~  r$   r  r   r  )<
__future__r   r3   rx   r  rt   r  rR   rk  r(   collectionsr   rI   r   r   typingr   r   r   r   fastapinumpyr   anyio.to_threadr	   gradior
   r   gradio.data_classesr   gradio.exceptionsr   gradio.helpersr   gradio.server_messagesr   r   r   r   r   r   r   r   r   gradio.utilsr   r   r   r   r   r   block_functionr   gradio.block_functiongradio.blocksr    r!   rD   rM   r   r6   r6   r6   r7   <module>   sJ    , 	#      T