o
    `۷i                     @   s  d dl Z d dlZd dlmZ d dlmZmZ d dlZd dlmZ d dl	Z	d dl	m
Z
 d dlmZ d dlmZ dZd	Zd
d Ze	jG dd dZdedededefddZdedededededeeef fddZdd Zedkre	  e
  e  Ze  e e!e  dS dS )    N)pprint)DictUnion)Request)serve)run_throughput_benchmark)DeploymentHandle   d   c              	      sh   | j d|d4 I d H }| I d H }|dksJ |W d   I d H  d S 1 I d H s-w   Y  d S )Nzhttp://localhost:8000/dataok)gettext)sessionr   response r   b/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/serve/_private/benchmarks/microbenchmark.pyfetch   s
   .r   c                   @   s   e Zd Zdd Zdd ZdS )Clientc                 C   s   dS )Nr   r   selfr   r   r   ready   s   zClient.readyc              	      sb   t  4 I d H }t|D ]
}t||I d H  qW d   I d H  d S 1 I d H s*w   Y  d S N)aiohttpClientSessionranger   )r   numr   r   _r   r   r   
do_queries!   s   .zClient.do_queriesN)__name__
__module____qualname__r   r   r   r   r   r   r      s    r   intermediate_handlesnum_replicasmax_batch_sizemax_ongoing_requestsc                    sT   t jddG dd d}t j||dG  fddd}| r&|| S | S )Ni  )r&   c                   @   s(   e Zd ZdefddZdefddZdS )zbuild_app.<locals>.Upstreamhandlec                 S   s   || _ tdtj d S Nz	ray.serve)_handlelogging	getLoggersetLevelWARNING)r   r'   r   r   r   __init__/   s   z$build_app.<locals>.Upstream.__init__reqc                    s   | j | I d H I d H S r   )r)   remotebodyr   r/   r   r   r   __call__5      z$build_app.<locals>.Upstream.__call__N)r    r!   r"   r   r.   r   r3   r   r   r   r   Upstream-   s    r5   )r$   r&   c                       sB   e Zd Zdd Zej ddd Zdeeef f fddZ	d	S )
zbuild_app.<locals>.Downstreamc                 S   s   t dt j d S r(   )r*   r+   r,   r-   r   r   r   r   r.   =   s   z&build_app.<locals>.Downstream.__init__r%   c                    s   dgt | S )N   ok)len)r   reqsr   r   r   batchA   s   z#build_app.<locals>.Downstream.batchr/   c                    s    dkr|  |I d H S dS )N   r7   )r:   r2   r6   r   r   r3   E   s   z&build_app.<locals>.Downstream.__call__N)
r    r!   r"   r.   r   r:   r   bytesr   r3   r   r6   r   r   
Downstream8   s
    

r=   )r   
deploymentbind)r#   r$   r%   r&   r5   r=   r   r6   r   	build_app'   s   

r@   	data_sizereturnc                    s  i }d| d| d| d| d|  
}t d|  d| d| d	| d
| 
 t| |||}t| |dkr:d n|dkrCdd ntdt 4 I d H 2fdd}t|tdI d H \}	}
t d	d	||	|
 d| }|	||< W d   I d H  n1 I d H sw   Y  dd t
tD  tdd  D   fdd}t|tt  dI d H \}}||dt  d| < |S )Nzreplica:z/batch_size:z/concurrent_queries:z/data_size:z/intermediate_handle:zintermediate_handles=z,num_replicas=z,max_batch_size=z,max_ongoing_requests=z,data_size=smalllarges   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaai   z'data_size should be 'small' or 'large'.c                     s$   t tD ]
} t I d H  qd S r   )r   CALLS_PER_BATCHr   )r   )r   r   r   r   single_clientv   s   ztrial.<locals>.single_client)
multiplierz	{} {} +- {} requests/szsingle client {} dataznum_client:1/c                 S   s   g | ]}t  qS r   )r   r0   ).0r   r   r   r   
<listcomp>   s    ztrial.<locals>.<listcomp>c                 S   s   g | ]}|j  qS r   )r   r0   )rH   clientr   r   r   rI      s    c                      s   t fdd D  d S )Nc                    s   g | ]	}|j t qS r   )r   r0   rE   )rH   ar   r   r   rI      s    z/trial.<locals>.many_clients.<locals>.<listcomp>)rayr   r   )clientsr   r   r   many_clients   r4   ztrial.<locals>.many_clientsznum_client:/)printr@   r   run
ValueErrorr   r   r   rE   formatr   NUM_CLIENTSrL   r   r8   )r#   r$   r%   r&   rA   resultstrial_key_baseapprF   single_client_avg_tpssingle_client_std_tpskeyrN   multi_client_avg_tpsr   r   )rM   r   r   r   trialQ   sn   



(
r\   c                     sb   i } dD ]!}dD ]}dD ]\}}dD ]}|  t|||||I d H  qqq	qtd t|  | S )N)FT)r;   r	   ))r;   r;   )r;   '  )r]   r]   )rC   zResults from all conditions:)updater\   rP   r   )rU   r#   r$   r%   r&   rA   r   r   r   main   s*   r_   __main__)"asyncior*   r   typingr   r   r   starlette.requestsr   rL   r   $ray.serve._private.benchmarks.commonr   ray.serve.handler   rT   rE   r   r0   r   boolintr@   strfloatr\   r_   r    initstartnew_event_looploopset_event_looprun_until_completer   r   r   r   <module>   sZ   

*

F
