o
    پio~                  	   @   s&  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m
Z
mZ ddlmZmZ ddlmZmZmZmZ ddlZddlZddlZddlmZ ddlmZmZ eeZdd	 ZeG d
d dZ eG dd dZ!G dd de
Z"G dd de"Z#G dd de"Z$	d^de dej%dee de!fddZ&	d^de dej%dee de!fddZ'dee! de(fddZ)d_d!e*d"e+ddfd#d$Z,d%d& Z-ed'krej.d(d)Z/e/j0d*e*dd+d, e/j0d-e*dd.d, e/j0d/e*d0d1d, e/j0d2e+d3d4d, e/j0d5e*d6d7d, e/j0d8e*d9d9d:gd;d< e/j0d=e*g d>dd?d@ e/j0dAe*ddBd, e/j0dCe+dDdEd, e/j0dFe+dGdHd, e/j0dIe(e(dJdKd, e/j0dLe+ddMd, e/j0dNe+ddOd, e/j0dPe+ddQd, e/j0dRe+ddSd, e/j0dTe*ddUd, e/j0dVdWdXdY e/j0dZe*d[g d\d]d< e/1 Z2ee2 e3e-e2 dS dS )`a  
Benchmark online serving for diffusion models (Image/Video Generation).


Usage:
    # launch a server and benchmark on it

    # T2V or T2I or any other multimodal generation model
    sglang serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --num-gpus 1 --port 1231

    # benchmark it and make sure the port is the same as the server's port
    python3 -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --num-prompts 20 --port 1231
    N)ABCabstractmethod)	dataclassfield)AnyDictListOptional)tqdm)configure_loggerinit_loggerc                 C   s   t j| ott | S N)ospathisdirboollistdir)r    r   b/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/benchmarks/bench_serving.pyis_dir_not_empty(      r   c                   @   s   e Zd ZU eed< eed< eed< dZee ed< dZee ed< dZ	ee ed< dZ
ee ed< eed	Zeeef ed
< dZeee  ed< edd d	Zeed< dS )RequestFuncInputpromptapi_urlmodelNwidthheight
num_framesfpsdefault_factory
extra_bodyimage_pathsc                   C   s   t t S r   )struuiduuid4r   r   r   r   <lambda>7   s    zRequestFuncInput.<lambda>
request_id)__name__
__module____qualname__r#   __annotations__r   r	   intr   r   r   r   dictr!   r   r   r"   r   r'   r   r   r   r   r   ,   s   
 r   c                   @   sd   e Zd ZU dZeed< dZeed< dZe	ed< dZ
eed< eedZee	ef ed	< dZeed
< dS )RequestFuncOutputFsuccessg        latency error
start_timer   response_bodypeak_memory_mbN)r(   r)   r*   r/   r   r+   r0   floatr2   r#   r3   r   r-   r4   r   r   r5   r   r   r   r   r.   :   s   
 r.   c                   @   s\   e Zd ZdedefddZedefddZededefd	d
Z	ede
e fddZdS )BaseDatasetr   r   c                 C   s   || _ || _|| _d S r   )argsr   r   selfr8   r   r   r   r   r   __init__E   s   
zBaseDataset.__init__returnc                 C      d S r   r   r:   r   r   r   __len__J      zBaseDataset.__len__idxc                 C   r=   r   r   r:   rA   r   r   r   __getitem__N   r@   zBaseDataset.__getitem__c                 C   r=   r   r   r>   r   r   r   get_requestsR   r@   zBaseDataset.get_requestsN)r(   r)   r*   r#   r;   r   r,   r?   r   rC   r   rD   r   r   r   r   r7   D   s    r7   c                       sV  e Zd ZdZdZdZdedef fddZdee	ee
f  fd	d
ZdededdfddZdee	ee
f  fddZdefddZdedee	ee
f  fddZdedee	ee
f  fddZdee	ee
f  fddZdee	ee
f  fddZdee	ee
f  dee	ee
f  fdd Zdefd!d"Zd#edefd$d%Zdee fd&d'Z  ZS )(VBenchDatasetzC
    Dataset loader for VBench prompts.
    Supports t2v, i2v.
    znhttps://raw.githubusercontent.com/Vchitect/VBench/master/prompts/prompts_per_dimension/subject_consistency.txtzZhttps://raw.githubusercontent.com/Vchitect/VBench/master/vbench2_beta_i2v/download_data.shr   r   c                    s8   t  ||| tjtjddd| _|  | _d S )N~z.cachesglang)	superr;   r   r   join
expanduser	cache_dir
_load_dataitemsr9   	__class__r   r   r;   `   s   zVBenchDataset.__init__r<   c                 C   s8   | j jdv r
|  S | j jdv r|  S tdt j )N)text-to-videotext-to-imagevideo-to-video)image-to-videoimage-to-imagez,Illegal task name is found in VBenchDataset )r8   	task_name_load_t2v_prompts_load_i2v_data
ValueErrorr>   r   r   r   rL   e   s   
zVBenchDataset._load_dataurl	dest_pathNc                 C   sd   t jt j|dd t|}|  t|d}||j	 W d   dS 1 s+w   Y  dS )z-Download a file from URL to destination path.Texist_okwN)
r   makedirsr   dirnamerequestsgetraise_for_statusopenwritetext)r:   rY   rZ   respfr   r   r   _download_fileo   s   
"zVBenchDataset._download_filec              
   C   s   | j j}|sHtj| jd}tj|sHtd| d z	| 	| j
| W n! tyG } ztd|  ddigd W  Y d }~S d }~ww g }t|d}|D ]}| }|ra|d|i qRW d    n1 slw   Y  | |S )	Nzvbench_subject_consistency.txtz"Downloading VBench T2V prompts to ...z#Failed to download VBench prompts: r   zA cat sitting on a bench2   r)r8   dataset_pathr   r   rI   rK   existsloggerinforh   T2V_PROMPT_URL	Exceptionrc   stripappend_resize_data)r:   r   epromptsrg   liner   r   r   rV   w   s,   
zVBenchDataset._load_t2v_promptsc              
   C   s  t j| jdd}t j|dd}t j|dd}t j|dd}t j|r1t|r1t|r1|S td| d zht j| jd}t j|d	}| | j	| t 
|d
 td ddl}|jd|g|ddd}|jdkrxtd|j td|j}	|	rtt|	}	ddd |	D }
td|
 dtd|  W n% ty } ztd|  td td W Y d}~dS d}~ww t j|r|S dS )zBAuto-download VBench I2V dataset and return the dataset directory.
vbench_i2vvbench2_beta_i2vdatai2v-bench-info.jsoncroporiginz"Downloading VBench I2V dataset to ri   zdownload_data.shi  z5Executing download_data.sh (this may take a while)...r   NbashT)cwdcapture_outputre   zDownload script failed: z(\S+): command not found, c                 s   s    | ]	}d | d V  qdS )'Nr   ).0cmdr   r   r   	<genexpr>   s    z;VBenchDataset._auto_download_i2v_dataset.<locals>.<genexpr>zIDownload script failed because the following commands are not installed: zN.
Please install them (e.g., on Ubuntu: `sudo apt install ...`) and try again.z.Successfully downloaded VBench I2V dataset to z'Failed to download VBench I2V dataset: z3Please manually download following instructions at:zKhttps://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v#22-download)r   r   rI   rK   rm   r   rn   ro   rh   I2V_DOWNLOAD_SCRIPT_URLchmod
subprocessrun
returncodeRuntimeErrorstderrrefindalllistsetrq   )r:   vbench_i2v_dirinfo_json_pathcrop_dir
origin_dir
cache_rootscript_pathr   resultmissing_packagespackage_listru   r   r   r   _auto_download_i2v_dataset   s^   




z(VBenchDataset._auto_download_i2v_dataset	json_pathc           	      C   s   t |d}t|}W d   n1 sw   Y  tjtj|}tj|dd}g }|D ](}tj||dd}tj|rP|	|dd|d q0t
d	|  q0t
d
t| d |S )z.Load I2V data from i2v-bench-info.json format.rk   Nrz   r}   	file_namer1   captionr   
image_pathzImage not found: zLoaded z$ I2V samples from VBench I2V dataset)rc   jsonloadr   r   r_   rI   ra   rm   rs   rn   warningro   len)	r:   r   rg   rM   base_dirr   rz   itemimg_pathr   r   r   _load_from_i2v_json   s   
z!VBenchDataset._load_from_i2v_jsonr   c              
   C   s   g d}g }|D ]H}| ttj|| | ttj||  tj|dd}tj|rP| ttj|| | ttj||  qdd |D S )zScan directory for image files.)z*.jpgz*.jpegz*.pngz*.webprz   r}   c                 S   s*   g | ]}t jt j|d  |dqS )r   r   )r   r   splitextbasename)r   rg   r   r   r   
<listcomp>   s    z<VBenchDataset._scan_directory_for_images.<locals>.<listcomp>)extendglobr   r   rI   upperrm   )r:   r   extsfilesextr   r   r   r   _scan_directory_for_images   s   z(VBenchDataset._scan_directory_for_imagesc                 C   s   t d tj| jd}tj|sIz%ddlm} tj	| jdd |j
ddd	d
}|| t d|  W n tyH   t d g  Y S w d|dgd S )z>Create dummy data with a placeholder image in cache directory.z,No I2V data found. Using dummy placeholders.zdummy_image.jpgr   )ImageTr[   RGB)d   r   red)colorzCreated dummy image at z-PIL not installed, cannot create dummy image.zA moving catr   
   )rn   ro   r   r   rI   rK   rm   PILr   r^   newsaveImportError)r:   dummy_imager   imgr   r   r   _create_dummy_data   s   


z VBenchDataset._create_dummy_datac                 C   s   | j j}|s|  }|s| |  S tj|dd|dr!|ndg}|D ]3}|rYtj	|rYz| | 
|W   S  tyX } ztd| d|  W Y d}~q&d}~ww q&tj|rl| |}|rl| |S | |  S )z<Load I2V data from VBench I2V dataset or user-provided path.rz   r{   z.jsonNzFailed to load : )r8   rl   r   rt   r   r   r   rI   endswithrm   r   rq   rn   ro   r   r   )r:   r   info_json_candidatesr   ru   rz   r   r   r   rW      s,   "

zVBenchDataset._load_i2v_datarz   c                 C   sH   | j js|S t|| j jk r| j jt| d }|| }|d| j j S )z!Resize data to match num_prompts.   N)r8   num_promptsr   )r:   rz   factorr   r   r   rt     s   zVBenchDataset._resize_datac                 C   s
   t | jS r   )r   rM   r>   r   r   r   r?   )  s   
zVBenchDataset.__len__rA   c              
   C   sT   | j | }d|v r|d gnd }t|dd| j| j| jj| jj| jj| jj	|dS )Nr   r   r1   )r   r   r   r   r   r   r   r"   )
rM   r   ra   r   r   r8   r   r   r   r   )r:   rA   r   r"   r   r   r   rC   ,  s   

zVBenchDataset.__getitem__c                        fddt t D S )Nc                       g | ]} | qS r   r   r   ir>   r   r   r   <      z.VBenchDataset.get_requests.<locals>.<listcomp>ranger   r>   r   r>   r   rD   ;  r   zVBenchDataset.get_requests)r(   r)   r*   __doc__rp   r   r#   r;   r   r   r   rL   rh   rV   r   r   r   r   rW   rt   r,   r?   r   rC   rD   __classcell__r   r   rN   r   rE   W   s"    
6*rE   c                   @   sP   e Zd ZdedefddZdefddZdedefd	d
Zde	e fddZ
dS )RandomDatasetr   r   c                 C   s"   || _ || _|| _|jpd| _d S )Nr   )r8   r   r   r   r9   r   r   r   r;   @  s   zRandomDataset.__init__r<   c                 C   s   | j S r   )r   r>   r   r   r   r?   F  s   zRandomDataset.__len__rA   c              	   C   s2   t d| d| j| j| jj| jj| jj| jjdS )NzRandom prompt z" for benchmarking diffusion models)r   r   r   r   r   r   r   )r   r   r   r8   r   r   r   r   rB   r   r   r   rC   I  s   
zRandomDataset.__getitem__c                    r   )Nc                    r   r   r   r   r>   r   r   r   U  r   z.RandomDataset.get_requests.<locals>.<listcomp>r   r>   r   r>   r   rD   T  r   zRandomDataset.get_requestsN)r(   r)   r*   r#   r;   r,   r?   r   rC   r   rD   r   r   r   r   r   ?  s
    r   inputsessionpbarr<   c              
      s,  t  }t |_| jrt| jdkrt }|d| j	 |d| j
 |dd | jr>| jr>|d| j d| j  | j D ]\}}||t| qCt| jD ].\}}tj|rp|jdt|d	tj|d
d qUd| |_d|_|r|d |  S zR|j| j|d4 I d H 9}	|	jdkr|	 I d H }
|
|_d|_d|
v r|
d |_nd|	j d|	 I d H  |_d|_W d   I d H  n1 I d H sw   Y  W n t y } zt||_d|_W Y d }~nd }~ww | j	| j
ddd}| jr| jr| j d| j |d< || j zU|j| j|d4 I d H ;}	|	jdkr?|	 I d H }
|
|_d|_d|
v r>|
d |_nd|	j d|	 I d H  |_d|_W d   I d H  n1 I d H sbw   Y  W n t y } zt||_d|_W Y d }~nd }~ww t |j |_!|r|d |S )Nr   r   r   response_formatb64_jsonsizeximagerbapplication/octet-streamfilenamecontent_typeImage file not found: Fr   rz      Tr5   zHTTP r   )r   r   nr   r   )"r.   timeperf_counterr3   r"   r   aiohttpFormData	add_fieldr   r   r   r   r!   rM   r#   	enumerater   r   rm   rc   r   r2   r/   updatepostr   statusr   r4   r5   re   rq   r0   )r   r   r   outputrz   keyvaluerA   r   response	resp_jsonru   payloadr   r   r   async_request_image_sglangX  s   




(



*


r   c              
      sl  t  }t |_d }| jrt| jdkrt }|d| j	 |d| j
 | jr<| jr<|d| j d| j  | jrI|dt| j | jrU|dt| j | jra|dt| j | jd }tj|r}|jd	t|d
tj|dd nd| |_d|_|r|d |S z\|j| j|d4 I d H C}|jdkr| I d H }|d}n&d|j d| I d H  |_d|_|r|d |W  d   I d H  W S W d   I d H  n1 I d H sw   Y  W n t y }	 zdt|	 |_d|_|r|d |W  Y d }	~	S d }	~	ww | j	| j
d}
| jr.| jr.| j d| j |
d< | jr7| j|
d< | jr@| j|
d< |
| j z_|j| j|
d4 I d H E}|jdkrg| I d H }|d}n'd|j d| I d H  |_d|_|r|d |W  d   I d H  W S W d   I d H  n1 I d H sw   Y  W n( t y }	 zdt|	 |_d|_|r|d |W  Y d }	~	S d }	~	ww |sd|_d|_|r|d |S | j d| }	 z||4 I d H }|jdkrN| I d H }|d}|dkr(d|_||_!d|v r|d |_"	 W d   I d H  W n||dkrEd|_d|d  |_	 W d   I d H  W n_t#$d!I d H  nd|_d"|j d| I d H  |_	 W d   I d H  W n8W d   I d H  n1 I d H s}w   Y  W n t y }	 zd|_d#t|	 |_W Y d }	~	nd }	~	ww qt |j |_%|r|d |S )$Nr   r   r   r   r   r!   r   r   input_referencer   r   r   r   Fr   r   r   idzSubmit failed HTTP r   zSubmit exception: )r   r   r   zNo job_id returned/Tr   	completedr5   failedzJob failed: r2         ?zPoll failed HTTP zPoll exception: )&r.   r   r   r3   r"   r   r   r   r   r   r   r   r   r!   r   dumpsr   r#   r   r   r   rm   rc   r   r2   r/   r   r   r   r   ra   re   rq   r4   r5   asynciosleepr0   )r   r   r   r   job_idrz   r   r   r   ru   r   	check_urlstatus_datar   r   r   r   async_request_video_sglang  s  





(




*







*
r   outputstotal_durationc                 C   s   dd | D }dd | D }t |}dd |D }dd |D }||t ||dkr-|| nd|r5t|nd|r=t|nd|rFt|dnd|rOt|dnd|rVt|nd|r^t|nd|rft|ndd	}|S )
Nc                 S   s   g | ]}|j r|qS r   r/   r   or   r   r   r   L      z%calculate_metrics.<locals>.<listcomp>c                 S   s   g | ]}|j s|qS r   r   r  r   r   r   r   M  r  c                 S   s   g | ]}|j qS r   )r0   r  r   r   r   r   P  s    c                 S   s   g | ]
}|j d kr|j qS )r   )r5   r  r   r   r   r   Q  s    r   c   rj   )durationcompleted_requestsfailed_requeststhroughput_qpslatency_meanlatency_medianlatency_p99latency_p50peak_memory_mb_maxpeak_memory_mb_meanpeak_memory_mb_median)r   npmeanmedian
percentilemax)r   r   success_outputserror_outputsnum_success	latenciespeak_memoriesmetricsr   r   r   calculate_metricsK  s$   r    base_urltimeoutc                 C   s   t d|  d t }	 ztj|  ddd}|jdkr&t d W d S W n tjjy2   Y nw t | |krFtd	|  d
| dt	d q)NzWaiting for service at ri   Tz/healthr   r  r   zService is ready.zService at z did not start within z	 seconds.)
rn   ro   r   r`   ra   status_code
exceptionsRequestExceptionTimeoutErrorr   )r  r  r3   rf   r   r   r   wait_for_serviced  s$   


r$  c              
      s^  ddl m} | jd u rd| j d| j | _t| j z,tj| j ddd}|jdkrG|	 }d	|v rG|d	 rG|d	 | _
td
| j
  W n tyf } ztd| d| j
  W Y d }~nd }~ww d}| jrx| j}td|  nWtj| j
rtj| j
d}tj|rt|d}t	|}	W d    n1 sw   Y  |	dd}td|  nd}td|  n|| j
j}td|  ||vrtd| dd| |dv r| j d}
t n|dkr| j dn| j d}
t t| d| | jdkrt| |
| j
}n| jd kr!t| |
| j
}ntd!| j td" | }td#t| d$| j d% | jd urNt !| jnd  fd&d'}t"t|| j#d(}t$% 4 I d H L}t&' }g }|D ]*}| j(t)d)krt*j+,d*| j( }t -|I d H  t .||||}|/| qqt j0| I d H }t&' | }W d   I d H  n1 I d H sw   Y  |1  t2||}t3d+j4d,d-d.d/ t3d04d1| t3d04d2| j
 t3d04d3| j t3d4  t3d54d6|d7  t3d04d8t5| j( t3d04d9| jrt5| jnd: t3d;4d<|d= t| t3d4  t3d54d>|d?  t3d@4dA|dB  t3d@4dC|dD  t3d@4dE|dF  |dG dkr|t3d4  t3d54dH|dG  t3d54dI|dJ  t3d54dK|dL  t3dM | j6rt| j6dN}t	j7||dOdP W d    n	1 sw   Y  t3dQ| j6  d S d S )RNr   )
model_infozhttp://:z/v1/model_info   r  r   
model_pathz Updated model name from server: zFailed to fetch model info: z. Using default: )rP   rS   rR   rQ   rT   zUsing task from --task: zconfig.jsonrk   pipeline_tagrQ   z&Inferred task from local config.json: z*No config.json found, defaulting task to: z-Inferred task from HuggingFace pipeline_tag: zTask 'zK' is not a valid multimodal generation task. Use --task to specify one of: r   )rP   rS   rR   z
/v1/videosrT   z/v1/images/editsz/v1/images/generationsrU   vbenchrandomzUnknown dataset: zLoading requests...z	Prepared z requests from z	 dataset.c              	      sf   r*4 I d H   | ||I d H W  d   I d H  S 1 I d H s#w   Y  d S  | ||I d H S r   r   )reqr   r   request_func	semaphorer   r   limited_request_func  s   0z'benchmark.<locals>.limited_request_func)totaldisableinfr   z
{s:{c}^{n}}z Serving Benchmark Result <   =)sr   cz{:<40} {:<15}zTask:zModel:zDataset:z2--------------------------------------------------z{:<40} {:<15.2f}zBenchmark duration (s):r  zRequest rate:zMax request concurrency:znot setz{:<40} {}/{:<15}zSuccessful requests:r  zRequest throughput (req/s):r  z{:<40} {:<15.4f}zLatency Mean (s):r	  zLatency Median (s):r
  zLatency P99 (s):r  r  zPeak Memory Max (MB):zPeak Memory Mean (MB):r  zPeak Memory Median (MB):r  z<============================================================r]      )indentzMetrics saved to )8huggingface_hubr%  r  hostportr$  r`   ra   r   r   r   rn   ro   rq   taskr   r   rm   rI   rc   r   r)  rX   r   r   setattrdatasetrE   r   rD   r   max_concurrencyr   	Semaphorer
   disable_tqdmr   ClientSessionr   r   request_rater6   r  r+  exponentialr   create_taskrs   gathercloser  printformatr#   output_filedump)r8   r%  rf   ro   ru   valid_tasksrU   config_pathrg   configr   r?  requests_listr0  r   r   r3   tasksr,  intervalr=  r   r   r  r   r-  r   	benchmarky  s   



$	

*



rS  __main__z'Benchmark serving for diffusion models.)descriptionz	--backendz]DEPRECATED: --task is deprecated and will be ignored. The task will be inferred from --model.)typedefaulthelpz
--base-urlzKBase URL of the server (e.g., http://localhost:30000). Overrides host/port.z--host	localhostzServer host.z--porti0u  zServer port.z--modelrW  zModel name.z	--datasetr*  r+  zDataset to use.)rV  rW  choicesrX  z--task)rP   rS   rQ   rT   rR   z|The task will be inferred from huggingface pipeline_tag. When huggingface pipeline_tag is not provided, --task will be used.)rV  rZ  rW  rX  z--dataset-pathz&Path to local dataset file (optional).z--num-promptsr   zNumber of prompts to benchmark.z--max-concurrencyr   a  Maximum number of concurrent requests, default to `1`. This can be used to help simulate an environment where a higher level component is enforcing a maximum number of concurrent requests. While the --request-rate argument controls the rate at which requests are initiated, this argument will control how many are actually allowed to execute at a time. This means that when used in combination, the actual request rate may be lower than specified with --request-rate, if the server is not processing requests fast enough to keep up.z--request-rater3  zNumber of requests per second. If this is inf, then all the requests are sent at time 0. Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.z--widthzImage/Video width.z--heightzImage/Video height.z--num-frameszNumber of frames (for video).z--fpszFPS (for video).z--output-filezOutput JSON file for metrics.z--disable-tqdm
store_truezDisable progress bar.)actionrX  z--log-levelINFO)DEBUGr]  WARNINGERRORz
Log level.r   )r  )4r   argparser   r   r   r   r   r   r$   abcr   r   dataclassesr   r   typingr   r   r   r	   r   numpyr  r`   tqdm.asyncior
   1sglang.multimodal_gen.runtime.utils.logging_utilsr   r   r(   rn   r   r   r.   r7   rE   r   rC  r   r   r6   r  r#   r,   r$  rS  ArgumentParserparseradd_argument
parse_argsr8   r   r   r   r   r   <module>   s   	 i
]
  
/