o
    iL                  #   @   s  U d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ ddlmZ zd dlmZ d dlZd dlZ W n e!yw   ed"dZedZedZY nw eG dd deZ#eG dd de#Z$eG dd de#Z%eG dd de#Z&eG dd de#Z'eG dd de#Z(eG dd de#Z)e$e%e'e)e&e(d Z*e+e,e-e# f e.d!< G d"d# d#e/e# Z0eG d$d% d%Z1d&e1iZ2e+e,e-e1 f e.d'< G d(d) d)e/e1 Z3d*ed+e/e+e,e4f  fd,d-Z5d.e/e+e,e4f  d+e/e+e,e4f  fd/d0Z6d1e+e,e4f d2e,fd3d4Z7d1e+e,e4f d5e/e, fd6d7Z8d8ed9e9e9e,e,f d:f d;e,fd<d=Z:G d>d? d?Z;d8ed@e9e9e9e,e,f d:f e/e+e,e4f  f dAe/e, dBe/e, dCe/e, dDe,dEe,dFe0dGe3dHe,dB dIe,dB dJe<d;e,dKe<dLe=dMe>f dNdOZ?dPdQdRdSdTdUed8edVe/e, dAe/e, dBe/e, dCe/e, dDe,dEe,dFe0dGe3dHe,dB dIe,dB dJe<d;e,dKe<dLe=dMe>f"dWdXZ@eG dYdZ dZZAd[eAfd\d]ZBd[e jCfd^d_ZDeEd`kre jFeAjGdaZHeAIeH eDeHJ  dS dS )b    N)ABCabstractmethod)ProcessPoolExecutor)	dataclass)partial)Path)TracebackType)ClassVar)Selfoverride)full_groupby)PlaceholderModule   )sanitize_filename
matplotlibpyplotpandasseabornc                   @   s>   e Zd ZU eed< eed< edefddZedd	d
ZdS )PlotFilterBasevartargetsc                 C   s\   t D ]}||v r!||\}}t | |||dd  S qtd| dtt  )N'"z"Invalid operator for plot filter ''. Valid operators are: )PLOT_FILTERSsplitremoveprefixstrip
ValueErrorsortedclsr   op_keykeyvalue r&   P/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/benchmarks/sweep/plot.py	parse_str#   s   zPlotFilterBase.parse_strdfpd.DataFramereturnc                 C   s   t )z#Applies this filter to a DataFrame.)NotImplementedErrorselfr)   r&   r&   r'   apply2   s   zPlotFilterBase.applyNr)   r*   r+   r*   )	__name__
__module____qualname__str__annotations__classmethodr(   r   r/   r&   r&   r&   r'   r      s   
 r   c                   @      e Zd ZedddZdS )PlotEqualTor)   r*   r+   c                 C   s:   zt | j}W n ty   | j}Y nw ||| j |k S Nfloatr   r   r   r.   r)   r   r&   r&   r'   r/   :      
zPlotEqualTo.applyNr0   r1   r2   r3   r   r/   r&   r&   r&   r'   r8   8       r8   c                   @   r7   )PlotNotEqualTor)   r*   r+   c                 C   s:   zt | j}W n ty   | j}Y nw ||| j |k S r9   r:   r<   r&   r&   r'   r/   F   r=   zPlotNotEqualTo.applyNr0   r>   r&   r&   r&   r'   r@   D   r?   r@   c                   @   r7   )PlotLessThanr)   r*   r+   c                 C   s   ||| j  t| jk  S r9   r   r;   r   r-   r&   r&   r'   r/   R      zPlotLessThan.applyNr0   r>   r&   r&   r&   r'   rA   P   r?   rA   c                   @   r7   )PlotLessThanOrEqualTor)   r*   r+   c                 C   s   ||| j  t| jk S r9   rB   r-   r&   r&   r'   r/   Y   rC   zPlotLessThanOrEqualTo.applyNr0   r>   r&   r&   r&   r'   rD   W   r?   rD   c                   @   r7   )PlotGreaterThanr)   r*   r+   c                 C   s   ||| j  t| jk S r9   rB   r-   r&   r&   r'   r/   `   rC   zPlotGreaterThan.applyNr0   r>   r&   r&   r&   r'   rE   ^   r?   rE   c                   @   r7   )PlotGreaterThanOrEqualTor)   r*   r+   c                 C   s   ||| j  t| jk S r9   rB   r-   r&   r&   r'   r/   g   rC   zPlotGreaterThanOrEqualTo.applyNr0   r>   r&   r&   r&   r'   rF   e   r?   rF   )z==z!=z<=z>=<>r   c                   @   (   e Zd ZedefddZd
ddZd	S )PlotFiltersr   c                 C   "   |s|  S | dd | dD S )Nc                 s       | ]}t |V  qd S r9   )r   r(   .0er&   r&   r'   	<genexpr>}       z(PlotFilters.parse_str.<locals>.<genexpr>,r   r"   r   r&   r&   r'   r(   x      zPlotFilters.parse_strr)   r*   r+   c                 C      | D ]}| |}q|S r9   r/   r.   r)   itemr&   r&   r'   r/         zPlotFilters.applyNr0   r1   r2   r3   r6   r4   r(   r/   r&   r&   r&   r'   rJ   w       rJ   c                   @   s:   e Zd ZU eed< eed< edefddZdd	d
ZdS )
PlotBinnerr   bin_sizer   c                 C   sT   t D ]}||v r||\}}t | |t||  S qtd| dtt  )Nz"Invalid operator for plot binner 'r   )PLOT_BINNERSr   r;   r   r   r    r!   r&   r&   r'   r(      s   zPlotBinner.parse_strr)   r*   r+   c                 C   s(   |  }|| j | j | j || j< |S )z#Applies this binner to a DataFrame.)copyr   r^   r-   r&   r&   r'   r/      s   zPlotBinner.applyNr0   )	r1   r2   r3   r4   r5   r;   r6   r(   r/   r&   r&   r&   r'   r]      s   
 r]   %r_   c                   @   rI   )PlotBinnersr   c                 C   rK   )Nc                 s   rL   r9   )r]   r(   rM   r&   r&   r'   rP      rQ   z(PlotBinners.parse_str.<locals>.<genexpr>rR   rS   rT   r&   r&   r'   r(      rU   zPlotBinners.parse_strr)   r*   r+   c                 C   rV   r9   rW   rX   r&   r&   r'   r/      rZ   zPlotBinners.applyNr0   r[   r&   r&   r&   r'   rb      r\   rb   pathr+   c                 C   s8   |  d}t|W  d    S 1 sw   Y  d S )Nrb)openjsonload)rc   fr&   r&   r'   _json_load_bytes   s   $ri   datac                 C   sd   g }| D ]+}i }|  D ]\}}t|tr%|dv r t|||< q|||< q|||< q|| q|S )z
    Convert string values "inf", "-inf", and "nan" to their float equivalents.

    This handles the case where JSON serialization represents inf/nan as strings.
    )infz-infnan)items
isinstancer4   r;   append)rj   converted_datarecordconverted_recordr$   r%   r&   r&   r'   _convert_inf_nan_strings   s   


rs   run_data
metric_keyc              
   C   s:   z| | W S  t y } ztd|d| |d }~ww )NzCannot find metric z in run_data=)KeyErrorr   )rt   ru   excr&   r&   r'   _get_metric   s   
rx   
group_keysc                    s   t  fdd|D S )Nc                 3   s"    | ]}|t t |fV  qd S r9   )r4   rx   rN   krt   r&   r'   rP           z_get_group.<locals>.<genexpr>)tuple)rt   ry   r&   r|   r'   
_get_group   s   r   fig_dirgroup.fig_namec                 C   sB   t t  }|| |r|dd |D  | td|d  S )Nc                 s   s"    | ]\}}| d | V  qdS )=Nr&   )rN   r{   vr&   r&   r'   rP      r}   z _get_fig_path.<locals>.<genexpr>-z.png)listr4   ro   extendr   join)r   r   r   partsr&   r&   r'   _get_fig_path   s
   

r   c                   @   sH   e Zd ZeZdefddZdee dB dedB dedB ddfdd	Z	dS )
DummyExecutorr+   c                 C   s   | S r9   r&   )r.   r&   r&   r'   	__enter__   s   zDummyExecutor.__enter__exc_typeN	exc_valueexc_tracebackc                 C   s   d S r9   r&   )r.   r   r   r   r&   r&   r'   __exit__   s   zDummyExecutor.__exit__)
r1   r2   r3   mapr
   r   typeBaseExceptionr   r   r&   r&   r&   r'   r      s    
r   fig_group_datarow_bycol_bycurve_byvar_xvar_y	filter_bybin_byscale_xscale_ydry_run
error_bars
fig_heightfig_dpic             
      sh  |\}}t |fddd}t|}t fdd|D }t| ||}td tdt|  td| d	| d
 td|  |rJtd d S t|}tj	||j
vrftd|dj
  |j
vrxtd|dj
  D ]}|j
vrtd|dj
  qz D ]}|j
vrtd|dj
  q|D ]}|j
vrtd|dj
  q|||rψj|drtjfddD ddjdjddndd<  rtjfdd D ddjdjddndd< tjdd|d}r r|d nr!|d n r*|d  n|d! |	r8|j|	d" |
rA|j|
d# t|d$krqg |d d d R ^}}}}|jtj|||||d%|rfd&nd d' |j|d( n0|rtjfd)d|D ddjdjddndd*< |jtj||d*d%|rd&nd d+ |  |j||d, t|j td d S )-Nc                    
   t |  S r9   r   rY   )r   r&   r'   <lambda>
     
 z_plot_fig.<locals>.<lambda>r$   c                 3   s,    | ]\}}t t| fd ddV  qdS )c                    r   r9   r   r   r   r&   r'   r     r   z%_plot_fig.<locals>.<genexpr>.<lambda>r   N)lenr   )rN   _row_datar   r&   r'   rP     s
    
z_plot_fig.<locals>.<genexpr>z[BEGIN FIGURE]zGroup: zGrid: z rows x z colszOutput file: z[END FIGURE]zCannot find var_x=z2 in parameter sweep results. Available variables: zCannot find var_y=zCannot find row_by=zCannot find col_by=zCannot find curve_by=)byc                    "   g | ]}|d   |  t qS r   astyper4   rz   r)   r&   r'   
<listcomp>G     " z_plot_fig.<locals>.<listcomp>r   )axis
z(All)	row_groupc                    r   r   r   rz   r   r&   r'   r   P  r   	col_group)rowcolheightz{row_name}
{col_name}z
{row_name}z
{col_name} )xscale)yscale   Tsd)xyhuestylesizemarkerserrorbar)titlec                    r   r   r   rz   r   r&   r'   r   y  r   curve_group)r   r   r   r   r   )dpi)r   r   maxr   printdictrs   pd	DataFramefrom_recordscolumnsr   tolistr/   sort_valuesconcataggr   sns	FacetGrid
set_titlessetmap_dataframelineplot
add_legendsavefigpltclosefigure)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   	fig_groupfig_data
row_groupsnum_rowsnum_colsfig_pathr{   gr   r   r   r   r&   )r   r)   r   r'   	_plot_fig   s   







	
		r   FIGURET皙@,  )r   r   r   r   
output_dirfig_byc                   s   dd |  dD }|std|  |jddd t| fddd	}t|d
kr-t nt #}t|t	t
||||||||	|
||||||d| W d    d S 1 sVw   Y  d S )Nc                 S   s   g | ]}t |D ]}|qqS r&   )ri   )rN   rc   rt   r&   r&   r'   r     s    zplot.<locals>.<listcomp>z**/summary.jsonz/Did not find any parameter sweep results under T)parentsexist_okc                    r   r9   r   r   r   r&   r'   r     r   zplot.<locals>.<lambda>r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   )rglobr   mkdirr   r   r   r   allr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   all_data
fig_groupsexecutorr&   r   r'   plot  sF   
"r   c                   @   s  e Zd ZU eed< eed< ee ed< ee ed< ee ed< ee ed< eed< eed< eed	< eed
< edB ed< edB ed< e	ed< dZ
eed< dZe	ed< dZeed< dZeed< dZee ed< dZee ed< edejfddZedejdejfd d!ZdS )"SweepPlotArgsr   r   r   r   r   r   r   r   r   r   Nr   r   r   r   r   Tr   r   r   r   r   r   parser_namez5Plot performance curves from parameter sweep results.parser_helpargsc                 C   s  t |j}| std| |jsg n|jd}|js g n|jd}|js+g n|jd}|js6g n|jd}| di d|d||j	 d|d|d|d|d	|j
d
|jdt|jdt|jd|jd|jd|jd|jd|j d|jd|jS )Nz!No parameter sweep results under rR   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r&   )r   
OUTPUT_DIRexistsr   r   r   r   r   r   r   r   r   rJ   r(   r   rb   r   r   r   r   r   no_error_barsr   r   )r"   r   r   r   r   r   r   r&   r&   r'   from_cli_args  sT   

	
zSweepPlotArgs.from_cli_argsparserr+   c                 C   s2  |j dtddd |j dtddd |j dtdd	d |j d
tddd |j dtddd |j dtd dd |j dtddd |j dtddd |j dtddd |j dtddd |j dtd dd |j dtd dd |j dtdd d |j d!d"d#d$ |j d%td&d'd |j d(td)d*d |j d+d"d,d$ |S )-Nr   resultsznThe directory containing the results to plot, i.e., the `--output-dir` argument to the parameter sweep script.)r   defaulthelpz	--fig-dirr   zdThe directory to save the figures, relative to `OUTPUT_DIR`. By default, the same directory is used.z--fig-byztA comma-separated list of variables, such that a separate figure is created for each combination of these variables.z--row-byzqA comma-separated list of variables, such that a separate row is created for each combination of these variables.z--col-byztA comma-separated list of variables, such that a separate column is created for each combination of these variables.z
--curve-byzsA comma-separated list of variables, such that a separate curve is created for each combination of these variables.z--var-xrequest_throughputzThe variable for the x-axis.z--var-yp99_e2el_mszThe variable for the y-axisz--filter-bya  A comma-separated list of statements indicating values to filter by. This is useful to remove outliers. Example: `max_concurrency<1000,max_num_batched_tokens<=4096` means plot only the points where `max_concurrency` is less than 1000 and `max_num_batched_tokens` is no greater than 4096.z--bin-byzA comma-separated list of statements indicating values to bin by. This is useful to avoid plotting points that are too close together. Example: `request_throughput%%1` means use a bin size of 1 for the `request_throughput` variable.z	--scale-xzThe scale to use for the x-axis. Currently only accepts string values such as 'log' and 'sqrt'. See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.htmlz	--scale-yzThe scale to use for the y-axis. Currently only accepts string values such as 'log' and 'sqrt'. See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.htmlz
--fig-namer   zName prefix for the output figure file. Group data is always appended when present. Default: 'FIGURE'. Example: --fig-name my_performance_plotz--no-error-bars
store_truezJIf set, disables error bars on the plot. By default, error bars are shown.)actionr  z--fig-heightr   z.Height of each subplot in inches. Default: 6.4z	--fig-dpir   z>Resolution of the output figure in dots per inch. Default: 300z	--dry-runzZIf set, prints the information about each figure to plot, then exits without drawing them.)add_argumentr4   r;   int)r"   r   r&   r&   r'   add_cli_args  s   
	zSweepPlotArgs.add_cli_args)r1   r2   r3   r   r5   r   r4   rJ   rb   boolr   r   r   r;   r   r  r   r	   r   r6   argparse	Namespacer   ArgumentParserr	  r&   r&   r&   r'   r     s0   
 r   r   c                 C   s   t di d| jd| jd| jd| jd| jd| jd| jd| jd	| j	d
| j
d| jd| jd| jd| jd| jd| jd| jS )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r&   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r&   r&   r'   run_main  sF   	
r  c                 C   s   t t|  d S r9   )r  r   r   r  r&   r&   r'   main  s   r  __main__)description)Kr  rf   abcr   r   concurrent.futuresr   dataclassesr   	functoolsr   pathlibr   typesr   typingr	   typing_extensionsr
   r   vllm.utils.collection_utilsr   vllm.utils.import_utilsr   utilsr   matplotlib.pyplotr   r   r   r   r   r   ImportErrorplaceholder_attrr   r8   r@   rA   rD   rE   rF   r   r   r4   r   r5   r   rJ   r]   r_   rb   objectri   rs   rx   r   r~   r   r   r
  r;   r  r   r   r   r  r  r  r1   r  r   r   r	  
parse_argsr&   r&   r&   r'   <module>   s  

*&&	

 .	

? 5

