o
    ٷiI                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ eeZdd Zdd Zd	d
 Zdd Zdd ZedkrVe  dS dS )    N)setup_logger)BenchmarkRecord)WhisperConfigWhisperProcessorc                  C   s  t  } | jddtddd | jddd dd	 | jd
dd ddgdd | jddtdd | jddtdd | jddddd | jddddd | jdtdd  | jd!td"d  | jd#tdd$d | jd%tdg d&d'd( | jd)tdg d*d+d( | jd,td-d.d/ | jd0ddd1d | jd2tdd3d/ | jd4td d5d/ | jd6ddd7 |  }t|d8|jd9d: 	d;d< d=|j
 d<|j }|js||_tj|jdd> | jd?9  _|S )@Nz-a--audio-pathTz0Path to folder of audio files for E2E evaluation)typerequiredhelpz-lz
--languagezLanguage of audio file)defaultr	   z-tz--task
transcribe	translatezTask to complete)r
   choicesr	   z-w--warmup-runs   )r   r
   z-n
--num-runs
   z--hf-pt-eagerF
store_truez,Benchmark in PyTorch without `torch.compile`)r
   actionr	   z--hf-pt-compilez)Benchmark in PyTorch with `torch.compile`--hf-ort-dir-pathzDPath to folder containing ONNX models for Optimum + ORT benchmarking)r   r	   --ort-model-pathz'Path to ONNX model for ORT benchmarking--model-namez9Model name in Hugging Face (e.g. openai/whisper-large-v2)--precision)int8fp16fp32zPrecision to run model)r   r   r   r	   --device)cpucudarocmzDevice to benchmark models--device-idr   zGPU device ID)r   r
   r	   z	--verbosezPrint detailed logsz	--timeoutz8Number of mins to attempt the benchmark before moving on--log-folderz'Path to folder to save logs and results--tune)r
   r   
model_size/.-z./)exist_ok<   )argparseArgumentParseradd_argumentstrint
parse_argssetattr
model_namesplitreplacer"   	precision
log_folderosmakedirstimeout)parserargslog_folder_name r;   i/home/ubuntu/.local/lib/python3.10/site-packages/onnxruntime/transformers/models/whisper/benchmark_all.pyget_args   s    r=   c                  C   s  g }d }d}d}d}d}d\}	}
d\}}d\}}}}d\}}d}d}d	}d
}t |}|D ]}|dd}||v r>d}n||v rEd}n
||v sM||v rOd}||v rat|t||d }q.||v rt|t||d }|dkr}||}	}
d }|dkr||}}d }q.||v rt|t||d }|| }|d }q.||v r4d|v rt||dd |d d }n |||t| d  dd}t||  d }t|d }|	r|	nd|r|nd |r|nd }|d }|r|| nd}t	
d| d t	
d| d t	
d|  |||	|
|r |nd|r&|nd|||||g
 }|| q.W d    |S 1 sAw   Y  |S )NzLoad audio: zFeature extraction: zEvaluating PyTorch...zEvaluating ONNX Runtime...)NN)NNNNz	Latency: zThroughput: zGenerated token length: zpeak=
 z
load-audiozfeature-extractionprocess   CPU=   z MB'"max_used_MBr   r$   zTotal latency: z szAudio duration: zReal-time factor: )openr2   floatlenrfindr-   findjsonloadsloggerinfoappend) 	device_idlog_filebase_resultsentriesstepload_audio_patternfeat_ext_patternpytorch_patternonnxruntime_patternload_audio_latency_sload_audio_throughput_sfeat_ext_latency_sfeat_ext_throughput_stoken_length	latency_sper_token_latency_sper_token_latency_ms
throughputmemorylatency_patternthroughput_patterntoken_length_patternmemory_patternf
input_linelinepeakusagetotal_latencyaudio_durationrtfentryr;   r;   r<   process_log_file   s   




&"




CCrs   c                 C   s  dd l }|j| g dd}|d d|d< |d d|d< |d d|d< |d	 d|d	< |d
 d|d
< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< dd l}|j}tdd |D }d}d}|r|d dd }|d dd }g }	| D ]\}
}|d dkrt|d |d |d |d ||}nt|d |d |d |d t	j
t	j}|d |jjd< |d |j_|d |j_|d |jjd< |d	 |jjd < |d
 |jjd!< |d |jjd"< |d |jjd#< |d |jjd$< |d |jjd%< |d |jjd&< |d d' |j_|d |j_|d |j_|	| qt||	 t|d(d)|	 td*| d+ d S ),Nr   )Warmup RunsMeasured Runs
Model NameEngine	PrecisionDevice
Audio FileDuration (s)Token LengthLoad Audio Latency (s)Load Audio Throughput (qps)Feature Extractor Latency (s)"Feature Extractor Throughput (qps)Latency (s)Per Token Latency (ms/token)Throughput (qps)Memory (GB)Real Time Factor (RTF))columnsrt   r-   ru   r{   rJ   r|   r}   r~   r   r   r   r   r   r   r   c                 S   s(   g | ]}|j d v r|j  d|j qS ))onnxruntimezonnxruntime-gpu==)keyversion).0ir;   r;   r<   
<listcomp>&  s   ( z save_results.<locals>.<listcomp>r?   r   rE   rw   r   rv   rx   ry   rz   
audio_filedurationr`   load_audio_latencyload_audio_throughputfeature_extractor_latency_s feature_extractor_throughput_qpsrc   rq   rB   .csvz.jsonzResults saved in !)pandas	DataFrameastypepkg_resourcesworking_setsortedr1   iterrowsr   torch__name____version__config
customizedwarmup_runsmeasured_runsmetricslatency_ms_meanthroughput_qpsmax_memory_usage_GBrR   save_as_csvsave_as_jsonr2   rP   rQ   )resultsfilenamepddfr   installed_packagesinstalled_packages_listort_pkg_nameort_pkg_versionrecords_rowrecordr;   r;   r<   save_results   sn    r   c              	   C   s   | dt j  dd}tj| j|}t|d'}tj|||d}z|	| j
 W n tjy8   |  Y nw W d    n1 sCw   Y  td | j| j| j|| j| j||g}	t| j||	}
|
S )Nr   %Y-%m-%d_%H:%M:%Sz.logw)stdoutstderrz Gathering data from log files...)datetimenowr5   pathjoinr4   rI   
subprocessPopenwaitr7   TimeoutExpiredkillrP   rQ   r   num_runsr0   r3   devicers   rS   )r9   benchmark_cmdenginer   r   log_filenamelog_pathrT   r@   rU   r   r;   r;   r<   	benchmarkQ  s.   

r   c                  C   sr  t  } t| j t| j dtjj_	t
| j}t| j}|j| j| jd}|jgdd |D  }| jr?| jr?dt|gng }| jrM| jrMdt|gng }| jrUdgng }g }t| jD ]:}	tj| j|	}
ztj|
d}W n' ty } zd}tjd	| dd
 td|	 d W Y d }~nd }~ww td|
 d | jrdddd|
ddd| jd| jd| jdt| j dt| j!dt| j"d| j#g| }td t	| |d|	|}|$| | j%rdddd|
ddd| jd| jd| jdt| j dt| j!dt| j"d| j#g| }td t	| |d |	|}|$| | j&rYdddd|
dd!d"| j&d| jd| jd| jdt| j dt| j!dt| j"d| j#g| }td# t	| |d$|	|}|$| | j'rdddd|
dd%d&| j'd| jd| jd| jdt| j dt| j!dt| j"d| j#g| | }td' t	| |d(|	|}|$| q_| j( d)| j d*t)j)* d+d,}t+|tj| j#| d S )-NT)languagetaskc                 S   s   g | ]}|d  qS )rE   r;   )r   token_idr;   r;   r<   r   w  s    zmain.<locals>.<listcomp>z--decoder-input-idsr!   )r   r$   z@An error occurred while trying to calculate the audio duration: )exc_infozJIf you get an error that says:
	soundfile.LibsndfileError: Error opening 'zt': File contains data in an unknown format.
you may not have installed `ffmpeg` in addition to installing `librosa`.zTesting z...pythonz-mzmodels.whisper.benchmarkr   z--benchmark-typezhf-pt-eagerr   r   r   r   r   r   r    z'Benchmark PyTorch without torch.compilezpytorch-eagerzhf-pt-compilez$Benchmark PyTorch with torch.compilezpytorch-compilezhf-ortr   z Benchmark Optimum + ONNX Runtimezoptimum-ortortr   zBenchmark ONNX Runtimer   r&   r   r   r   ),r=   r   verboserP   rQ   __dict__r   backendscudnnr   r   from_pretrainedr0   r   get_decoder_prompt_idsr   r   decoder_start_token_idr,   tuner5   listdir
audio_pathr   r   librosaget_duration	Exceptionwarninghf_pt_eagerr3   r   rS   r   r   r4   extendhf_pt_compilehf_ort_dir_pathort_model_pathr"   r   r   r   )r9   r   	processorhf_forced_decoder_idsort_forced_decoder_idshf_decoder_input_ids_cmdort_decoder_input_ids_cmdort_tune_cmdall_resultsr   r   r   er   r   csv_filer;   r;   r<   mainl  s*  










"r   __main__)r)   r   rN   loggingr5   r   r   r   benchmark_helperr   r   r   transformersr   r   	getLoggerr   rP   r=   rs   r   r   r   r;   r;   r;   r<   <module>   s,   
 [Z "
