o
    ٷi.                     @   s|  d dl Z d dlZd dlZd dlZz
d dlmZmZ W n ey+   d dlmZmZ Y nw d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ eeedk rpede dg dZdefddZ	d%dededB defddZ de!e"eef  defddZ#de!e"eef  dedefddZ$d d! Z%d&d"d#Z&e'd$kre%  dS dS )'    N)PackageNotFoundErrorversion)Path)Any)load_dataset)	evaluator)ORTModelForQuestionAnswering)__version__)r   )AutoTokenizerpipelinez1.13.1z1Please install optimum>=1.13.1. Current version: .)z5bert-large-uncased-whole-word-masking-finetuned-squadzdeepset/roberta-base-squad2z%distilbert-base-cased-distilled-squadpackage_namec                 C   s    zt | W S  ty   Y d S w N)r   r   )r    r   c/home/ubuntu/.local/lib/python3.10/site-packages/onnxruntime/transformers/models/bert/eval_squad.pyget_package_version7   s
   
r   CUDAExecutionProviderFmodel_id	onnx_pathuse_io_bindingc                 C   sz   |du r)t j| d||d}tjd| }|| tj|d}td| ||fS t jtj|t|j	||d}||fS )a  Load onnx model given pretrained model name and optional ONNX model path. If onnx_path is None,
    the default onnx model from optimum will be used.

    Args:
        model_id (str): pretrained model name or checkpoint path
        onnx_path (Optional[str], optional): path of onnx model to evaluate. Defaults to None.

    Returns:
        model: ORTModel for the onnx model
        onnx_path: the path of onnx model
    NT)exportproviderr   r   z
model.onnxzModel is exported to onnx file:)	file_namer   r   )
r   from_pretrainedospathjoinsave_pretrainedprintdirnamer   name)r   r   r   r   modelsave_onnx_dirr   r   r   load_onnx_model>   s    



r#   resultscsv_filenamec                 C   s|   t |dddd%}g d}tj||d}|  | D ]}|| q|  W d   n1 s0w   Y  td|  dS )	zOutput a CSV file with detail of each test results.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
    a asciimodenewlineencoding)pretrained_model_namer   r   disable_fused_attention
batch_sizesequence_lengthr   exactf1totalHasAns_exact	HasAns_f1HasAns_total
best_exactbest_exact_threshbest_f1best_f1_threshtotal_time_in_secondssamples_per_secondlatency_in_seconds
fieldnamesNz&Detail results are saved to csv file: )opencsv
DictWriterwriteheaderwriterowflushr   )r$   r%   csv_filecolumn_names
csv_writerresultr   r   r   output_detailsb   s   
rJ   metric_namec              	      s  t |dddd}g d tdd | D }|  tdd | D }|  td	d | D }|  g }|D ]}|D ]}	|d
|	 d|  q<q8tj| | d}
|
  |D ]^}i }i }|t	|d | D ]8}|d |kr|| r fdd|
 D }|s|| |d }	|d }d
|	 d| }||v r|| ||< qk|r|D ]
}||d||< q|
| qZ|  W d   n1 sw   Y  td| d|  dS )a  Output a CSV file with summary of a metric on combinations of batch_size and sequence_length.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
        metric_name (str): the metric to summarize
    r&   r'   r(   r)   )r-   r   r   r.   r   c                 S      h | ]}|d  qS )r   r   .0rI   r   r   r   	<setcomp>       z!output_summary.<locals>.<setcomp>c                 S   rL   )r/   r   rM   r   r   r   rO      rP   c                 S   rL   )r0   r   rM   r   r   r   rO      rP   b_sr>   r   c                    s   i | ]\}}| v r||qS r   r   )rN   kvheader_namesr   r   
<dictcomp>   s    z"output_summary.<locals>.<dictcomp>r/   r0   NzSummary results for z are saved to csv file: )r@   listsortappendrA   rB   rC   updatedictfromkeysitemsgetrD   rE   r   )r$   r%   rK   rF   
model_listbatch_sizessequence_lengths	key_namesr0   r/   rH   r!   rowvaluesrI   headerskeyr   rU   r   output_summary   sL   


6rh   c                  C   s8  t  } t|  dD ]}t|}|rt| d| q	| j}| jr/tj| js/td| j tj	
dddk}g }t|}| jD ]}||_t|d d|_| jd u rXtd	 t }t|| j| j| j\}	}
t | }td
|dd t|	j ||	jjkrtdtd|	|d| jd}td}td t }td| jdkrd| j dndd}t | }td|dd td t }|j||ddd}t | }td|dd | j|d< ||d< ||d< |
|d < | j|d!< ||d"< | j|d#< t| || qBt|d$ d%D ]}t|| d&| qd S )'N)zonnxruntime-gpuonnxruntimeonnxtorchtransformersoptimumdatasetsevaluatez versionz Onnx model path does not exist: ORT_DISABLE_FUSED_ATTENTION01      z4Exporting onnx model. It might take a few minutes...z!Onnx model exported or loaded in z.1fz secondszTsequence length should not be larger than {ort_model.config.max_position_embeddings}zquestion-answeringT)r!   	tokenizerquestion_firstr/   zLoading dataset...squadr   zvalidation[:]
validation)splitzDataset loaded in z<Evaluating squad_v2 with ORT. It might take a few minutes...squad_v2)model_or_pipelinedatametricsquad_v2_formatzEvaluation done in r   r.   r-   r   r/   r0   r   z
detail.csv)r2   r1   r<   z.csv) parse_argumentsr   r   
model_namerj   r   r   existsRuntimeErrorenvironr_   r
   r   rb   model_max_lengthmin
doc_stridetimer#   r   r   configmax_position_embeddingsr   r/   r   r   r3   computerZ   rJ   rh   )argsr    package_versionr-   r.   all_resultsru   r0   
start_time	ort_modelr   latencyqa_pipelinetask_evaluatorsquad_datasetrI   rK   r   r   r   main   sr   



$



r   c                 C   s   t  }|jdddttd dt d |jddd	td
gdd |jddtddd |jddtddd |jddtd dd |jddddd |jddddd |jdd || }|S ) Nz-mz--model_nameFr   z=Checkpoint directory or pre-trained model names in the list: )requiredtypedefaulthelpz-sz--sequence_lengths+i  zFSequence lengths for onnx model inputs. It could have multiple values.)nargsr   r   r   z-bz--batch_size   zbatch size for inference.)r   r   r   z-tz--totalz+Total samples to test. 0 means all samples.z--onnxzbOptional onnx model path. If not specified, optimum will be used to export onnx model for testing.z
--providerr   zRSelect which Execution Provider to use for runs. Default is CUDAExecutionProvider.)r   r   r   z--use_io_binding
store_truezUse IO Binding for GPU.)r   actionr   )r   )argparseArgumentParseradd_argumentstrPRETRAINED_SQUAD_MODELSintset_defaults
parse_args)argvparserr   r   r   r   r     sT   		
r   __main__)Nr   Fr   )(r   rA   r   r   importlib.metadatar   r   ImportErrorimportlib_metadatapathlibr   typingr   rn   r   ro   r   optimum.onnxruntimer   optimum.versionr	   optimum_version	packagingversion_checkrl   r
   r   parser   r   r   boolr#   rX   r\   rJ   rh   r   r   __name__r   r   r   r   <module>   sH   
$")A
F6
