o ٷi<@sddlZddlZddlZddlZddlmZddlZddlZddlmZm Z m Z m Z m Z ddl mZmZmZmZddlmZddlmZddlmZddlmZed Zdd d Zd d ZedkrpeZe ej eedSdS)N)datetime) Precisioncreate_onnxruntime_sessionget_ort_environment_variablesprepare_environment setup_logger)DEFAULT_TOLERANCE MODEL_CLASSESPRETRAINED_GPT2_MODELS Gpt2Helper)version)QuantizeHelper) AutoConfig) __version__c Cs6t}|jdddtddtd|jddtd ttd dtd |jd dtt j d ddd|jddtt j d ddd|jdddt dd|jdddddd|jdddddd|j dd |jd!t d"dgd#d$d%|jd&ddd'd|j dd(|jd)d*t t jtt d+d,|jd-ddd.d|j dd/|jd0d1d2t d3gd4d5|jd6d2t d3gd7d5|jd8d9d2t gd:d;d5|jdd?|jd@dt dAdBd|jdCdddD|j ddE|jdFdddD|j ddG|jdHdddD|j ddI|jdJdddD|j ddK||}|S)LNz-mz--model_name_or_pathTz;Model path, or pretrained model name selected in the list: z, )requiredtypehelpz --model_classFGPT2LMHeadModelz!Model type selected in the list: )rrdefaultchoicesrz --cache_dir. cache_modelsz%Directory to cache pre-trained models)rrrrz --onnx_dir onnx_modelszDirectory to store onnx modelsz --test_timesdz8Number of repeat times to get average inference latency.)rrrrz-vz--validate_onnx store_truezValidate ONNX model)ractionrz-oz--optimize_onnxz'Use optimizer.py to optimize onnx model) optimize_onnxz--stager)ra6Stage in generation: 1 (initial decoder), 2 (decoder), 0 (both). 1 - decode the first token when past_sequence_length is zero; 2 - decode the remaining tokens when past_sequence_length is not zero; 0 - one onnx model for both stages 1 and 2. Note that we will optimize 1 and 2 differently for best performance.)rrrrrz --use_gpuzuse GPU for inference)use_gpuz-pz --precisionzfPrecision of model to run. fp32 for full precision, fp16 for half precision, and int8 for quantization)rrrrz --torchscriptzuse Torchscript) torchscriptz-bz --batch_sizes+rz batch size)nargsrrrz--sequence_lengthsz!sequence lengths (excluding past)z-sz--past_sequence_lengths) @zpast sequence lengthsz-rz --result_csvz$CSV file for saving summary results.)rrrz --thread_numzThreads to usez--include_copy_output_latency)rr)include_copy_output_latencyz --verbose)verbosez--output_torch_latency)output_torch_latencyz--disable_io_binding)disable_io_binding)argparseArgumentParser add_argumentstrjoinr listr keysospathint set_defaultsrFLOAT32 parse_args)argvparserargsr?g/home/ubuntu/.local/lib/python3.10/site-packages/onnxruntime/transformers/models/gpt2/benchmark_gpt2.pyparse_arguments!s              rAc!Cstttdkrtdtd||jtjkr&|j r"|j s&Jd|jtj kr3|j r3Jd|j dkrB|j dgksBJdt|jdkrOtjd d n|jttj|j}|j}t|||j t|jd}t}tj|j|j|d }|j|j||d }t |j rd nd}|!||j"dk}|j#||j|jd |d} | d} t|jd} |j$||| |j%|| | d|j s|jtj&kr| |jtj krt'|jnd} |j | d| |jtjk|j(j)|j(j*|d |j d|jtj krtdt+,| | d|t+-|}td| d} |jr|j|||| | d}t.| |j d|j|j%d} | dur0dS|/t0|j1t0|j t0|j2||j} |3| ||jtjk}|j4p[d5t678d}t9|ddde}gd }t:j;||d!}|<|j1D]I}|j2D]A}|j D]9}|dkr|dkr|dksJt=d"||||j>||||j)|j*|j"|j?||jtjk| | d# }|/|||||j}z|j@s|jAr|B|||jC\}}tD|D],\}}tE|tFrt=d$|d%tG|d&|djHqt=d$|d'|jHqnd}d}|jIr|J| ||jC\}}n|jK| ||||jCd|jLd(\}}|j@rb|}|jIsDg}|D] }|M|NOq7|jP|||jtQ|jtQ|jd)rbtd*tQ|jd+td,|||||jIrod-nd|rud.nd|j|j|j tR|j |j|j |j||||jI|r|d/nd0|d/d } |S| WqtTytjUd1d d2YWddSwq}qwWdn 1swYtd3||S)4Nz3.1.0z/This tool requires transformers 3.1.0 or later.z Arguments:z'fp16 requires --optimize_onnx --use_gpuzquantization only supports CPUrrzrC output_dirrV gpt2helperrDmodelr{use_external_data_formatonnx_model_pathsonnx_model_path use_paddingsessionmax_output_shapesoutput_buffers csv_filenamecsv_file column_names csv_writerr[r\r] dummy_inputs output_shapesoutputsr^ivalue ort_outputs ort_latency copy_outputsoutputrowr?r?r@mains    "           "        r__main__)N)!r/rloggingr6rrsrpbenchmark_helperrrrrr gpt2_helperrr r r packagingr quantize_helperr transformersrrri getLoggerrkrAr__name__r>r,r?r?r?r@s.       s