o ٷi%a@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl ZddlZddlmZddlmZmZddlmZmZmZmZddlZeeZddd Zd d Z d d Z!ddZ"ddZ#ddZ$ddZ%edkre%dSdS)) annotationsN) setup_logger)add_io_bindings_as_tensorsget_initial_inputs_and_outputs) AutoConfigAutoModelForCausalLM AutoTokenizerBitsAndBytesConfigargsargparse.Namespacec Cs|jdvrd}|jdkr9|jdkr9tdddtjd}tj|jdkr$|jn|j |j |j |j |j dd||jd id }n[z&tj|jdkrD|jn|j |j |j |j |j d|jdkrVdnd d |j}Wn4ty}z(td |tj|jdkrv|jn|j |j |j |j |j ddd |j}WYd}~nd}~ww||jdkrt|}|St}|jdkrdd|jifnd}tj|j||gd}|S)Npt-eager pt-compileint4cudaTnf4) load_in_4bitbnb_4bit_use_double_quantbnb_4bit_quant_typebnb_4bit_compute_dtypeflash_attention_280GB) cache_dir torch_dtypeuse_auth_tokentrust_remote_code use_cacheattn_implementationquantization_config max_memorysdpa)rrrrrrz&Try to load a model using eager mode: eagerrCUDAExecutionProvider device_idCPUExecutionProvider) sess_options providers)benchmark_typeonnx_precisiondevicer torchfloat16rfrom_pretrained hf_dir_path model_namerrauthtrustr$to target_device ExceptionprintevalcompileortSessionOptionsInferenceSessiononnx_model_path)r model bnb_configer&epr@g/home/ubuntu/.local/lib/python3.10/site-packages/onnxruntime/transformers/models/llama/benchmark_e2e.py get_model8st       rBc Cs"|jdkr t|di|}Wdn1swYd}|jdvr4|jdkr3tj|jnt||||j|j }| t }t |D]8}|jdvryt|di|}|jdkritj|jWdn1sswYqJ|||qJt }|||} | |fS)Nrr cpur@)r(r+no_gradr*r synchronizer3ruse_fp16use_buffer_sharesynchronize_inputstime perf_counterrangerun_with_iobindingsynchronize_outputs) r r<runsinputsoutputs io_bindingstart_endavgr@r@rA run_inferencexs4           rVc CsFtt|||||j|j|j|j\}}t|||j||\}}||fSN) clear_cacherr3rFrGenginerV warmup_runs) r r<config tokenizer prompt_lengthpromptrOrPrSr@r@rAprepare_model_for_inferences r_cCsttjdSrW)gccollectr+r empty_cacher@r@r@rArXsrXcCsvtj|ddddddddd |d d d |d d d |d d |d ddgd}|j|ddtd|ddS)Nz Batch Sizez Prompt LengthzPrompt Processing Latency (ms)z"Prompt Processing Throughput (tps)zSampling Latency (ms)zSampling Throughput (tps)z"First Token Generated Latency (ms)z&First Token Generated Throughput (tps)Average Latency of First z Tokens Generated (ms)Average Throughput of First z Tokens Generated (tps)zWall-Clock Latency (s)zWall-Clock Throughput (tps))columnsF)indexzResults saved in !)pd DataFrameto_csvloggerinfo)resultsfilename gen_lengthdfr@r@rA save_resultss(  rrc Cst}|jddtdgdd|jddtdd d |jd d dd dd|jdddd dd|jddttjdddd|jdtddd|jddddd|jd d!dtjdd"d#d$d%d&|jd'dd d(d|jd)dd d*df|jd+d,d-d.|jd/d0d1d.|jd2d3dtd4gd5d6d7|jd8d9td:d;d|jdnd?d?d>gd@|jdAdBtdCdD|jdEdFtdGdD|jdHdItdJdD|jdKtdLdD| }t j |jt|jdM|jvrt|dN|jdO|jdPkr|jdQ|jif|_|jdMkr|jsJdR|jdS|_|jdS|_t|dT|j|jdUvs8|jdVkr:|jd?kr:d4ndW|_|jd?krIdX|jn|j}|jdWkrUtjntj}|jdMkr`dMndY}t|dZ|t|d[|t|d\|t|d]|jdWk|jo|dMk|_|S)^Nz-btz--benchmark-typeT)r rr8)typerequiredchoicesz-mz --model-nameFzr~rrrzcuda:ptr3rrYrF)argparseArgumentParser add_argumentstrospathjoinintr+r is_available parse_argsnprandomseed manual_seedr(setattrr*upperrr$r; batch_sizessplitprompt_lengths precisionr,float32rG)parserr r3rrYr@r@rAget_argss       *   rc9st}tdt|jd}t|j}tj|ddd}Wdn1s(wYt j |j dkr7|j n|j |j |j|jd}tj |j dkrL|j n|j |j |j|jd}t|}g}t|j|jD]u\}}t|t|}}td|d|t||j} ||vrttd |d |jd |jd |d |d|d|d|d|jd|d||g|} ||g} ztdt|||||| \} } t|||j| | \}} |d}|||}td|dtd|||d| ||gtdtt|||||| \} } | d }|j!d}|j"}t#|dr,|j$n|j%|j&}t'j(||j)t'j*d}g}g}t+,}|| krt||d| | \}} |-|t+,}| d j!ddkr| d!.dd}|j/dd"0d|j12|d|j1}t'3| d d|4}n | d dddddf}t'j5|dd"}||B|j6k}|7||j68|dg}t+,} |-| |t'j9||gdd"}|d7}|| d<t'9| d!|:t'j;8|dgd| d!<d#| vrt'j<| d#dd"d$8|dd| d#<| d j!ddkr| d ddddddf=| d <| d >|j?d%kr,| d&| d&<nh|j@stA|jBD]}!| d'|!d(| d)|!d(<| d'|!d*| d)|!d*<q5| d!j!d}"tA|jBD]2}!t'j(|||"||j)|jCd}#t'j(|||"||j)|jCd}$| Dd'|!d(|#=d'|!d*|$=iqa|| ksIt+,}%|Ed$|jFrd+tG|tH|}&tItJfd,d|}tH|}'td-|&|'d.d/dd0t.|tH|}(|(d})|d|(}*td1|)dtd2|*d|d$}+|+d},|d|+}-td3|,dtd4|-d|jd5}.t.|d|.tH|d|.}/|/d}0|d|/}1td6|.d7|0dtd8|.d7|1dt.|tH|}2|2d}3|d|2}4td6|jd7|3dtd8|jd7|4d|%|}5|||j|5}6td9|5d:td;|||j|5dtd<| |)|*|,|-|0|1|3|4|5|6g |-| WqetKy}7ztd=|d|d>|7WYd}7~7qed}7~7wwd?|j?d@tLjLMdAdB}8tN||8|jdS)CNFcSsdd|DS)NcSsi|] \}}t||qSr@)r).0kvr@r@rA jsz*main....)items)dr@r@rAjszmain..) object_hookr)rrrzRunning batch size = z, prompt length = z2 A prompt of size z was not found in 'zv'. There are a couple of solutions to fix this. 1) You can change one of the keys in 'z' to be z). If za < actual prompt's length, the benchmark E2E tool will repeat the first word in the prompt until zB = actual prompt's length. If zm > actual prompt's length, the benchmark E2E tool will automatically trim the actual prompt's length so that zd = actual prompt's length. 2) You can add a new key-value entry in 'z' of the form 'z,': 'your prompt goes here'. zMeasuring prompt processing...iz&Average Latency of Prompt Processing: z msz)Average Throughput of Prompt Processing: z tpszMeasuring token generation... input_idshead_dim)r*dtypelogitsattention_mask)dim position_idsrrpast_key_valueszpresent.z.keyzpast_key_values.z.value cs |kSrWr@)acc_timeanomaly_threshold_factor min_time_sr@rAr s z Filtered out z$ anomaly accelerator times that are zx greater than z ms...zAverage Latency of Sampling: z Average Throughput of Sampling: z"Latency of First Token Generated: z%Throughput of First Token Generated: rdrcz Tokens Generated: rezWall-Clock Latency: z szWall-Clock Throughput: zAdding results to CSVz$Could not benchmark at batch size = z - benchmark__e2e_z%Y-%m-%d_%H:%M:%Sz.csv)Orrrlrm__dict__open prompts_filejsonloadrr-r.r/rr0r1rrB itertoolsproductrrrrXgeneration_lengthNotImplementedErrortextwrapdedentr_rVnum_runsextendcloneshapenum_key_value_headshasattrr hidden_sizenum_attention_headsr+zerosr3boolrIrJappendsum unsqueezerepeat vocab_sizeviewgathersqueezeargmax eos_token_id masked_fillreshapecatr2int64max contiguouszero_rYrGrKnum_hidden_layersrupdatepopanomaly_filteringminlenlistfilterr4datetimenowrr)9r size_to_promptfr[r\r<all_csv_metrics batch_sizer] max_lengthr^ csv_metricsrOrPaccelerator_prompt_latency_saccelerator_prompt_latency_msaccelerator_prompt_thrpt all_token_idscurrent_length num_heads head_sizehas_eosaccelerator_timessampling_timeswall_clock_start_timeaccelerator_time_latency_ssampling_start_timeprompt_end_indicesidxsnext_token_logits next_tokens tokens_to_addsampling_end_timeinew_sequence_length present_key present_valuewall_clock_end_time orig_sizenew_sizeavg_sampling_latency_savg_sampling_latency_msavg_sampling_thrptfirst_token_latency_sfirst_token_latency_msfirst_token_thrpthalfwayhalfway_token_latency_shalfway_token_latency_mshalfway_token_thrptall_token_latency_sall_token_latency_msall_token_thrptwall_clock_latency_swall_clock_thrptr>ror@rrAmainbs               &&    O         (r__main__)r r )& __future__rrrr`rrloggingrrrInumpyrpandasrir+benchmark_helperr llama_inputsrr transformersrrrr onnxruntimer8 getLogger__name__rlrBrVr_rXrrrrr@r@r@rAs<    @ %~