o -i@sVdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl Z ddl Z ddl m Z ddl mZmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddl m!Z!m"Z"ddl#m$Z$m%Z%dd l&m'Z'm(Z(dd l)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0m1Z1ddl2m3Z3 d2de4ede5de%de6de6de7e8e4e,dBff ddZ9 d2de4ede5de%de6de6de7e8e4e,ff ddZ:  d3de4ede5de$de6de6de6de8fddZ; d2de4eded$e?efd0d1ZFdS)4z'Benchmark offline inference throughput.N)Any)tqdm)AutoModelForCausalLMPreTrainedTokenizerBase) AIMODatasetBurstGPTDatasetConversationDatasetInstructCoderDatasetMultiModalConversationDatasetPrefixRepetitionRandomDataset RandomDatasetRandomDatasetForRerankingRandomMultiModalDataset SampleRequestShareGPTDataset SonnetDatasetVisionArenaDatasetadd_random_dataset_base_args"add_random_multimodal_dataset_args)#convert_to_pytorch_benchmark_format write_to_json)AsyncEngineArgs EngineArgs) TextPrompt TokensPrompt) LoRARequest) RequestOutput)BeamSearchParams) TokenizerLike get_tokenizer)merge_async_iteratorsFrequestsn engine_args do_profiledisable_detokenizereturnc sddlm}m}|dit|tfdd|Ds!Jdg}g}|D]9} d| jvr6t| jddnt| jd} | j rLt | j t sGJ| j | d <| | | ||d d d | j | d q'd} |jrmd d|D} d} d} | st}|r}j||| d d} |rt}n?| dusJddd|D}|dj }|D] } | j |ksJqt}|r|t||d d|r̈t}||| fS)NrLLMSamplingParamsc3&|]}jjj|j|jkVqdSN llm_engine model_config max_model_len prompt_lenexpected_output_len.0requestllmW/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/benchmarks/throughput.py 8  zrun_vllm..pPlease ensure that max_model_len is greater than the sum of prompt_len and expected_output_len for all requests.prompt_token_idsr<promptmulti_modal_data?Tr" temperaturetop_p ignore_eos max_tokens detokenizecSg|]}|jqSr7) lora_requestr2r7r7r8 Zzrun_vllm..F)rIuse_tqdmz$BeamSearch API does not support LoRAcSrHr7r>r2r7r7r8rJkrK) beam_widthrFrEr7)vllmr(r) dataclassesasdictallr?rrr@ isinstancedictappendr1 enable_loratime perf_counter start_profilegenerate stop_profile beam_searchr)r!r"r#r$r%r(r)promptssampling_paramsr4r? lora_requestsuse_beam_searchoutputsstartend output_lenr7r5r8run_vllm.s~           rdc sddlm}m}|d it|tfdd|Ds!Jdg}g}|D]} || j|||ddd| j| dq't } |rI j ||dd } |rW t } | | | fS) z Run vLLM chat benchmark. This function is recommended ONLY for benchmarking multimodal models as it properly handles multimodal inputs and chat formatting. For non-multimodal models, use run_vllm() instead. rr'c3r*r+r,r2r5r7r8r9r:z run_vllm_chat..r;rATrB)rLNr7)rNr(r)rOrPrQrTr?r1rVrWrXchatrZ) r!r"r#r$r%r(r)r\r]r4rar`rbr7r5r8 run_vllm_chats:       rf disable_frontend_multiprocessingc sddlm}ddlm}|||d4IdH}|jtfdd|Ds*Jdg} g} g} |D]?} d| jvrAt| jdd nt| jd } | j rWt | j t sRJ| j | d <| ||d d d | j | d| | | | jq2g}t}|r|IdHtt| | | D]\}\} }}|j| ||d|d}| |qt|}|2z3dHW\}}q6|r|IdHt}||WdIdHS1IdHswYdS)Nr)r))*build_async_engine_client_from_engine_args)rgc3s"|] }j|j|jkVqdSr+)r/r0r1r2r.r7r8r9s  z!run_vllm_async..r;r<r=r>r@rATrBtest)rI request_id)rNr)"vllm.entrypoints.openai.api_serverrhr.rQr?rrr@rRrSrTr1rIrVrWrX enumerateziprYr rZ)r!r"r#r$rgr%r)rhr6r\r]r^r4r? generatorsraisplr generatorall_gensresrbr7rir8run_vllm_asyncsn           0rvmodel tokenizermax_batch_sizetrust_remote_codec Csjt|ts Jdtj|tj|d}|jjdkr|j|_ | }t t |d}t } g} d} d} tt |D]u} || j}|| j}|| j}| |t| |} t| |} t | |kr| t |dkr|| dj}|| dj}t| |t| |dkrq7|| dd d j}|j| d |d d d | d }|s|j|d d |t | g} d} d} q7t }|| S)Nz*the hf backend only supports HF tokenizers)dtyperzllama)totalriptT)return_tensorspaddingrA) input_ids do_samplenum_return_sequencesrCrD use_cachemax_new_tokens)skip_special_tokens)rRrrfrom_pretrainedtorchfloat16config model_type eos_token pad_tokencudarlenrVrWranger?r0r1rTmaxrrY batch_decodeupdate)r!rwrxr"ryrzr%r6pbarrabatchmax_prompt_lenmax_output_lenrpr?r0rcnext_prompt_lennext_output_lenr llm_outputsrbr7r7r8run_hfs`         rargsresultscs\t|dgdgdfdddDd}|r,tj|jdd }t||dSdS) Nrequests_per_secondtokens_per_second)rrcsi|]}||qSr7r7)r3krr7r8 Hs z4save_to_pytorch_benchmark_format..) elapsed_time num_requeststotal_num_tokens)rmetrics extra_inforz .pytorch.json)rospathsplitext output_jsonr)rr pt_recordspt_filer7rr8 save_to_pytorch_benchmark_format?s  rc CsL|j|jd}||j|j|jd}|jdks |jdur\|jdvr\|j|d<t|dd}|dur1|n|j|d<t|dd}|durB|n|j |d <t|d d}|durS|n|j |d <t }n|jd krxt }|j d krld|d<|j durv|j |d <n|jdkr|js|jsJdt}|j|d<d|d<|j dur|j |d <|j dur|j |d <n`|jdkrt}nW|jdkr'|j dur|j |d <|jtjvrt}d|d<d|d<d|d<n1|jtjvrt}d|d<n#|jtjvrt}|j|d<|j|d<d|d<n |jtjvrt}|j|d<|j|d<d|d<n|jtjvr&t}d|d<d|d<n|jdkrDt}|j|d<|j|d<|j|d<|j|d <n|jdkrt}t|dd}|durY|nt|d d|d <t|d d}|durn|nt|d d|d <t|dd|d<t|dd|d<t|d d|d!<t|d"d|d#<d|d<t|dd}t|dd}|dur|n||d<|j|d<nQ|jd$krt }t|dd}|dur|nt|d d|d <t|d d}|dur|nt|d d|d <t|d%d&|d'<t|d(d) |d*<|j|d<nt!d+|jd,d-|"D}|d.i|j#d.i|} t$| |j%} | S)/N) dataset_path random_seed)rx lora_path max_lorasrrandom>prefix_repetition random-mm random-rerank range_ratiorandom_prefix_len prefix_lenrandom_input_len input_lenrandom_output_lenrcsharegpt vllm-chatTenable_multimodal_chatsonnetz;Tokenizer/model must have chat template for sonnet dataset.return_prompt_formattedburstgpthfdataset_subsettrain dataset_splitr suffix_len num_prefixesr random_mm_base_items_per_requestbase_items_per_request"random_mm_num_mm_items_range_rationum_mm_items_range_ratiorandom_mm_limit_mm_per_promptlimit_mm_per_promptrandom_mm_bucket_config bucket_configrrandom_batch_sizer~ batchsize no_rerankerF is_rerankerzUnknown dataset name: cSsi|] \}}|dur||qSr+r7)r3rvr7r7r8rsz get_requests..r7)&rseedrr num_prompts dataset_namerandom_range_ratiogetattrrrrcr rbackend chat_templatedefault_chat_templaterrrSUPPORTED_DATASET_PATHSr r hf_subsethf_splitrrr prefix_repetition_prefix_lenprefix_repetition_suffix_lenprefix_repetition_num_prefixesprefix_repetition_output_lenrr ValueErroritemssamplefilter_requests_for_dpdata_parallel_size) rrx common_kwargs sample_kwargsrrr dataset_clsrr!r7r7r8 get_requestsRs                                                        rcsLdkr|Sttjd}ttjd}||fddt|DS)Nr~RANK WORLD_SIZEcs g|] \}}|kr|qSr7r7)r3rprdata_parallel_rankrr7r8rJs  z*filter_requests_for_dp..)intrenvironrm)r!r global_rank world_sizer7rr8rs  rcCs|jdurtjddd|j|_t|dds|j|_hd}|j|vr+td|j|jsP|jsP|j dvrPt d d |_ t|d d}|j durP|durPtd |j d krmt|ddduset|dddurmtjdddn@|j d kr|jt j tj Btj Bvr|jdksJ|jdn|jtj tj Bvr|jdksJ|jdnt|jd|j dvr|jdurtjddd|j dkrt|dddur|jdkrtjddd|j dkrt|ddrtjddd|j dvr|jdurtjd dd|j dvrHt|d d}t|d!d}t|d"d}|j dur$|dur$tjd#dd|jdur6|dur6tjd$dd|jdurH|durHtjd%ddt|d&drY|jdkrYtd't|d&drj|jdurjtd(|jd krz|jdurztd)|jd kr|jdurtd*|jd+vrt|d,ddurtd-|jd.kr|jd/krtd0|jd.kr|jdkrtd1|jd.kr|j|jkrtd2|jdkr|jd3ks|jrtd4dSdS)5z* Validate command-line arguments. NzzThe '--dataset' argument will be deprecated in the next release. Please use '--dataset-name' and '--dataset-path' instead.) stacklevelrx>rmiirNrzUnsupported backend: >rz?When dataset path is not set, it will default to random datasetrrzNEither --input-len or --random-input-len must be provided for a random datasetrrrz\--hf-subset and --hf-split will be ignored since --dataset-name is not 'hf'.rz' needs to use vllm-chat as the backend.rNz" needs to use vllm as the backend.z is not supported by hf dataset.>rrrz{--random-range-ratio will be ignored since --dataset-name is not 'random', 'random-mm', or 'random-rerank'.rrr~zd--random-batch-size will be ignored since --dataset-name is not 'random-rerank'.rFzZ--no-reranker will be ignored since --dataset-name is not 'random-rerank'.>Nrrrzu--prefix-len will be ignored since --dataset-name is not 'random', 'random-mm', 'sonnet', or not set.rrz}Both --input-len and --random-input-len are specified. The random version (--random-input-len) will be preferred in this run.zBoth --output-len and --random-output-len are specified. The random version (--random-output-len) will be preferred in this run.zBoth --prefix-len and --random-prefix-len are specified. The random version (--random-prefix-len) will be preferred in this run.rUz4LoRA benchmarking is only supported for vLLM backendz3LoRA path must be provided when enable_lora is Truez,HF max batch size is required for HF backendz)HF max batch size is only for HF backend.>rr quantizationz&Quantization is only for vLLM backend.rautoz#dtype must be auto for MII backend.zn must be 1 for MII backend.z8Tokenizer must be the same as the model for MII backend.external_launcherzData parallel is only supported with external launcher mode with synchronous engine in offline benchmark, please use benchmark serving instead)datasetwarningswarnrrrwrxrrrprintrrrkeysr rr rrrrrcrhf_max_batch_sizer{r"rdistributed_executor_backend async_engine)rvalid_backendsrrrr7r7r8 validate_argss                     rparsercCs|jdtgddd|jdtgdddd |jd tdd d |jd tddd |jdtddd |jdtddd |jdtddd |jdtddd |jdtddd |jdtddd |jdddd d!|jd"ddd#d!|jd$dd%d&|jd'tdd(d |jd)td*d+d |jd,tdd-d |jd.tdd/d |jd0ddd1d!|jd2tdd3d |jd4tdd5d |jd6tdd7d |jd8tdd9d t|t|t|}dS):Nz --backend)rNrrrrN)typechoicesdefaultz--dataset-name)rrrrrrrrz$Name of the dataset to benchmark on.r)r r helpr z --datasetzPath to the ShareGPT dataset, will be deprecated in the next release. The dataset is expected to be a json in form of list[dict[..., conversations: list[dict[..., value: ]]]])r r r z--dataset-pathzPath to the datasetz --input-lenz$Input prompt length for each requestz --output-lenzMOutput length for each request. Overrides the output length from the dataset.z--nr~z)Number of generated sequences per prompt.z --num-promptsizNumber of prompts to process.z--hf-max-batch-sizez"Maximum batch size for HF backend.z --output-jsonz3Path to save the throughput results in JSON format.z--async-engine store_trueFz,Use vLLM async engine rather than LLM class.)actionr r z"--disable-frontend-multiprocessingz(Disable decoupled async engine frontend.z--disable-detokenizez[Do not detokenize the response (i.e. do not include detokenization time in the measurement))rr z --lora-pathztPath to the lora adapters to use. This can be an absolute path, a relative path, or a Hugging Face model identifier.z --prefix-lenrzRNumber of fixed prefix tokens before the random context in a request (default: 0).z --hf-subsetzSubset of the HF dataset.z --hf-splitzSplit of the HF dataset.z --profilezEUse vLLM Profiling. --profiler-config must be provided on the server.z--prefix-repetition-prefix-lenzMNumber of prefix tokens per request, used only for prefix repetition dataset.z--prefix-repetition-suffix-lenz|Number of suffix tokens per request, used only for prefix repetition dataset. Total input length is prefix_len + suffix_len.z --prefix-repetition-num-prefixesz|Number of prefixes to generate, used only for prefix repetition dataset. Prompts per prefix is num_requests // num_prefixes.z--prefix-repetition-output-lenzMNumber of output tokens per request, used only for prefix repetition dataset.) add_argumentstrrrrr add_cli_args)rr7r7r8rs     rc Cst||jdur d|_t|j|jdks|jdkr$|jdkr$d|_t|j|j|jd}t||}t dd|D}d}|jdkrm|j r[t t ||jt||j|j|jd }nTt||jt||j|jd \}}nB|jdkr|jd ksyJ|jrtd t||j||j|j|j|j}n|jd krt||jt||j|jd \}}ntd|j|rd}d}|D] }t|tsq||jrt |jnd7}|t!dd|j"D7}q||} nt!dd|D} t!dd|D}| |}|r|jd krt#d|jdt#dt ||dd| |dd||ddt#d|t#d||j$rf|t || t ||| |d} t%|j$d} t&j'| | ddWdn 1sZwYt(|| dSdS)Nrrrr)tokenizer_moderzcss|]}|jduVqdSr+)r@r2r7r7r8r9Uszmain..rN)rgr%r$)r%r$r~z/Profiling not implemented yet for backend='hf'.rzUnknown backend: css|] }|rt|jVqdSr+)r token_ids)r3or7r7r8r9scss|] }|j|jVqdSr+)r0r1r3rr7r7r8r9scss|]}|jVqdSr+)r1rr7r7r8r9sz+WARNING: Multi-modal request with z backend detected. The following metrics are not accurate because image tokens are not counted. See vllm-project/vllm/issues/9778 for details.z Throughput: z.2fz requests/s, z total tokens/s, z output tokens/szTotal num prompt tokens: zTotal num output tokens: )rrrrrw)indent))rrrrrrrxrzranyruvlooprunrvr"r from_cli_argsrgr%profilerdrtensor_parallel_sizeNotImplementedErrorrrwrrfrrRrr<rsumr`rropenjsondumpr) rrxr!is_multi_modalrequest_outputsrtotal_prompt_tokenstotal_output_tokensrorrfr7r7r8mainCs             r*)F)FF)G__doc__argparserOr"rrrVrtypingrrrr transformersrrvllm.benchmarks.datasetsrrrr r r r r rrrrrrrvllm.benchmarks.lib.utilsrrvllm.engine.arg_utilsrr vllm.inputsrrvllm.lora.requestr vllm.outputsrvllm.sampling_paramsrvllm.tokenizersrrvllm.utils.async_utilsr listrbooltuplefloatrdrfrvrr NamespacerSrrrrArgumentParserrr*r7r7r7r8s  D     X 6 P D  5