o ۾i9 @sdZddlZddlZddlZddlZddlmZddlmZddlZ ddl m Z m Z ddl mZddlmZddlmZdd lmZdd lmZzddlZWn ey[ed ZYnw d"d ed edeeeeffddZdeeeefdeedeeeeefffddZ ddZ!dej"deeeffddZ#dej$ddfddZ%dej"ddfddZ&e'dkrej$d d!Z(e%e(e()Z*e&e*dSdS)#a?Benchmark multimodal processor latency. This benchmark measures the latency of the mm processor module using multimodal prompts from datasets. MM processor stats are automatically enabled. Run: vllm bench mm-processor \ --model \ --dataset-name random-mm \ --num-prompts 10 \ N)datetime)Any)MultiModalConversationDatasetVisionArenaDataset) get_requests EngineArgs)#get_timing_stats_from_engine_client)freeze_gc_heap)PlaceholderModulepandas llm_enginenum_warmup_reqsreturncCsdt|}gd}dd|D}t||d}|D]}|D]}||vr.||||qq|S)z Collect multimodal processor timing stats. Returns a dictionary mapping stage names to lists of timing values (in seconds). )hf_processor_time hashing_timecache_lookup_timeprompt_update_timepreprocessor_total_timeencoder_forward_timenum_encoder_callscSsi|]}|gqSr).0keyrrP/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/mm_processor.py >z.collect_mm_processor_stats..N)r listvaluesappend)r r all_stats stat_keysstats_by_stage stats_list stats_dictrrrrcollect_mm_processor_stats+s r%r"selected_percentilescsi}|D]E\}}|sdddddd|D||<q|dk}|r$|ndd|Dttttttdfdd|D||<q|S) z: Calculate aggregate metrics from stats by stage. )meanmedianstdcSsi|]}d|dqS)pr'rrr+rrrrZsz2calculate_mm_processor_metrics..rcSsg|]}|dqS)r)rtrrr _z2calculate_mm_processor_metrics..cs$i|]}d|tt|qSr+floatnp percentiler,rrrres$)itemsr3r4r(r)r*)r"r&metrics stage_nametimesis_count_metricrr6rcalculate_mm_processor_metricsKs&     r<cCst|dds |j|_t|dsd|_t|dsd|_t|ds"d|_|jdkr.|js.td|jdkrLt j t j B}|j|vrNt|jdt |dSdS) zE Validate command-line arguments for mm_processor benchmark. tokenizerN dataset_path lora_path max_lorashfz--dataset-path is required when using --dataset-name hf. For multimodal benchmarking, specify a dataset like 'lmarena-ai/VisionArena-Chat'.zK is not a supported multimodal dataset. Supported multimodal datasets are: )getattrmodelr=hasattrr>r?r@ dataset_name ValueErrorrSUPPORTED_DATASET_PATHSkeysrsorted)argssupported_mm_datasetsrrr validate_argsks2      rLrJc# sXddlm}mt||jdurd|_t|}|d'it| }t ||}t fdd|Ds;Jddd|D}d d|D}fd d|D}d dt |d d  dD}tt |dd} | dkrtd| dtjd'it|} | | _| jd7_t | |} dd| D} dd| D} fdd| D}j| |t |dd dtdt|dt}j||t |dd d}t}||}tj| }t|stdt||}tdd|D}t||}g|D]@}|jr |jdur q|j}t |dddur=t |dddur=t |dddur=|j}t d|j!|j"}#||d qsU|dkrUtd!||}|d g|rwt$t%&}t$t%'}t$t%(}fd"d|D}n d}d}d}d#d|D}i} d$|vr|d$r|d$}!t)t*|!t|!d%} |||||||| d&}"|"S)(z1 Run the multimodal processor benchmark. r)LLMSamplingParamsNc3s&|]}jjj|j|jkVqdSN)r model_config max_model_len prompt_lenexpected_output_lenrrequest)llmrr s  z1benchmark_multimodal_processor..zpPlease ensure that max_model_len is greater than the sum of prompt_len and expected_output_len for all requests.cSg|]}|jqSrpromptrTrrrr/rz2benchmark_multimodal_processor..cSrXrrSrTrrrr/rcsg|] }dd|ddqS)r'T)n temperature max_tokens detokenizerr output_lenrNrrr/scSg|]}t|qSrr3r,rrrr/metric_percentiles99, num_warmupsz Processing z warmup requests...r\cSrXrrYrreqrrrr/rcSrXrr[rkrrrr/rcsg|]}|dqS))r_rrarcrrr/s  disable_tqdmF)use_tqdmz requests...u ⚠️ Warning: No MM processor stats found in registry. This may indicate that: - No multimodal requests were processed - Stats were already retrieved (registry is cleared after retrieval) cSsg|]}|jr|qSr)finished)rorrrr/sfirst_token_latency last_token_tsfirst_token_tsr'r-u ⚠️ Warning: Detailed end-to-end latency metrics not available. Falling back to average request latency (total_time / num_completed_requests). cs g|] }|tt|fqSrr2r,) e2el_timesrrr/scSsg|]}|dfqS)r'rr,rrrr/ r0r)total_encoder_callsnum_requests_with_encoder_calls) completedfailed mean_e2el_msmedian_e2el_ms std_e2el_mspercentiles_e2el_msmm_processor_statsencoder_summaryr)+vllmrMrNrLseedr from_cli_args dataclassesasdict get_tokenizerrallrBsplitr printargparse Namespacevars num_promptschatlentime perf_counterr%r anyrr<ror8rqmaxrrrsrr3r4r(r)r*intsum)#rJrM engine_argsr=requestspromptsexpected_output_lenssampling_paramsr&rj warmup_argswarmup_requestswarmup_promptswarmup_output_lenswarmup_sampling_params start_timeoutputsend_time total_timemm_stats_by_stagemm_processor_metricsrwrxoutputr8ttft decode_timeavg_time_per_requestryrzr{r|r~ encoder_callsbenchmark_resultr)rNrtrVrbenchmark_multimodal_processors                   rparsercCsddlm}|||jdd|jdtdddgdd |jd td d d |jdtddd ddlm}m }|||||jdtddd |jdtddd |jdtddd |jdtddd |jdtddd |jdtddd |jd d!d"d#dS)$z9Add CLI arguments for the multimodal processor benchmark.rrT)enable_mm_processor_statsz--dataset-namez random-mmrAz=Name of the dataset to benchmark on. Defaults to 'random-mm'.)typedefaultchoiceshelpz --num-prompts zNumber of prompts to process.)rrrz --num-warmupsr\z$Number of warmup prompts to process.)add_random_dataset_base_args"add_random_multimodal_dataset_argsz--dataset-pathNzlPath to the dataset file or HuggingFace dataset name (e.g., 'yale-nlp/MMVU', 'lmarena-ai/VisionArena-Chat').z --hf-subsetz-Subset of the HuggingFace dataset (optional).z --hf-splitzGSplit of the HuggingFace dataset (e.g., 'train', 'test', 'validation').z --output-lenzVOutput length for each request. Overrides the default output lengths from the dataset.z --output-jsonz2Path to save the benchmark results in JSON format.z--metric-percentilesrhzDComma-separated list of percentiles to calculate (e.g., '50,90,99').z--disable-tqdm store_truezDisable tqdm progress bar.)actionr) vllm.engine.arg_utilsr add_cli_args set_defaults add_argumentstrrvllm.benchmarks.datasetsrr)rrrrrrrr&s    rcstdt|}tdtdtdd|vrtdddt|d d d D}g}|dD]=\}}|d k}|r>d nd}|||dd|dd|ddd}|D]|ddd|d<qW||q2t|} t| j ddd|vr|dr|dd} |dd} td| d| dd|vrtd d!dt|d d d D}d"|ddd#d$|d%dd#d&|d'dd#g} |D]t fd(d)|d*Dd} | d| dd#qt| }t|j dd|j rA|j |j t|d+d,t|d-d,d.|d/<t|d0<t|j d1}tj||d2d3Wd,n 1s2wYtd4|j d,Sd,S)5z8Main entry point for the multimodal processor benchmark.z*Starting multimodal processor benchmark...zQ ================================================================================z&Multimodal Processor Benchmark ResultszP================================================================================r}z MM Processor Metrics:cSrdrrer,rrrr/rfzmain..rgrhrirz (ms)r(z.2fr)r*)StageMeanMedianStdr+r'PF)indexr~rurvz Summary: z total encoder calls across z requests.ryz End-to-End Latency (ms):cSrdrrer,rrrr/rfr)Metricz Value (ms)rrzrr{c3s |] \}}|kr|VqdSrOr)rpctvalr1rrrWszmain..r|random_input_lenNrandom_output_len)rCr input_lenrbconfig timestampw)indentz Results saved to )rrrBrr7getrpd DataFrame to_stringnext output_jsonrCrrnow isoformatopenjsondump)rJresultr&mm_datastager8is_countunitrowmm_df total_calls num_requests e2el_datapercentile_valuee2el_dffrr1rmainys    "        r__main__zBenchmark mm processor latency) description)r)+__doc__rrrrrtypingrnumpyr4rrrvllm.benchmarks.throughputrrr"vllm.multimodal.processing.contextr vllm.utils.gc_utilsr vllm.utils.import_utilsr r r ImportErrorrdictrrr3r%r<rLrrArgumentParserrr__name__r parse_argsrJrrrrsb                 SR