o i5@s\ddlZddlZddlZddlZddlmZddlmZddlmZddl m Z ddl m Z ddl mZmZdd lmZdd lmZzddlZWn eyUe d ZYnwejd d deedeededededef ddZdeeefdededefddZdedBdeedededededefddZ d ed!ed"efd#d$Z!d%ededBfd&d'Z"d!ed(ed efd)d*Z#dedBdeed!ed"ed%ed+edefd,d-Z$d d deedeedeeded.ed/ed ed+eded0ee%eefdefd1d2Z&eGd3d4d4Z'd5e'fd6d7Z(d5ej)fd8d9Z*e+d:kr,ej,e'j-d;Z.e'/e.e*e.0dSdS)<N) dataclass)datetime)Path)ClassVar)PlaceholderModule)ParameterSweepParameterSweepItem) ServerProcess)sanitize_filenamepandas,)server_ready_timeout serve_cmdafter_bench_cmd show_stdoutserve_overridesdry_runrccs||}tdtd|td||r#dVtddSt|||d}|j|d|VWdn1s>wYtddS)Nz[BEGIN SERVER]zServer overrides: zServer command: z [END SERVER])r)timeout) apply_to_cmdprintr wait_until_ready)rrrrrr server_cmdserverrQ/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/benchmarks/sweep/serve.py run_servers   rrun_databench_overrides run_numbercCs ||d<|||||S)Nr)update)rrrrrrr_update_run_data4s  r!r bench_cmd output_pathc Csg||ddddt|jd|j}tdtd|td|td |td ||rbtd |d }t|} t | |||WdS1s]wY|duru|sot d |tddS|jj ddd| || |d  }t|} Wdn1swYt | |||} |d}tj| |ddWdn1swYtd| S)Nz--percentile-metricszttft,tpot,itl,e2elz --save-resultz --result-dirz--result-filenamez[BEGIN BENCHMARK]zBenchmark overrides: z Run Number: zBenchmark command: z Output file: z!Found existing results. Skipping.rbzCannot find results at z[END BENCHMARK]T)parentsexist_okwindent)rstrparentnamerexistsopenjsonloadr! ValueErrormkdirrun_subcommand after_benchdump) rr"rrrr#r benchmark_cmdfrrrr run_benchmarkAsn          r9 output_dir serve_comb bench_combcCsDtt}|r|d|jf|r|d|jf|td|S)NzSERVE-zBENCH--)listr+extendr-r join)r:r;r<partsrrr_get_comb_base_paths rB base_pathcCs |dur|dS|d|dS)Nz summary.jsonzrun=z.jsonr)rCrrrr_get_comb_run_pathsrD bench_combscCs0|D]}t|||}t|ddsdSqdS)NrTF)rBrDr.)r;rEr:r<rCrrr_comb_needs_servers  rGnum_runsc Csttttf}t|D]}t|||||t|||d} | dur&|| q |r+dSt|ddd} t j || ddWd|S1sHwY|S)N)rrrr#rrFr'r(r)) r>dictr+objectranger9rDappendr/r0r6) rr"r;r<rCrHr comb_datarrr8rrrrun_combs,   rN serve_params bench_paramslinksc sttttf} |D]Tt||rt||||| dnt6} |D]+tfdd| D} | s5q$t |}t | ||||d}|durO| |q$Wdn1sZwYq |rddSt j | }||d|S)N)rrrrc3s4|]\}}|vo|vo||kVqdSNr).0 serve_key bench_keyr<r;rr s zrun_combs..)r;r<rCrHrz summary.csv)r>rIr+rJrGr contextlib nullcontextallrBrNr?pd DataFrame from_recordsto_csv)rr"rrrOrPr:rHrrQrall_datar should_runrCrM combined_dfrrVr run_combssR     $ rbc@seZdZUeeed<eeed<eeed<eed<eed<eed<eed<e ed<eed <ed Bed <ee eefd Bed <e ed <dZ e eed<dZ e eed<edejfddZedejdejfddZededee eeffddZd S)SweepServeArgsrr"rrrOrPr:rHrNresume link_varsrserve parser_namez2Run vLLM server benchmark under multiple settings. parser_helpargsc Cst|j}t|j}|jdurgnt|j}|jr#t|j}ntig}|j r3t|j }ntig}| |j }|j }|dkrJt d|||||j||t|j||j|j||jd S)Nrz `num_runs` should be at least 1.) rr"rrrOrPr:rHrrdrer)shlexsplitrr"rrOr read_jsonr]rPparse_link_varsrerHr2rrr:rrdr) clsrirr"rrOrPrerHrrr from_cli_argss8     zSweepServeArgs.from_cli_argsparserreturncCs|jdtddd|jdtddd|jdtddd |jd d d d |jdtddd |jdtddd |jdtddd |jddtddd |jdtddd |jdd dd |jdtddd |jd td!d"d |S)#Nz --serve-cmdTz4The command used to run the server: `vllm serve ...`)typerequiredhelpz --bench-cmdz=The command used to run the benchmark: `vllm bench serve ...`z--after-bench-cmdzlAfter a benchmark run is complete, invoke this command instead of the default `ServerWrapper.clear_cache()`.)rrdefaultrtz --show-stdout store_truez^If set, logs the standard output of subcommands. Useful for debugging but can be quite spammy.)actionrtz--server-ready-timeoutr z:Timeout in seconds to wait for the server to become ready.z--serve-paramsaPath to JSON file containing parameter combinations for the `vllm serve` command. Can be either a list of dicts or a dict where keys are benchmark names. If both `serve_params` and `bench_params` are given, this script will iterate over their Cartesian product.z--bench-paramsa Path to JSON file containing parameter combinations for the `vllm bench serve` command. Can be either a list of dicts or a dict where keys are benchmark names. If both `serve_params` and `bench_params` are given, this script will iterate over their Cartesian product.z-oz --output-dirresultsz+The directory to which results are written.z --num-runsz)Number of runs per parameter combination.z --dry-runzFIf set, prints the commands to run, then exits without executing them.z--resumezSet this to the name of a directory under `output_dir` (which is a timestamp) to resume a previous execution of this script, i.e., only run parameter combinations for which there are still no output files.z --link-varszComma-separated list of linked variables between serve and bench, e.g. max_num_seqs=max_concurrency,max_model_len=random_input_len) add_argumentr+int)rnrprrr add_cli_argsCs    zSweepServeArgs.add_cli_argsscCsD|sgSg}|dD]}|d\}}|||fq |S)N,=)rkrLstrip)r~pairsitemabrrrrmszSweepServeArgs.parse_link_vars)__name__ __module__ __qualname__r>r+__annotations__boolrrr|tuplergrrh classmethodargparse NamespaceroArgumentParserr} staticmethodrmrrrrrc s*    &\$rcric Cs|jp td}|j|}|jr|std|dzt|j|j |j |j |j |j ||j|j|j|jd WStyM}z td|d|d}~ww)Nz %Y%m%d_%H%M%Sz+Cannot resume from non-existent directory ()) rr"rrrOrPr:rHrrQrz/The script was terminated early. Use `--resume z2` to continue the script from its last checkpoint.)rdrnowstrftimer:r.r2rbrr"rrrOrPrHrrer BaseException RuntimeError)ri timestampr:excrrrrun_mains4   rcCstt|dSrR)rrcro)rirrrmainsr__main__) description)1rrXr0rj dataclassesrrpathlibrtypingrvllm.utils.import_utilsr param_sweeprr rr utilsr r r[ ImportErrorcontextmanagerr>r+rr|rrIrJr!r9rBrDrGrNrrbrcrrrrrrhrpr} parse_argsrrrrs               G    /    =!