o پi'@sdZddlZddlZddlZddlZddlmZmZmZm Z ddZ de dede fd d Z d d Zd dgZedkreZejde dddejde dddejdeddejde ddejdedddejde dd ejd!ed"ejd#ed$d ejd%ed&d ejd'ed(d ejd)ed*d ejd+edd,dejd-e d"ejd.de ed/d0ejd1e d2d3dejd4e dd5dejd6ed7dejd8ed9dejd:ed;ddeZeedSdS)?zX Usage: python3 -m sglang.test.run_eval --port 30000 --eval-name mmlu --num-examples 10 N)ChatCompletionSamplerEval make_report set_ulimitcCs6t|dd}|tvr|dkrd}nd}d|diiSiS)N thinking_mode deepseek-v3thinkingenable_thinkingchat_template_kwargsT)getattrTHINKING_MODE_CHOICES)argsrthinking_paramrH/home/ubuntu/.local/lib/python3.10/site-packages/sglang/test/run_eval.pyget_thinking_kwargss rbase_urleval_objreturnc Cst|}|r |ni}t|dd}|dur||d<t|jt|ddt|dd|t|ddt|dd|r6|ndd }t}||}t|} || |fS) Ntop_k max_tokenstop_p? temperaturereasoning_effort)modelrrrrr extra_body)rcopyr rrtime perf_counter) r rrthinking_kwargsrrsamplerticresultlatencyrrr run_eval_once!s$         r'c sddlm}tdtjvrdtjd<jrjdn djdjdjdkr=dd l m }d }||j j njd krYdd l m}td d}d}|||j j nֈjdkrlddlm}|j j nÈjdkrddlm}|j j dgdnjdkrddlm}d}||j j njdkrddlm}|j j njdkrddlm} j} jrˆjdnd} | j| j j | tddtdddnTjd krdd!lm} | j j td"dd#n;jd$krdd%lm } | j j n'jd&kr'dd'l!m"}|j j td(d)td*dd+nt#d,jtd-d.d.krt$\}}}|j%d/|j&iB}t'd0|d1d2t'd3|d/d1|jd4|d/|jjd5d6|jd7||jjd5d6n}dd8l(m)}|j*d9fd:d;t+j*D}g}|D]}|,\}}}|-|j&qt.|t/|}dj*d?|d1t'd@|t'd=|j%dA|iB}|dB|iB}|jdC||jjj*dDd60jdE|j1dFdE}dG|dH}t'dI|t2|dJ}|3t4|Wdn 1s.wYt'|dG|dK}t2|dJ}|3t5j6|dLdMWdn 1sXwYt'dN|tdOdPro||fS|S)QNr) dump_metricOPENAI_API_KEYEMPTYz/v1zhttp://:mmlu)MMLUEvalz@https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csvmath)MathEvalz gpt-4-turbo)rzEhttps://openaipublic.blob.core.windows.net/simple-evals/math_test.csvmgsm)MGSMEvalmgsm_enen) languagesgpqa)GPQAEvalzHhttps://openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv humaneval) HumanEval longbench_v2)LongBenchV2Eval,max_context_lengthmin_context_length)r data_source num_examples num_threads categoriesr<r=mmmu) MMMUVLMEvalresponse_answer_regex)rDaime25) AIME25Evalgsm8k) GSM8KEval num_shotsgsm8k_data_path)r?r@rI data_pathzInvalid eval name: repeatscorezTotal latency: .3fz szScore: _score)reval)labels_latency)ThreadPoolExecutor) max_workerscsg|] }tqSr)submitr').0_r rrexecutorrr szrun_eval..cSsg|]}|dqS)rPr)rXsrrrr\sz====================zRepeat: z, mean: zScores: scores mean_score _mean_score)rrRrMrY/z/tmp/z.htmlzWriting report to wz.json)indentzWriting results to return_latencyF)7sglang.test.test_utilsr(rosenvironrhostport eval_namesglang.test.simple_eval_mmlur-r?r@sglang.test.simple_eval_mathr/rsglang.test.simple_eval_mgsmr1sglang.test.simple_eval_gpqar6!sglang.test.simple_eval_humanevalr8$sglang.test.simple_eval_longbench_v2r: dataset_pathrAsplitrr sglang.test.simple_eval_mmmu_vlmrCsglang.test.simple_eval_aime25rFsglang.test.simple_eval_gsm8krH ValueErrorr'metricsrOprintconcurrent.futuresrUrMranger%appendsumlenshutdownreplaceopenwriterjsondumps)r r(r-filenamer/equality_checkerr1r6r8r:r>rArCrFrHr%r&r#rxrUfutures scores_repeatfr_ file_stemreport_filenamefhresult_filenamerrZrrun_eval=s   &                                       rrqwen3__main__z --base-urlz7Server or API base url if not using http host and port.)typedefaulthelpz--hostz0.0.0.0zDefault host is 0.0.0.0.z--portznIf not set, the default port is configured according to its default value for different LLM Inference Engines.)rrz--modelzZName or path of the model. If not set, the default model will request /v1/models for conf.z--repeatrNzrepeat the evaluation n timesz --eval-namer,)rrz--num-examples)rz --num-threadsiz --max-tokensrz --temperaturerz--top-prz--top-kzTop-k sampling parameterz--reasoning-effortz--thinking-modezmEnable thinking mode in Deepseek V3.1/3.2, or Qwen3.--reasoning-parser must be set when launching the server.)rrchoicesrz--dataset-pathzTHUDM/LongBench-v2zAPath to dataset file or HuggingFace dataset name for LongBench-v2z --categoriesz?Comma-separated list of categories to evaluate for LongBench-v2z--max-context-lengthz5Maximum context length in characters for LongBench-v2z--min-context-lengthz5Minimum context length in characters for LongBench-v2z --num-shotsrJz2Number of few-shot examples for GSM8K (default: 5)z--gsm8k-data-pathz*Path to GSM8K data file (e.g., test.jsonl))__doc__argparserrgr sglang.test.simple_eval_commonrrrrrstrdictr'rr __name__ArgumentParserparser add_argumentintfloat parse_argsr rrrrs"