o پi&A@sddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZmZmZmZdZded efd d Zd d Zddeeded dfddZGdddZGdddeZeZdS)N)Path)SimpleNamespace) temp_set_env)kill_process_tree)!DEFAULT_TIMEOUT_FOR_SERVER_LAUNCHDEFAULT_URL_FOR_TESTCustomTestCasepopen_launch_serverg? error_outputreturncCs d|vod|vod|vpd|vS)z6Check if error is due to MMMU parquet file corruption. ArrowInvalidzParquet magic bytes not foundMMMUzlmms-lab--MMMU)r rrQ/home/ubuntu/.local/lib/python3.10/site-packages/sglang/test/kits/mmmu_vlm_kit.py_is_mmmu_parquet_corruptions rc Cstd}|r |}ntjdtjd}t|dd}|rVtd|zt |td|WdSt yU}ztd |d |WYd }~d Sd }~wwtd |dd S)z>Clean up corrupted MMMU dataset cache to allow fresh download.z%/hf_home/hub/datasets--lmms-lab--MMMUHF_HOMEz~/.cache/huggingfacehubzdatasets--lmms-lab--MMMUz4Detected corrupted MMMU parquet cache. Cleaning up: z&Successfully removed corrupted cache: Tz Warning: Failed to remove cache : NFzMMMU cache not found at z, skipping cleanup) rexistsosenvirongetpath expanduserprintshutilrmtreeOSError) ci_hf_homemmmu_cache_pathhf_homeerrr_cleanup_mmmu_dataset_cache s$ r"cmdtimeoutc Csztj|d|ddd}|j|j}t|rjtdtrZtdtddd&tj|d|ddd}|jr;t|jdd |jrEt|jdd Wd n 1sOwYWd SWd Std |jd |jtd |jrtt|jdd |jrt|jdd Wd SWd Stj y}zk|j|j}t|rtdtrtdtddd&tj|d|ddd}|jrt|jdd |jrt|jdd Wd n1swYntd|jd |jtd|jd |jWYd }~d Sd }~ww)zFRun lmms_eval command with automatic retry on MMMU parquet corruption.T)checkr%capture_outputtextzHDetected MMMU parquet corruption error in output. Attempting recovery...z)Retrying lmms_eval with fresh download...0force_redownload)HF_HUB_OFFLINEHF_DATASETS_DOWNLOAD_MODE)endNzGFailed to cleanup corrupted MMMU cache. Output from lmms_eval: Stdout: z Stderr: z&Failed to cleanup corrupted MMMU cachez>Detected MMMU parquet corruption error. Attempting recovery...zFFailed to cleanup corrupted MMMU cache. Error from lmms_eval: Stdout: z2lmms_eval failed with an unhandled error. Stdout: ) subprocessrunstdoutstderrrrr"r RuntimeErrorCalledProcessError)r$r%resultcombined_output retry_resultr!r rrr_run_lmms_eval_with_retry9s  (    r8c@sReZdZUdZeed<gZeeed<dZ de dedefdd Z de fd d Z d S) MMMUMixinzMixin for MMMU evaluation. Use with MMMUServerBase for single-model tests: class TestMyModel(MMMUMixin, MMMUServerBase): model = "my/model" accuracy = 0.4 accuracy mmmu_args sk-123456self model_version output_pathc Csd}d}d}d}d}tj|ddd|d|}d d d d |d |d|dt|dd|dt|g|j} t|j|jdd t| WddS1sMwYdS) Evaluate a VLM on the MMMU validation set with lmms-eval. Only `model_version` (checkpoint) and `chat_template` vary; We are focusing only on the validation set due to resource constraints. openai_compatiblemmmu_val@Texist_okmodel_version="",tp=python3-m lmms_eval--model --model_args--tasks --batch_size --log_samples--log_samples_suffix --output_path/v1)OPENAI_API_KEYOPENAI_API_BASEN)rmakedirsstrr;rapi_keybase_urlr8) r=r>r?modeltptasks batch_size log_suffix model_argsr$rrr run_mmmu_evals@   "zMMMUMixin.run_mmmu_evalc Cst|}||j|tj|ddd}|s t|d}|s)td||d}t|d}t|}t d|Wd n1sIwY|d d d }t d |jd|d| ||j d |jd|dd|j ddWd d S1swYd S)zRun MMMU evaluation test. /**/*.jsonT recursive/*.jsonNo JSON result files found in rrzResult: NresultsrC mmmu_acc,noneModel z achieved accuracy: .4f accuracy () below expected threshold ()) tempfileTemporaryDirectoryr`rZglobFileNotFoundErroropenjsonloadrassertGreaterEqualr:)r=r? result_filesresult_file_pathfr5 mmmu_accuracyrrr test_mmmus(   "zMMMUMixin.test_mmmuN) __name__ __module__ __qualname____doc__float__annotations__r;listrWrXrr`rzrrrrr9s  2r9c@sneZdZdZdZgZgZeddZeddZ ddde d e d e dBfd d Z   dddZ ddZdS)MMMUMultiModelTestBaseaBase class for multi-model MMMU tests. This class is for tests that need to evaluate multiple models, starting and stopping a server for each model within the test method. For single-model tests, use MMMUMixin with MMMUServerBase instead. NcCsft|_d|_t|_|jdurttd|_tj d|_ tj d|_ |jtj d<|jdtj d<dS)Nr<)mem_fraction_staticrTrUrS) rrYrXrtime_out parsed_argsrDEFAULT_MEM_FRACTION_STATICrrr_original_openai_api_key_original_openai_api_baseclsrrr setUpClasss  z!MMMUMultiModelTestBase.setUpClasscCs^|jdur |jtjd<n dtjvrtjd=|jdur"|jtjd<dSdtjvr-tjd=dSdS)NrTrU)rrrrrrrr tearDownClasss     z$MMMUMultiModelTestBase.tearDownClass)envr>r?rc Cspd}d}d}d}d}tj|ddd|d|} d d d d |d | d|dt|dd|dt|g|j} t| dS)r@rArBrCrDTrErGrHrIrJrKrLrMrNrOrPrQrRN)rrVrWr;r8) r=r>r?rrZr[r\r]r^r_r$rrrr`s6  z$MMMUMultiModelTestBase.run_mmmu_evalr-infoFcCstd|j|d}d}d} zzEtj} |r | |d| d<d} d} |r4tdd} td d} t|j|j|j |j d d d d dt |j j d|g|j| |rT| | fndd}||j|tj|ddd} | sst|d} | s|td|| d}t|d}t|}td|d|Wdn1swY|ddd}td|jd|d|d|r|r|} |||jd|jd |dd!|jdd"|| WW|dur|durtd#|jzt|jWnty}z td$|WYd}~nd}~ww|rW| r| | r%| d%D]0}ztj|r6t|Wq'tyV}ztd&|d|WYd}~q'd}~wwSSty}z!td'|j|d||d(|j|d|WYd}~nd}~wwW|dur|durtd#|jzt|jWnty}z td$|WYd}~nd}~ww|r| r| | r| d%D]1}ztj|rt|Wqty}ztd&|d|WYd}~qd}~wwdSdS|dur;|dur;td#|jzt|jWnty:}z td$|WYd}~nd}~ww|r~| rE| | rL| d%D]0}ztj|r]t|WqNty}}ztd&|d|WYd}~qNd}~wwww))a Common method to run VLM MMMU benchmark test. Args: model: Model to test output_path: Path for output logs test_name: Optional test name for logging custom_env: Optional custom environment variables log_level: Log level for server (default: "info") capture_output: Whether to capture server stdout/stderr z Testing model: Nrr-1SGLANG_USE_CUDA_IPC_TRANSPORT/tmp/server_stdout.logw/tmp/server_stderr.logz--trust-remote-codez--cuda-graph-max-bs64z--enable-multimodalz--mem-fraction-staticz --log-level)rYr%rX other_argsrreturn_stdout_stderrraTrbrdrerfResultz : rgrCrhriz achieved accuracyrrjrkrlrmzCleaning up process zError killing process: )rrzError removing zError testing zTest failed for ) rrZrrcopyupdaterrr rYrrXrWrrrr`rprqrsrt_read_output_from_filesrurypollpidr Exceptioncloserrremovefail)r=rZr? test_name custom_env log_levelr'processry server_output process_env stdout_file stderr_filervrwrxr5r!filenamerrr_run_vlm_mmmu_test>s          " "( " "z)MMMUMultiModelTestBase._run_vlm_mmmu_testc Csg}ddg}|D]P\}}z.tj|r9t|d}|D]}||d|qWdn1s4wYWqtyX}ztd|d|WYd}~qd}~wwd |S)N)rz[STDOUT])rz[STDERR]rf zError reading z file:  ) rrrrrappendrstriprrlowerjoin)r= output_lines log_filesrtagrxliner!rrrrs$   $ z.MMMUMultiModelTestBase._read_output_from_files)r-NrF)r{r|r}r~rrr; classmethodrrrWdictr`rrrrrrrs.   2 r)r#)rprsrrr/rnpathlibrtypesrsglang.srt.environrsglang.srt.utilsrsglang.test.test_utilsrrrr rrWboolrr"rintr8r9rMMMUVLMTestBaserrrrs&     Hbo