o پi(@sdZddlZddlZddlZddlZddlZddlmZddlZddl m Z e e Z dZ dZdZgdd d gd Zd d giZeeddZdddZdddZdefddZdeeddfddZddZe dkrpedSdS)a  Test runner for multimodal_gen that manages test suites and parallel execution. Usage: python3 run_suite.py --suite --partition-id --total-partitions Example: python3 run_suite.py --suite 1-gpu --partition-id 0 --total-partitions 4 N)Path) init_logger test_update_weights_from_disk.py SGLANG_MMGEN_UPDATE_WEIGHTS_PAIR)zFLUX.2-klein-base-4Bz Qwen-Image)ztest_server_a.pyztest_server_b.pyztest_lora_format_adapter.pyz ../cli/test_generate_t2i_perf.pyz#../test_sampling_params_validate.pyrztest_server_2_gpu_a.pyztest_server_2_gpu_b.py)z1-gpuz2-gpuz1-npuzascend/test_server_1_npu.pycCstjdd}|jdtdttdd|jdtdd d |jd td d d |jdtddd |jddtddd |jddddd|S)NzRun multimodal_gen test suite) descriptionz--suiteTz*The test suite to run (e.g., 1-gpu, 2-gpu))typerequiredchoiceshelpz--partition-idrz7Index of the current partition (for parallel execution))rdefaultr z--total-partitionszTotal number of partitionsz --base-dirserverz9Base directory for tests relative to this script's parent-kz--filterz.Pytest filter expression (passed to pytest -k)z--continue-on-error store_trueFzlContinue running remaining tests even if one fails (for CI consistency; pytest already continues by default))actionr r ) argparseArgumentParser add_argumentstrlistSUITESkeysint parse_args)parserrX/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/test/run_suite.pyr<sP  rcCsBtjddddg}|r|d|g||tdd|tj|ddd }|jd vr[d |jd d|d }|jrF|d|jd 7}|j rR|d|j d 7}t |t ||jdkrdtdg}|j d D]&}| }|rd|vr|dsd|vr|dn|}d|vr||qntdt|d|S)zLCollect test item node IDs from the given files using pytest --collect-only.-mpytestz--collect-onlyz-qrzCollecting tests with command:  T)capture_outputtext)rz,pytest --collect-only failed with exit code z Command:  zstderr: zstdout: r"zINo tests were collected (exit code 5). This may be expected with filters.z::)=-rrz Collected z test items)sys executableextendprintjoin subprocessrun returncodestderrstdoutloggererror RuntimeErrorstripsplit startswithappendlen)files filter_exprcmdresult error_msg test_itemslinetest_idrrrcollect_test_itemsgs>     r@cCs|stddStjddddg}|r|d|gd}t|d D]}t|}|dkr0|d |||dkrDtd |d |d tdd|tj |tj tj dd}t } |j d}|sfntj j|tj j||q]||j} | dkrdS| dkrtddS|jddd} d| vod| v} d| vpd| v} d| vpd| v} | s| s| s| Sq!td| S)NzNo files to run.rrrz-sz-vrr z --last-failedz(Performance assertion failed. Retrying (/z) with --last-failed...zRunning command: r)r/r.bufsizeTir"zwNo tests collected (exit code 5). This is expected when filters deselect all tests in a partition. Treating as success.zutf-8replace)errorsz/multimodal_gen/test/server/test_server_utils.pyAssertionErrorSafetensorErrorFileNotFoundErrorz out of memoryz oom killerzMax retry exceeded)r)r&r'r(rangerr6r*r+PopenPIPESTDOUT bytearrayr/readbufferwriteflushwaitr-decodelower)r8r9base_cmd max_retriesir:process output_byteschunkr- full_outputis_perf_assertionis_flaky_ci_assertion is_oom_errorrrr run_pytestsn        r_returncCstjdddvS)NSGLANG_IS_IN_CI)1trueyeson)osenvirongetrTrrrr _is_in_cisrjsuite_files_relcCsntsdSt|vr dStjtr tdtdtjtdStt }|tjt<tdtd|ddS)Nz Using preset r$ Selected z for this CI run) rj#_UPDATE_WEIGHTS_FROM_DISK_TEST_FILErgrhri_UPDATE_WEIGHTS_MODEL_PAIR_ENVr)randomchoice_UPDATE_WEIGHTS_MODEL_PAIR_IDS)rk selected_pairrrr$_maybe_pin_update_weights_model_pairs   rscsrttt}|j}|j}|s"td|dt dt j }t |g}|D]}||}|sEtd|d|dq/| t|q/|s]tdj dt d t|jd }|sttd j dt d fd d t|D}jddjdjd} ddg} j | gg} tj| | ddd} | dt|d7} |D] } | d| d7} qt| ddtdj djdjtdt|d|D] }tdtj|q|stdt d td t|d!d"|t|}dtj| | ddd} | d#t|d7} |D] } | d| d7} q t| ddt |dS)$NzError: Target directory z does not exist.r zWarning: Test file z not found in z . Skipping.z%No valid test files found for suite 'z'.r)r9zNo test items found for suite 'cs$g|]\}}|jjkr|qSr)total_partitions partition_id).0rWitemargsrr &s zmain..rBz (0-based id=)Suite Partitionpsql)headerstablefmtr#u ✅ Enabled z test(s): z - T)rQzSuite: z | Partition: rlz files:z5No items assigned to this partition. Exiting success.zRunning z items in this shard: z, u ✅ Executed )rr__file__resolveparentbase_direxistsr)r&exitrsuitersr6rr@filter enumeraterurttabulater7rgpathbasenamer*r_)current_file_path test_root_dir target_dirrksuite_files_absf_relf_absall_test_itemsmy_itemspartition_inforrowsmsgrwf exit_coderrxrmainsf            r__main__)N)__doc__rrgror+r&pathlibrr1sglang.multimodal_gen.runtime.utils.logging_utilsr__name__r0rmrnrqr suites_ascendupdaterr@r_boolrjrrrsrrrrrs@      + 3SM