EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

ci: update lmms-eval to speed up multimodal CI (#11000)

Browse Source
This commit is contained in:
b8zhong
2025-10-18 11:51:19 -07:00
committed by GitHub
parent c44e985dc2
commit f4f8a1b4d8
3 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/ci/amd_ci_install_dependency.sh
View File

@@ -22,7 +22,7 @@ case "${GPU_ARCH}" in
docker exec ci_sglang rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml
docker exec ci_sglang pip install -e "python[dev_hip]" --no-deps # TODO: only for mi35x
# For lmms_evals evaluating MMMU
docker exec -w / ci_sglang git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
docker exec -w / ci_sglang git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
docker exec -w /lmms-eval ci_sglang pip install -e . --no-deps # TODO: only for mi35x
;;
mi30x|mi300|mi325)
@@ -30,7 +30,7 @@ case "${GPU_ARCH}" in
docker exec ci_sglang rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml
docker exec ci_sglang pip install -e "python[dev_hip]"
# For lmms_evals evaluating MMMU
docker exec -w / ci_sglang git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
docker exec -w / ci_sglang git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
docker exec -w /lmms-eval ci_sglang pip install -e .
;;
*)

2
scripts/ci/ci_install_dependency.sh
View File

@@ -66,7 +66,7 @@ $PIP_CMD install mooncake-transfer-engine==0.3.6.post1 nvidia-cuda-nvrtc-cu12 py
if [ "$IS_BLACKWELL" != "1" ]; then
# For lmms_evals evaluating MMMU
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
$PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX
# Install xformers

12
test/srt/models/test_vlm_models.py
View File

@@ -81,7 +81,7 @@ class TestVLMModels(CustomTestCase):
model = "openai_compatible"
tp = 1
tasks = "mmmu_val"
batch_size = 2
batch_size = 32
log_suffix = "openai_compatible"
os.makedirs(output_path, exist_ok=True)
@@ -179,7 +179,15 @@ class TestVLMModels(CustomTestCase):
self.run_mmmu_eval(model.model, output_path)
# Get the result file
result_file_path = glob.glob(f"{output_path}/*.json")[0]
# Search recursively for JSON result files (lmms-eval v0.4.1+ creates subdirectories)
result_files = glob.glob(f"{output_path}/**/*.json", recursive=True)
if not result_files:
result_files = glob.glob(f"{output_path}/*.json")
if not result_files:
raise FileNotFoundError(f"No JSON result files found in {output_path}")
result_file_path = result_files[0]
with open(result_file_path, "r") as f:
result = json.load(f)