[CI] cleanup single/multi-card test (#5623)

1. speed up e2e light test.
2. create `2-cards` and `4-cards` folder in multicard
3. move ops to nightly
4. run test in Alphabetical Order

- vLLM version: v0.13.0
- vLLM main:
8be6432bda

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

.github/workflows/_e2e_test.yaml

@@ -83,12 +83,8 @@ jobs:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
         if: ${{ inputs.type == 'light' }}
         run: |
-          # pytest -sv --durations=0 tests/e2e/singlecard/test_aclgraph_accuracy.py
-          # pytest -sv --durations=0 tests/e2e/singlecard/test_quantization.py
-          pytest -sv --durations=0 tests/e2e/singlecard/test_aclgraph_mem.py
-          pytest -sv --durations=0 tests/e2e/singlecard/test_camem.py
-          pytest -sv --durations=0 tests/e2e/singlecard/test_vlm.py::test_multimodal_vl
-          pytest -sv --durations=0 tests/e2e/singlecard/pooling/test_classification.py::test_qwen_pooling_classify_correctness
+          pytest -sv --durations=0 tests/e2e/singlecard/test_aclgraph_accuracy.py::test_models_output
+          pytest -sv --durations=0 tests/e2e/singlecard/test_quantization.py::test_qwen3_w8a8_quant
 
       - name: Run e2e test
         env:
@@ -98,33 +94,41 @@ jobs:
         run: |
           # We found that if running aclgraph tests in batch, it will cause AclmdlRICaptureBegin error. So we run
           # the test separately.
-
-          pytest -sv --durations=0 tests/e2e/nightly/single_node/ops/singlecard_ops/triton/
-          pytest -sv --durations=0 tests/e2e/singlecard/test_completion_with_prompt_embeds.py
+          # basic 
           pytest -sv --durations=0 tests/e2e/singlecard/test_aclgraph_accuracy.py
+          pytest -sv --durations=0 tests/e2e/singlecard/test_aclgraph_mem.py
           pytest -sv --durations=0 tests/e2e/singlecard/test_async_scheduling.py
+          pytest -sv --durations=0 tests/e2e/singlecard/test_batch_invariant.py
+          pytest -sv --durations=0 tests/e2e/singlecard/test_camem.py
+          pytest -sv --durations=0 tests/e2e/singlecard/test_completion_with_prompt_embeds.py
+          pytest -sv --durations=0 tests/e2e/singlecard/test_cpu_offloading.py
           # xgrammar has parameter mismatching bug, please follows: https://github.com/vllm-project/vllm-ascend/issues/5524
           # pytest -sv --durations=0 tests/e2e/singlecard/test_guided_decoding.py
           # torch 2.8 doesn't work with lora, fix me
           #pytest -sv --durations=0 tests/e2e/singlecard/test_ilama_lora.py
+          pytest -sv --durations=0 tests/e2e/singlecard/test_models.py
+          pytest -sv --durations=0 tests/e2e/singlecard/test_multistream_overlap_shared_expert.py
           pytest -sv --durations=0 tests/e2e/singlecard/test_profile_execute_duration.py
           pytest -sv --durations=0 tests/e2e/singlecard/test_quantization.py
           pytest -sv --durations=0 tests/e2e/singlecard/test_sampler.py
           pytest -sv --durations=0 tests/e2e/singlecard/test_vlm.py
           pytest -sv --durations=0 tests/e2e/singlecard/test_xlite.py
-          pytest -sv --durations=0 tests/e2e/singlecard/test_models.py
-          pytest -sv --durations=0 tests/e2e/singlecard/pooling/
-          pytest -sv --durations=0 tests/e2e/singlecard/compile/test_norm_quant_fusion.py
-          pytest -sv --durations=0 tests/e2e/singlecard/test_multistream_overlap_shared_expert.py
-          pytest -sv --durations=0 tests/e2e/singlecard/test_cpu_offloading.py
 
-          # ------------------------------------ v1 spec decode test ------------------------------------ #
+          # compile
+          pytest -sv --durations=0 tests/e2e/singlecard/compile/test_norm_quant_fusion.py
+  
+          # model_runner_v2
+          pytest -sv --durations=0 tests/e2e/singlecard/model_runner_v2/test_basic.py
+
+          # pooling
+          pytest -sv --durations=0 tests/e2e/singlecard/pooling/test_classification.py
+          pytest -sv --durations=0 tests/e2e/singlecard/pooling/test_embedding.py
+          pytest -sv --durations=0 tests/e2e/singlecard/pooling/test_scoring.py
+
+          # spec_decode
           pytest -sv --durations=0 tests/e2e/singlecard/spec_decode/test_mtp_eagle_correctness.py
           pytest -sv --durations=0 tests/e2e/singlecard/spec_decode/test_v1_spec_decode.py
 
-          pytest -sv --durations=0 tests/e2e/singlecard/model_runner_v2/test_basic.py
-          pytest -sv --durations=0 tests/e2e/singlecard/test_batch_invariant.py
-
   e2e-2-cards:
     name: multicard-2
     runs-on: linux-aarch64-a3-2
@@ -184,7 +188,7 @@ jobs:
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
         run: |
-          pytest -sv --durations=0 tests/e2e/multicard/test_aclgraph_capture_replay.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py
 
       - name: Install Ascend toolkit & triton_ascend
         shell: bash -l {0}
@@ -200,36 +204,39 @@ jobs:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
         if: ${{ inputs.type == 'light' }}
         run: |
-          pytest -sv --durations=0 tests/e2e/multicard/test_qwen3_moe.py::test_qwen3_moe_distributed_mp_tp2_ep
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_qwen3_moe.py::test_qwen3_moe_distributed_mp_tp2_ep
 
       - name: Run vllm-project/vllm-ascend test (full)
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
         if: ${{ inputs.type == 'full' }}
         run: |
-          pytest -sv --durations=0 tests/e2e/multicard/test_quantization.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_full_graph_mode.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_data_parallel.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_expert_parallel.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_external_launcher.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_single_request_aclgraph.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_data_parallel.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_expert_parallel.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_external_launcher.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_full_graph_mode.py
           # torch 2.8 doesn't work with lora, fix me
-          #pytest -sv --durations=0 tests/e2e/multicard/test_ilama_lora_tp2.py
+          #pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_ilama_lora_tp2.py
+          
 
           # To avoid oom, we need to run the test in a single process.
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_deepseek_multistream_moe_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_qwen3_w4a8_dynamic_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_qwen3_moe_sp_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_qwen3_moe_fc2_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_qwen3_dense_fc1_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_qwen3_dense_prefetch_mlp_weight_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_deepseek_w4a8_accuracy_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_deepseek_v2_lite_fc1_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_deepseek_multistream_moe_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_qwen3_w4a8_dynamic_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_qwen3_moe_sp_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_deepseek_w4a8_accuracy_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_qwen3_moe_fc2_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_deepseek_v2_lite_fc1_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_qwen3_dense_fc1_tp2
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_inference_distributed.py::test_qwen3_dense_prefetch_mlp_weight_tp2
 
-          pytest -sv --durations=0 tests/e2e/multicard/test_prefix_caching.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_pipeline_parallel.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_qwen3_moe.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_weight_load.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_offline_weight_load.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_pipeline_parallel.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_prefix_caching.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_quantization.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_qwen3_moe.py
+          # This test is broken, fix me
+          #pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_shared_expert_dp.py
+          pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_single_request_aclgraph.py
 
   e2e-4-cards:
     name: multicard-4
@@ -300,11 +307,15 @@ jobs:
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
         run: |
-          pytest -sv --durations=0 tests/e2e/multicard/spec_decode/test_mtp_qwen3_next.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_deepseek_multistream_moe_tp2
-          pytest -sv --durations=0 tests/e2e/multicard/test_offline_inference_distributed.py::test_kimi_k2_thinking_w4a16_tp4
-          pytest -sv --durations=0 tests/e2e/multicard/test_data_parallel_tp2.py
-          pytest -sv --durations=0 tests/e2e/multicard/long_sequence/test_basic.py
-          pytest -sv --durations=0 tests/e2e/multicard/long_sequence/test_accuracy.py
-          pytest -sv --durations=0 tests/e2e/multicard/long_sequence/test_mtp.py
-          pytest -sv --durations=0 tests/e2e/multicard/test_qwen3_next.py
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/test_data_parallel_tp2.py
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/test_kimi_k2.py
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/test_qwen3_next.py
+
+          # long_sequence
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/long_sequence/test_accuracy.py
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/long_sequence/test_basic.py
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/long_sequence/test_chunked_prefill.py
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/long_sequence/test_mtp.py
+
+          # spec_decode
+          pytest -sv --durations=0 tests/e2e/multicard/4-cards/spec_decode/test_mtp_qwen3_next.py

docs/source/developer_guide/contribution/testing.md

@@ -231,13 +231,13 @@ VLLM_USE_MODELSCOPE=true pytest -sv tests/e2e/singlecard/test_offline_inference.
 ```bash
 cd /vllm-workspace/vllm-ascend/
 # Run all the single card tests
-VLLM_USE_MODELSCOPE=true pytest -sv tests/e2e/multicard/
+VLLM_USE_MODELSCOPE=true pytest -sv tests/e2e/singlecard/
 
 # Run a certain test script
-VLLM_USE_MODELSCOPE=true pytest -sv tests/e2e/multicard/test_dynamic_npugraph_batchsize.py
+VLLM_USE_MODELSCOPE=true pytest -sv tests/e2e/singlecard/test_aclgraph_accuracy.py
 
 # Run a certain case in test script
-VLLM_USE_MODELSCOPE=true pytest -sv tests/e2e/multicard/test_offline_inference.py::test_models
+VLLM_USE_MODELSCOPE=true pytest -sv tests/e2e/singlecard/test_aclgraph_accuracy.py::test_models_output
 ```
 
 ::::

tests/e2e/multicard/2-cards/test_external_launcher.py

@@ -40,7 +40,7 @@ DEVICE_NAME = torch_npu.npu.get_device_name(0)[:10]
 def test_qwen3_external_launcher(model):
     script = Path(
         __file__
-    ).parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
+    ).parent.parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
     env = os.environ.copy()
     # TODO: Change to 2 when ci machine has 4 cards
     cmd = [
@@ -81,7 +81,7 @@ def test_qwen3_external_launcher(model):
 def test_qwen3_moe_external_launcher_ep_tp2(model):
     script = Path(
         __file__
-    ).parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
+    ).parent.parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
     env = os.environ.copy()
     # TODO: Change to 2 when ci machine has 4 cards
     cmd = [
@@ -112,7 +112,7 @@ def test_qwen3_moe_external_launcher_ep_tp2(model):
 def test_qwen3_external_launcher_with_sleepmode():
     script = Path(
         __file__
-    ).parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
+    ).parent.parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
     env = os.environ.copy()
     # TODO: Change to 2 when ci machine has 4 cards
     cmd = [
@@ -157,7 +157,7 @@ def test_qwen3_external_launcher_with_sleepmode():
 def test_qwen3_external_launcher_with_sleepmode_level2():
     script = Path(
         __file__
-    ).parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
+    ).parent.parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
     env = os.environ.copy()
     model_path = snapshot_download("Qwen/Qwen3-8B")
     # TODO: Add moe model test
@@ -213,7 +213,7 @@ def test_qwen3_external_launcher_with_sleepmode_level2():
 def test_qwen3_external_launcher_with_matmul_allreduce(model):
     script = Path(
         __file__
-    ).parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
+    ).parent.parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
     env = os.environ.copy()
     cmd = [
         sys.executable,

tests/e2e/multicard/2-cards/test_full_graph_mode.py

@@ -16,11 +16,6 @@
 # This file is a part of the vllm-ascend project.
 # Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
 #
-"""Compare the short outputs of HF and vLLM when using greedy sampling.
-
-Run `pytest tests/e2e/multicard/test_qwen3_moe.py`.
-"""
-
 import os
 
 from vllm import SamplingParams

tests/e2e/multicard/2-cards/test_offline_inference_distributed.py

@@ -45,10 +45,6 @@ DEEPSEEK_W4A8_MODELS = [
     "vllm-ascend/DeepSeek-V3.1-W4A8-puring",
 ]
 
-KIMI_W4A16_MODELS = [
-    "vllm-ascend/Kimi-K2-Thinking-Pruning",
-]
-
 
 def test_deepseek_multistream_moe_tp2():
     example_prompts = [
@@ -216,24 +212,3 @@ def test_qwen3_dense_prefetch_mlp_weight_tp2(model):
             quantization="ascend",
     ) as vllm_model:
         vllm_model.generate_greedy(example_prompts, max_tokens)
-
-
-@pytest.mark.parametrize("model", KIMI_W4A16_MODELS)
-def test_kimi_k2_thinking_w4a16_tp4(model):
-    example_prompts = [
-        "Hello, my name is",
-    ]
-    max_tokens = 5
-
-    with VllmRunner(
-            model,
-            max_model_len=8192,
-            dtype="auto",
-            tensor_parallel_size=4,
-            enable_expert_parallel=True,
-            compilation_config={
-                "cudagraph_mode": "FULL_DECODE_ONLY",
-                "cudagraph_capture_sizes": [1],
-            },
-    ) as vllm_model:
-        vllm_model.generate_greedy(example_prompts, max_tokens)

tests/e2e/multicard/2-cards/test_offline_weight_load.py

@@ -34,7 +34,7 @@ MODELS = ["Qwen/Qwen3-30B-A3B"]
 def test_qwen3_offline_load_and_sleepmode_tp2(model):
     script = Path(
         __file__
-    ).parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
+    ).parent.parent.parent.parent.parent / "examples" / "offline_external_launcher.py"
     env = os.environ.copy()
     cmd = [
         sys.executable,

tests/e2e/multicard/2-cards/test_quantization.py

@@ -16,10 +16,6 @@
 # This file is a part of the vllm-ascend project.
 # Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
 #
-"""Compare the short outputs of HF and vLLM when using greedy sampling.
-
-Run `pytest tests/e2e/multicard/test_quantization.py`.
-"""
 from modelscope import snapshot_download  # type: ignore
 
 from tests.e2e.conftest import VllmRunner

tests/e2e/multicard/2-cards/test_qwen3_moe.py

@@ -16,11 +16,6 @@
 # This file is a part of the vllm-ascend project.
 # Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
 #
-"""Compare the short outputs of HF and vLLM when using greedy sampling.
-
-Run `pytest tests/e2e/multicard/test_qwen3_moe.py`.
-"""
-
 import json
 import os
 from unittest.mock import patch

tests/e2e/multicard/4-cards/long_sequence/test_basic.py

@@ -16,11 +16,6 @@
 # This file is a part of the vllm-ascend project.
 # Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
 #
-"""Compare the short outputs of HF and vLLM when using greedy sampling.
-
-Run `pytest tests/e2e/multicard/test_qwen3_moe.py`.
-"""
-
 import os
 
 from vllm import SamplingParams

tests/e2e/multicard/4-cards/long_sequence/test_chunked_prefill.py

@@ -16,11 +16,6 @@
 # This file is a part of the vllm-ascend project.
 # Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
 #
-"""Compare the short outputs of HF and vLLM when using greedy sampling.
-
-Run `pytest tests/e2e/multicard/test_qwen3_moe.py`.
-"""
-
 import os
 import random
 import string

tests/e2e/multicard/4-cards/test_data_parallel_tp2.py

@@ -1,7 +1,3 @@
-"""
-Run `pytest tests/e2e/multicard/test_data_parallel_tp2.py`.
-"""
-
 import os
 import subprocess
 import sys

tests/e2e/multicard/4-cards/test_kimi_k2.py

@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# Copyright 2023 The vLLM team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This file is a part of the vllm-ascend project.
+# Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
+#
+import os
+
+from tests.e2e.conftest import VllmRunner
+
+os.environ["PYTORCH_NPU_ALLOC_CONF"] = "max_split_size_mb:256"
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+
+def test_kimi_k2_thinking_w4a16_tp4():
+    example_prompts = [
+        "Hello, my name is",
+    ]
+    max_tokens = 5
+
+    with VllmRunner(
+            "vllm-ascend/Kimi-K2-Thinking-Pruning",
+            max_model_len=8192,
+            dtype="auto",
+            tensor_parallel_size=4,
+            enable_expert_parallel=True,
+            compilation_config={
+                "cudagraph_mode": "FULL_DECODE_ONLY",
+                "cudagraph_capture_sizes": [1],
+            },
+    ) as vllm_model:
+        vllm_model.generate_greedy(example_prompts, max_tokens)

tests/e2e/multicard/4-cards/test_qwen3_next.py

@@ -16,11 +16,6 @@
 # This file is a part of the vllm-ascend project.
 # Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
 #
-"""Compare the short outputs of HF and vLLM when using greedy sampling.
-
-Run `pytest tests/e2e/multicard/test_qwen3_next.py`.
-"""
-
 import os
 from unittest.mock import patch
 

tests/e2e/singlecard/test_aclgraph_accuracy.py

@@ -36,7 +36,7 @@ MODELS = [
 
 @pytest.mark.parametrize("model", MODELS)
 @pytest.mark.parametrize("max_tokens", [32])
-def test_models_output_between_eager_and_aclgraph(
+def test_models_output(
     model: str,
     max_tokens: int,
 ) -> None:

tests/e2e/singlecard/test_quantization.py

@@ -46,7 +46,7 @@ def test_qwen3_w8a8_quant():
         outputs_0_lst=vllm_target_outputs,
         outputs_1_lst=vllm_quant_w8a8_outputs,
         name_0="vllm_target_outputs",
-        name_1="vllm_w8a16_outputs",
+        name_1="vllm_quant_w8a8_outputs",
     )
 
 
@@ -75,5 +75,5 @@ def test_qwen3_dense_w8a16():
         outputs_0_lst=vllm_target_outputs,
         outputs_1_lst=vllm_quant_w8a16_outputs,
         name_0="vllm_target_outputs",
-        name_1="vllm_w8a16_outputs",
+        name_1="vllm_quant_w8a16_outputs",
     )