From fae1c59a79825839a3ab40cdf3ec430978e4218f Mon Sep 17 00:00:00 2001 From: zhangxinyuehfad <59153331+zhangxinyuehfad@users.noreply.github.com> Date: Tue, 11 Nov 2025 10:36:05 +0800 Subject: [PATCH] [Fix] Refactor and fix dist test to e2e full test (#3808) ### What this PR does / why we need it? Fix ci test on A3 1. delete lables 2. fix filter yaml file name 3. refactor dist test to e2e full test 4. skip test_models_distributed_Qwen3_MOE_TP2_WITH_EP & test_models_distributed_Qwen3_MOE_W8A8_WITH_EP because of https://github.com/vllm-project/vllm-ascend/issues/3895 - vLLM version: v0.11.0 - vLLM main: https://github.com/vllm-project/vllm/commit/83f478bb19489b41e9d208b47b4bb5a95ac171ac Signed-off-by: hfadzxy --- .github/workflows/_e2e_test.yaml | 86 ++++++++++++- .github/workflows/vllm_ascend_dist.yaml | 155 ------------------------ 2 files changed, 84 insertions(+), 157 deletions(-) delete mode 100644 .github/workflows/vllm_ascend_dist.yaml diff --git a/.github/workflows/_e2e_test.yaml b/.github/workflows/_e2e_test.yaml index ec25747c..98f91533 100644 --- a/.github/workflows/_e2e_test.yaml +++ b/.github/workflows/_e2e_test.yaml @@ -115,7 +115,7 @@ jobs: #pytest -sv tests/e2e/singlecard/ops/ e2e-2-cards: - name: multicard + name: multicard-2 runs-on: ${{ inputs.runner }}-2 container: image: ${{ inputs.image }} @@ -202,4 +202,86 @@ jobs: pytest -sv tests/e2e/multicard/test_pipeline_parallel.py pytest -sv tests/e2e/multicard/test_prefix_caching.py pytest -sv tests/e2e/multicard/test_qwen3_moe.py - + + e2e-4-cards: + name: multicard-4 + runs-on: linux-aarch64-a3-4 + container: + image: m.daocloud.io/quay.io/ascend/cann:8.3.rc1-a3-ubuntu22.04-py3.11 + env: + VLLM_LOGGING_LEVEL: ERROR + VLLM_USE_MODELSCOPE: True + steps: + - name: Check npu and CANN info + run: | + npu-smi info + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + + - name: Config mirrors + run: | + sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list + pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + apt-get update -y + apt install git wget curl -y + git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/ + + - name: Checkout vllm-project/vllm-ascend repo + uses: actions/checkout@v4 + with: + path: ./vllm-ascend + + - name: Install system dependencies + run: | + apt-get -y install `cat packages.txt` + apt-get -y install gcc g++ cmake libnuma-dev + + - name: Checkout vllm-project/vllm repo + uses: actions/checkout@v4 + with: + repository: vllm-project/vllm + ref: ${{ inputs.vllm }} + path: ./vllm-empty + + - name: Install vllm-project/vllm from source + working-directory: ./vllm-empty + run: | + VLLM_TARGET_DEVICE=empty pip install -e . + + - name: Install vllm-project/vllm-ascend + working-directory: ./vllm-ascend + run: | + export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux/devlib + pip install -r requirements-dev.txt + pip install -v -e . + + - name: Run vllm-project/vllm-ascend test for V1 Engine + working-directory: ./vllm-ascend + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_USE_MODELSCOPE: True + run: | + pytest -sv \ + tests/e2e/multicard/test_offline_inference_distributed.py::test_models_distributed_DeepSeek_multistream_moe \ + tests/e2e/multicard/test_offline_inference_distributed.py::test_models_distributed_DeepSeek_W4A8DYNAMIC + # tests/e2e/multicard/test_qwen3_moe.py::test_models_distributed_Qwen3_MOE_TP2_WITH_EP \ + # tests/e2e/multicard/test_qwen3_moe.py::test_models_distributed_Qwen3_MOE_W8A8_WITH_EP + + - name: Install Ascend toolkit & triton_ascend (for Qwen3-Next-80B-A3B-Instruct) + shell: bash -l {0} + run: | + wget -q https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/Ascend-BiSheng-toolkit_aarch64.run -O /tmp/Ascend-BiSheng-toolkit_aarch64.run + chmod a+x /tmp/Ascend-BiSheng-toolkit_aarch64.run + /tmp/Ascend-BiSheng-toolkit_aarch64.run --install + . /usr/local/Ascend/8.3.RC1/bisheng_toolkit/set_env.sh + python3 -m pip install "https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/triton_ascend-3.2.0.dev20250914-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" + + - name: Run vllm-project/vllm-ascend Qwen3 Next test + working-directory: ./vllm-ascend + shell: bash -el {0} + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_USE_MODELSCOPE: True + run: | + . /usr/local/Ascend/8.3.RC1/bisheng_toolkit/set_env.sh + pytest -sv tests/e2e/multicard/test_qwen3_next.py \ No newline at end of file diff --git a/.github/workflows/vllm_ascend_dist.yaml b/.github/workflows/vllm_ascend_dist.yaml deleted file mode 100644 index 2d7c8ef2..00000000 --- a/.github/workflows/vllm_ascend_dist.yaml +++ /dev/null @@ -1,155 +0,0 @@ -# -# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# This file is a part of the vllm-ascend project. -# - -name: 'e2e test / a3-test' - -on: - workflow_call: - push: - branches: - - 'main' - - '*-dev' - paths: - - '.github/workflows/vllm_ascend_dist.yml' - - 'tests/e2e/multicard/**' - - 'Dockerfile' - - 'vllm_ascend/**' - - 'setup.py' - - 'pyproject.toml' - - 'requirements.txt' - - 'cmake/**' - - 'CMakeLists.txt' - - 'csrc/**' - pull_request: - branches: - - 'main' - - '*-dev' - paths: - - '.github/workflows/vllm_ascend_dist.yml' - - 'tests/e2e/multicard/**' - - 'Dockerfile' - - 'vllm_ascend/**' - - 'setup.py' - - 'pyproject.toml' - - 'requirements.txt' - - 'cmake/**' - - 'CMakeLists.txt' - - 'csrc/**' - types: [ labeled ] - -# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly -# declared as "shell: bash -el {0}" on steps that need to be properly activated. -# It's used to activate ascend-toolkit environment variables. -defaults: - run: - shell: bash -el {0} - -# only cancel in-progress runs of the same workflow -# and ignore the lint / 8 cards test type -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - e2e: - # only trigger e2e test after lint passed and the change is e2e related with pull request. - if: ${{ contains(github.event.pull_request.labels.*.name, 'dist-test') && contains(github.event.pull_request.labels.*.name, 'ready-for-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'push'}} - strategy: - matrix: - os: [linux-aarch64-a3-4] - vllm_version: [v0.11.0] - name: vLLM Ascend test - runs-on: ${{ matrix.os }} - container: - image: m.daocloud.io/quay.io/ascend/cann:8.3.rc1-a3-ubuntu22.04-py3.11 - env: - DEBIAN_FRONTEND: noninteractive - steps: - - name: Check npu and CANN info - run: | - npu-smi info - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - - - name: Config mirrors - run: | - sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list - pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple - apt-get update -y - apt install git wget curl -y - git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/ - - - name: Checkout vllm-project/vllm-ascend repo - uses: actions/checkout@v4 - with: - path: ./vllm-ascend - - - name: Install system dependencies - run: | - apt-get -y install `cat packages.txt` - apt-get -y install gcc g++ cmake libnuma-dev - - - name: Checkout vllm-project/vllm repo - uses: actions/checkout@v4 - with: - repository: vllm-project/vllm - ref: ${{ matrix.vllm_version }} - path: ./vllm-empty - - - name: Install vllm-project/vllm from source - working-directory: ./vllm-empty - run: | - VLLM_TARGET_DEVICE=empty pip install -e . - - - name: Install vllm-project/vllm-ascend - working-directory: ./vllm-ascend - run: | - export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux/devlib - pip install -r requirements-dev.txt - pip install -v -e . - - - name: Run vllm-project/vllm-ascend test for V1 Engine - working-directory: ./vllm-ascend - env: - VLLM_WORKER_MULTIPROC_METHOD: spawn - VLLM_USE_MODELSCOPE: True - run: | - # TODO: enable more tests - pytest -sv \ - tests/e2e/multicard/test_offline_inference_distributed.py::test_models_distributed_DeepSeek_multistream_moe \ - tests/e2e/multicard/test_offline_inference_distributed.py::test_models_distributed_DeepSeek_W4A8DYNAMIC \ - tests/e2e/multicard/test_qwen3_moe.py::test_models_distributed_Qwen3_MOE_TP2_WITH_EP \ - tests/e2e/multicard/test_qwen3_moe.py::test_models_distributed_Qwen3_MOE_W8A8_WITH_EP - - - name: Install Ascend toolkit & triton_ascend (for Qwen3-Next-80B-A3B-Instruct) - shell: bash -l {0} - run: | - wget -q https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/Ascend-BiSheng-toolkit_aarch64.run -O /tmp/Ascend-BiSheng-toolkit_aarch64.run - chmod a+x /tmp/Ascend-BiSheng-toolkit_aarch64.run - /tmp/Ascend-BiSheng-toolkit_aarch64.run --install - . /usr/local/Ascend/8.3.RC1/bisheng_toolkit/set_env.sh - python3 -m pip install "https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/triton_ascend-3.2.0.dev20250914-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" - - - name: Run vllm-project/vllm-ascend Qwen3 Next test - working-directory: ./vllm-ascend - shell: bash -el {0} - env: - VLLM_WORKER_MULTIPROC_METHOD: spawn - VLLM_USE_MODELSCOPE: True - run: | - . /usr/local/Ascend/8.3.RC1/bisheng_toolkit/set_env.sh - pytest -sv tests/e2e/multicard/test_qwen3_next.py \ No newline at end of file