# # Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This file is a part of the vllm-ascend project. # # This workflow related to the resources atlas 800 A3 # **Please note**: current A3 resource pool's maximum allowed concurrency is 5*16 NPUs # We will limit the concurrency of jobs on A3 to avoid the risk of insufficient resources name: Nightly-A3 on: schedule: # Run test at 24:00 Beijing time (UTC+8) - cron: "0 16 * * *" workflow_dispatch: pull_request: branches: - 'main' types: [ labeled ] # Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly # declared as "shell: bash -el {0}" on steps that need to be properly activated. # It's used to activate ascend-toolkit environment variables. defaults: run: shell: bash -el {0} concurrency: group: ascend-nightly-${{ github.ref }}-a3 cancel-in-progress: true jobs: multi-node-tests: name: multi-node if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') strategy: fail-fast: false max-parallel: 1 matrix: test_config: - name: multi-node-deepseek-pd config_file_path: DeepSeek-V3.yaml size: 2 - name: multi-node-qwen3-dp config_file_path: Qwen3-235B-A22B.yaml size: 2 # - name: multi-node-dpsk-4node-pd # config_file_path: DeepSeek-R1-W8A8.yaml # size: 4 - name: multi-node-qwenw8a8-2node config_file_path: Qwen3-235B-W8A8.yaml size: 2 # - name: multi-node-deepseek-r1-w8a8-eplb # config_file_path: DeepSeek-R1-W8A8-EPLB.yaml # size: 4 - name: multi-node-qwenw8a8-2node-eplb config_file_path: Qwen3-235B-W8A8-EPLB.yaml size: 2 - name: multi-node-dpsk3.2-2node config_file_path: DeepSeek-V3_2-W8A8-A3-dual-nodes.yaml size: 2 - name: multi-node-qwen3-dp-mooncake-layerwise config_file_path: Qwen3-235B-A22B-Mooncake-Layerwise.yaml size: 2 - name: multi-node-deepseek-r1-w8a8-longseq config_file_path: DeepSeek-R1-W8A8-longseq.yaml size: 2 - name: multi-node-qwenw8a8-2node-longseq config_file_path: Qwen3-235B-W8A8-longseq.yaml size: 2 - name: multi-node-deepseek-V3_2-W8A8-cp config_file_path: DeepSeek-V3_2-W8A8-cp.yaml size: 2 - name: multi-node-qwen-disagg-pd config_file_path: Qwen3-235B-disagg-pd.yaml size: 2 - name: multi-node-qwen-vl-disagg-pd config_file_path: Qwen3-VL-235B-disagg-pd.yaml size: 2 - name: multi-node-kimi-k2-instruct-w8a8 config_file_path: Kimi-K2-Instruct-W8A8.yaml size: 2 - name: multi-node-deepseek-v3.1 config_file_path: DeepSeek-V3.1-BF16.yaml size: 2 - name: multi-node-deepseek-v3.2-W8A8-EP config_file_path: DeepSeek-V3_2-W8A8-EP.yaml size: 4 uses: ./.github/workflows/_e2e_nightly_multi_node.yaml with: soc_version: a3 runner: linux-aarch64-a3-0 image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' replicas: 1 size: ${{ matrix.test_config.size }} config_file_path: ${{ matrix.test_config.config_file_path }} secrets: KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} single-node-tests: name: single-node if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') needs: [multi-node-tests] strategy: fail-fast: false matrix: test_config: - name: qwen3-30b-acc os: linux-aarch64-a3-4 tests: tests/e2e/weekly/single_node/models/test_qwen3_30b_acc.py uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: runner: ${{ matrix.test_config.os }} image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' tests: ${{ matrix.test_config.tests }} name: ${{ matrix.test_config.name }} single-node-yaml-tests: name: single-node if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') needs: [multi-node-tests] strategy: fail-fast: false matrix: test_config: # YAML-driven tests - name: deepseek-r1-0528-w8a8 os: linux-aarch64-a3-16 config_file_path: DeepSeek-R1-0528-W8A8.yaml - name: deepseek-r1-w8a8-hbm os: linux-aarch64-a3-16 config_file_path: DeepSeek-R1-W8A8-HBM.yaml - name: deepseek-v3-2-w8a8 os: linux-aarch64-a3-16 config_file_path: DeepSeek-V3.2-W8A8.yaml - name: kimi-k2-thinking os: linux-aarch64-a3-16 config_file_path: Kimi-K2-Thinking.yaml - name: mtpx-deepseek-r1-0528-w8a8 os: linux-aarch64-a3-16 config_file_path: MTPX-DeepSeek-R1-0528-W8A8.yaml - name: qwen3-235b-a22b-w8a8 os: linux-aarch64-a3-16 config_file_path: Qwen3-235B-A22B-W8A8.yaml - name: qwen3-30b-a3b-w8a8 os: linux-aarch64-a3-4 config_file_path: Qwen3-30B-A3B-W8A8.yaml - name: qwen3-next-80b-a3b-instruct-w8a8 os: linux-aarch64-a3-4 config_file_path: Qwen3-Next-80B-A3B-Instruct-W8A8.yaml - name: qwq-32b os: linux-aarch64-a3-4 config_file_path: QwQ-32B.yaml - name: qwen3-32b-int8 os: linux-aarch64-a3-4 config_file_path: Qwen3-32B-Int8.yaml - name: qwen2-5-vl-7b os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-7B-Instruct.yaml - name: qwen2-5-vl-7b-epd os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-7B-Instruct-EPD.yaml - name: qwen2-5-vl-32b os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-32B-Instruct.yaml - name: qwen3-32b-int8-a3-feature-stack3 os: linux-aarch64-a3-4 config_file_path: Qwen3-32B-Int8-A3-Feature-Stack3.yaml - name: qwen3-32b-int8-prefix-cache os: linux-aarch64-a3-4 config_file_path: Prefix-Cache-Qwen3-32B-Int8.yaml - name: deepseek-r1-0528-w8a8-prefix-cache os: linux-aarch64-a3-16 config_file_path: Prefix-Cache-DeepSeek-R1-0528-W8A8.yaml uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: runner: ${{ matrix.test_config.os }} image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' config_file_path: ${{ matrix.test_config.config_file_path }} name: ${{ matrix.test_config.name }} custom-ops-tests: name: test ops if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') needs: multi-node-tests strategy: fail-fast: false matrix: test_config: - name: custom-multi-ops os: linux-aarch64-a3-16 tests: tests/e2e/nightly/single_node/ops/multicard_ops_a3/ uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: runner: ${{ matrix.test_config.os }} image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' tests: ${{ matrix.test_config.tests }} name: ${{ matrix.test_config.name }}