# # Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This file is a part of the vllm-ascend project. # # This workflow related to the resources atlas 800 A3 # **Please note**: current A3 resource pool's maximum allowed concurrency is 5*16 NPUs # We will limit the concurrency of jobs on A3 to avoid the risk of insufficient resources name: Nightly-A3 on: schedule: # Run test at 23:45 Beijing time (UTC+8) - cron: "45 15 * * *" workflow_dispatch: pull_request: branches: - 'main' - '*-dev' - 'releases/v*' types: [ labeled, synchronize ] permissions: contents: read pull-requests: read issues: read # Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly # declared as "shell: bash -el {0}" on steps that need to be properly activated. # It's used to activate ascend-toolkit environment variables. defaults: run: shell: bash -el {0} concurrency: group: ascend-nightly-${{ github.ref }}-a3 cancel-in-progress: true jobs: parse-trigger: name: Parse trigger and determine test scope if: >- github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'nightly-test') uses: ./.github/workflows/_parse_trigger.yaml build-image: name: Build nightly-a3 image if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' uses: ./.github/workflows/_nightly_image_build.yaml with: target: a3 secrets: HW_USERNAME: ${{ secrets.HW_USERNAME }} HW_TOKEN: ${{ secrets.HW_TOKEN }} GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }} multi-node-tests: name: multi-node needs: [parse-trigger, build-image] if: >- always() && needs.parse-trigger.outputs.run == 'true' && (needs.build-image.result == 'success' || needs.build-image.result == 'skipped') strategy: fail-fast: false max-parallel: 2 matrix: test_config: - name: multi-node-deepseek-pd config_file_path: DeepSeek-V3.yaml size: 2 - name: multi-node-qwen3-dp config_file_path: Qwen3-235B-A22B.yaml size: 2 - name: multi-node-qwenw8a8-2node config_file_path: Qwen3-235B-W8A8.yaml size: 2 - name: multi-node-qwenw8a8-2node-eplb config_file_path: Qwen3-235B-W8A8-EPLB.yaml size: 2 - name: multi-node-dpsk3.2-2node config_file_path: DeepSeek-V3_2-W8A8-A3-dual-nodes.yaml size: 2 - name: multi-node-qwen3-dp-mooncake-layerwise config_file_path: Qwen3-235B-A22B-Mooncake-Layerwise.yaml size: 2 - name: multi-node-deepseek-r1-w8a8-longseq config_file_path: DeepSeek-R1-W8A8-longseq.yaml size: 2 - name: multi-node-qwenw8a8-2node-longseq config_file_path: Qwen3-235B-W8A8-longseq.yaml size: 2 - name: multi-node-qwen-disagg-pd config_file_path: Qwen3-235B-disagg-pd.yaml size: 2 - name: multi-node-qwen-vl-disagg-pd config_file_path: Qwen3-VL-235B-disagg-pd.yaml size: 2 - name: multi-node-kimi-k2-instruct-w8a8 config_file_path: Kimi-K2-Instruct-W8A8.yaml size: 2 - name: multi-node-deepseek-v3.1 config_file_path: DeepSeek-V3.1-BF16.yaml size: 2 - name: multi-node-deepseek-v3.2-W8A8-EP config_file_path: DeepSeek-V3_2-W8A8-EP.yaml size: 4 uses: ./.github/workflows/_e2e_nightly_multi_node.yaml with: soc_version: a3 runner: linux-aarch64-a3-0 image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' replicas: 1 size: ${{ matrix.test_config.size }} config_file_path: ${{ matrix.test_config.config_file_path }} vllm_ascend_ref: ${{ needs.parse-trigger.outputs.ref }} should_run: >- ${{ needs.parse-trigger.outputs.run == 'true' && ( needs.parse-trigger.outputs.filter == 'all' || contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name)) ) }} secrets: KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} single-node-tests: name: single-node needs: [parse-trigger, build-image, multi-node-tests] if: >- always() && needs.parse-trigger.outputs.run == 'true' && (needs.build-image.result == 'success' || needs.build-image.result == 'skipped') strategy: fail-fast: false matrix: test_config: # pytest-driven tests - name: qwen3-30b-acc os: linux-aarch64-a3-4 tests: tests/e2e/weekly/single_node/models/test_qwen3_30b_acc.py - name: custom-multi-ops os: linux-aarch64-a3-16 tests: tests/e2e/nightly/single_node/ops/multicard_ops_a3/ # YAML-driven tests - name: deepseek-r1-0528-w8a8 os: linux-aarch64-a3-16 config_file_path: DeepSeek-R1-0528-W8A8.yaml - name: deepseek-r1-w8a8-hbm os: linux-aarch64-a3-16 config_file_path: DeepSeek-R1-W8A8-HBM.yaml - name: deepseek-v3-2-w8a8 os: linux-aarch64-a3-16 config_file_path: DeepSeek-V3.2-W8A8.yaml - name: glm-5-w4a8 os: linux-aarch64-a3-16 config_file_path: GLM-5.yaml - name: glm-4.7-w8a8 os: linux-aarch64-a3-16 config_file_path: GLM-4.7.yaml - name: kimi-k2-thinking os: linux-aarch64-a3-16 config_file_path: Kimi-K2-Thinking.yaml - name: kimi-k2.5 os: linux-aarch64-a3-16 config_file_path: Kimi-K2.5.yaml - name: minimax-m2-5 os: linux-aarch64-a3-16 config_file_path: MiniMax-M2.5-A3.yaml - name: mtpx-deepseek-r1-0528-w8a8 os: linux-aarch64-a3-16 config_file_path: MTPX-DeepSeek-R1-0528-W8A8.yaml - name: qwen3-235b-a22b-w8a8 os: linux-aarch64-a3-16 config_file_path: Qwen3-235B-A22B-W8A8.yaml - name: qwen3-30b-a3b-w8a8 os: linux-aarch64-a3-4 config_file_path: Qwen3-30B-A3B-W8A8.yaml - name: qwen3-next-80b-a3b-instruct os: linux-aarch64-a3-4 config_file_path: Qwen3-Next-80B-A3B-Instruct.yaml - name: qwen3-next-80b-a3b-instruct-w8a8 os: linux-aarch64-a3-4 config_file_path: Qwen3-Next-80B-A3B-Instruct-W8A8.yaml - name: qwq-32b os: linux-aarch64-a3-4 config_file_path: QwQ-32B.yaml - name: qwen3-32b-int8 os: linux-aarch64-a3-4 config_file_path: Qwen3-32B-Int8.yaml - name: qwen2-5-vl-7b os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-7B-Instruct.yaml - name: qwen2-5-vl-7b-epd os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-7B-Instruct-EPD.yaml - name: qwen2-5-vl-32b os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-32B-Instruct.yaml - name: qwen3-32b-int8-a3-feature-stack3 os: linux-aarch64-a3-4 config_file_path: Qwen3-32B-Int8-A3-Feature-Stack3.yaml - name: qwen3-32b-int8-prefix-cache os: linux-aarch64-a3-4 config_file_path: Prefix-Cache-Qwen3-32B-Int8.yaml - name: deepseek-r1-0528-w8a8-prefix-cache os: linux-aarch64-a3-16 config_file_path: Prefix-Cache-DeepSeek-R1-0528-W8A8.yaml uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: runner: ${{ matrix.test_config.os }} image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' tests: ${{ matrix.test_config.tests }} config_file_path: ${{ matrix.test_config.config_file_path }} name: ${{ matrix.test_config.name }} should_run: >- ${{ needs.parse-trigger.outputs.run == 'true' && ( needs.parse-trigger.outputs.filter == 'all' || contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name)) ) }}