# # Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This file is a part of the vllm-ascend project. # # This workflow related to the resources atlas 800 A3 # **Please note**: current A3 resource pool's maximum allowed concurrency is 5*16 NPUs # We will limit the concurrency of jobs on A3 to avoid the risk of insufficient resources name: Nightly-A3 on: schedule: # Run test at 24:00 Beijing time (UTC+8) - cron: "0 16 * * *" workflow_dispatch: pull_request: branches: - 'main' types: [ labeled, synchronize ] permissions: contents: read pull-requests: read issues: read # Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly # declared as "shell: bash -el {0}" on steps that need to be properly activated. # It's used to activate ascend-toolkit environment variables. defaults: run: shell: bash -el {0} concurrency: group: ascend-nightly-${{ github.ref }}-a3 cancel-in-progress: true jobs: parse-trigger: name: Parse trigger and determine test scope runs-on: linux-aarch64-a2b3-0 outputs: should_run: ${{ steps.parse.outputs.should_run }} test_filter: ${{ steps.parse.outputs.test_filter }} is_pr_event: ${{ steps.parse.outputs.is_pr_event }} steps: - name: Parse trigger id: parse uses: actions/github-script@v7 with: script: | const eventName = context.eventName; function parseNightlyComment(body) { if (!body) return null; const match = body.trim().match(/^\/nightly(?:\s+(.+))?$/m); if (!match) return null; const args = (match[1] || '').trim(); if (!args || args === 'all') return 'all'; // Wrap with commas for exact-name matching: ",name1,name2," return ',' + args.split(/\s+/).join(',') + ','; } // schedule / workflow_dispatch: run all tests with pre-built image if (eventName === 'schedule' || eventName === 'workflow_dispatch') { core.setOutput('should_run', 'true'); core.setOutput('test_filter', 'all'); core.setOutput('is_pr_event', 'false'); return; } // pull_request (labeled / synchronize) if (eventName === 'pull_request') { const labels = context.payload.pull_request.labels.map(l => l.name); if (!labels.includes('nightly-test')) { core.setOutput('should_run', 'false'); core.setOutput('test_filter', ''); core.setOutput('is_pr_event', 'true'); return; } // Search comments for latest /nightly command const prNumber = context.payload.pull_request.number; const comments = await github.paginate(github.rest.issues.listComments, { owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, per_page: 100, }); let testFilter = null; for (let i = comments.length - 1; i >= 0; i--) { const result = parseNightlyComment(comments[i].body); if (result !== null) { testFilter = result; break; } } // No /nightly comment found: do not run any tests if (testFilter === null) { core.info('nightly-test label present but no /nightly comment found; skipping.'); core.setOutput('should_run', 'false'); core.setOutput('test_filter', ''); core.setOutput('is_pr_event', 'true'); return; } core.setOutput('should_run', 'true'); core.setOutput('test_filter', testFilter); core.setOutput('is_pr_event', 'true'); return; } // Fallback core.setOutput('should_run', 'false'); core.setOutput('test_filter', ''); core.setOutput('is_pr_event', 'false'); multi-node-tests: name: multi-node needs: [parse-trigger] if: always() && needs.parse-trigger.outputs.should_run == 'true' strategy: fail-fast: false max-parallel: 1 matrix: test_config: - name: multi-node-deepseek-pd config_file_path: DeepSeek-V3.yaml size: 2 - name: multi-node-qwen3-dp config_file_path: Qwen3-235B-A22B.yaml size: 2 # - name: multi-node-dpsk-4node-pd # config_file_path: DeepSeek-R1-W8A8.yaml # size: 4 - name: multi-node-qwenw8a8-2node config_file_path: Qwen3-235B-W8A8.yaml size: 2 # - name: multi-node-deepseek-r1-w8a8-eplb # config_file_path: DeepSeek-R1-W8A8-EPLB.yaml # size: 4 - name: multi-node-qwenw8a8-2node-eplb config_file_path: Qwen3-235B-W8A8-EPLB.yaml size: 2 - name: multi-node-dpsk3.2-2node config_file_path: DeepSeek-V3_2-W8A8-A3-dual-nodes.yaml size: 2 - name: multi-node-qwen3-dp-mooncake-layerwise config_file_path: Qwen3-235B-A22B-Mooncake-Layerwise.yaml size: 2 - name: multi-node-deepseek-r1-w8a8-longseq config_file_path: DeepSeek-R1-W8A8-longseq.yaml size: 2 - name: multi-node-qwenw8a8-2node-longseq config_file_path: Qwen3-235B-W8A8-longseq.yaml size: 2 - name: multi-node-deepseek-V3_2-W8A8-cp config_file_path: DeepSeek-V3_2-W8A8-cp.yaml size: 2 - name: multi-node-qwen-disagg-pd config_file_path: Qwen3-235B-disagg-pd.yaml size: 2 - name: multi-node-qwen-vl-disagg-pd config_file_path: Qwen3-VL-235B-disagg-pd.yaml size: 2 - name: multi-node-kimi-k2-instruct-w8a8 config_file_path: Kimi-K2-Instruct-W8A8.yaml size: 2 - name: multi-node-deepseek-v3.1 config_file_path: DeepSeek-V3.1-BF16.yaml size: 2 - name: multi-node-deepseek-v3.2-W8A8-EP config_file_path: DeepSeek-V3_2-W8A8-EP.yaml size: 4 uses: ./.github/workflows/_e2e_nightly_multi_node.yaml with: soc_version: a3 runner: linux-aarch64-a3-0 image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' replicas: 1 size: ${{ matrix.test_config.size }} config_file_path: ${{ matrix.test_config.config_file_path }} vllm_ascend_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref_name }} is_pr_test: >- ${{ needs.parse-trigger.outputs.is_pr_event == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }} is_run: >- ${{ needs.parse-trigger.outputs.should_run == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }} secrets: KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} single-node-tests: name: single-node needs: [parse-trigger, multi-node-tests] if: always() && needs.parse-trigger.outputs.should_run == 'true' strategy: fail-fast: false matrix: test_config: - name: qwen3-30b-acc os: linux-aarch64-a3-4 tests: tests/e2e/weekly/single_node/models/test_qwen3_30b_acc.py uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: runner: ${{ matrix.test_config.os }} image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' tests: ${{ matrix.test_config.tests }} name: ${{ matrix.test_config.name }} is_pr_test: >- ${{ needs.parse-trigger.outputs.is_pr_event == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }} is_run: >- ${{ needs.parse-trigger.outputs.should_run == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }} single-node-yaml-tests: name: single-node if: always() && needs.parse-trigger.outputs.should_run == 'true' needs: [parse-trigger, multi-node-tests] strategy: fail-fast: false matrix: test_config: # YAML-driven tests - name: deepseek-r1-0528-w8a8 os: linux-aarch64-a3-16 config_file_path: DeepSeek-R1-0528-W8A8.yaml - name: deepseek-r1-w8a8-hbm os: linux-aarch64-a3-16 config_file_path: DeepSeek-R1-W8A8-HBM.yaml - name: deepseek-v3-2-w8a8 os: linux-aarch64-a3-16 config_file_path: DeepSeek-V3.2-W8A8.yaml - name: kimi-k2-thinking os: linux-aarch64-a3-16 config_file_path: Kimi-K2-Thinking.yaml - name: mtpx-deepseek-r1-0528-w8a8 os: linux-aarch64-a3-16 config_file_path: MTPX-DeepSeek-R1-0528-W8A8.yaml - name: qwen3-235b-a22b-w8a8 os: linux-aarch64-a3-16 config_file_path: Qwen3-235B-A22B-W8A8.yaml - name: qwen3-30b-a3b-w8a8 os: linux-aarch64-a3-4 config_file_path: Qwen3-30B-A3B-W8A8.yaml - name: qwen3-next-80b-a3b-instruct-w8a8 os: linux-aarch64-a3-4 config_file_path: Qwen3-Next-80B-A3B-Instruct-W8A8.yaml - name: qwq-32b os: linux-aarch64-a3-4 config_file_path: QwQ-32B.yaml - name: qwen3-32b-int8 os: linux-aarch64-a3-4 config_file_path: Qwen3-32B-Int8.yaml - name: qwen2-5-vl-7b os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-7B-Instruct.yaml - name: qwen2-5-vl-7b-epd os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-7B-Instruct-EPD.yaml - name: qwen2-5-vl-32b os: linux-aarch64-a3-4 config_file_path: Qwen2.5-VL-32B-Instruct.yaml - name: qwen3-32b-int8-a3-feature-stack3 os: linux-aarch64-a3-4 config_file_path: Qwen3-32B-Int8-A3-Feature-Stack3.yaml - name: qwen3-32b-int8-prefix-cache os: linux-aarch64-a3-4 config_file_path: Prefix-Cache-Qwen3-32B-Int8.yaml - name: deepseek-r1-0528-w8a8-prefix-cache os: linux-aarch64-a3-16 config_file_path: Prefix-Cache-DeepSeek-R1-0528-W8A8.yaml uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: runner: ${{ matrix.test_config.os }} image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' config_file_path: ${{ matrix.test_config.config_file_path }} name: ${{ matrix.test_config.name }} is_pr_test: >- ${{ needs.parse-trigger.outputs.is_pr_event == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }} is_run: >- ${{ needs.parse-trigger.outputs.should_run == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }} custom-ops-tests: name: test ops needs: [parse-trigger, multi-node-tests] if: always() && needs.parse-trigger.outputs.should_run == 'true' strategy: fail-fast: false matrix: test_config: - name: custom-multi-ops os: linux-aarch64-a3-16 tests: tests/e2e/nightly/single_node/ops/multicard_ops_a3/ uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: runner: ${{ matrix.test_config.os }} image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3' tests: ${{ matrix.test_config.tests }} name: ${{ matrix.test_config.name }} is_pr_test: >- ${{ needs.parse-trigger.outputs.is_pr_event == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }} is_run: >- ${{ needs.parse-trigger.outputs.should_run == 'true' && ( needs.parse-trigger.outputs.test_filter == 'all' || contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name)) ) }}