[Nightly] Nightly pre-build image (#7388)
### What this PR does / why we need it?
This pull request refactor nightly image build and simplify the logic of
multi workflows.
1. Nightly image build become the prerequisite when the test are
triggered by `schedule` or `workflow_dispatch`
2. Simplify the pull request select case logic
3. Next step: Implement replaceable nightly tests. Specifically, if
nightly tests are manually triggered, they can accept any optional
docker image to meet the needs of different commits(Which means the
image is customizable).
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
- vLLM version: v0.17.0
- vLLM main:
4034c3d32e
---------
Signed-off-by: wangli <wangli858794774@gmail.com>
This commit is contained in:
11
.github/workflows/_e2e_nightly_multi_node.yaml
vendored
11
.github/workflows/_e2e_nightly_multi_node.yaml
vendored
@@ -45,12 +45,9 @@ on:
|
||||
default: main
|
||||
type: string
|
||||
description: used for pr level tests
|
||||
is_pr_test:
|
||||
should_run:
|
||||
required: true
|
||||
type: boolean
|
||||
is_run:
|
||||
required: true
|
||||
type: boolean
|
||||
secrets:
|
||||
KUBECONFIG_B64:
|
||||
required: true
|
||||
@@ -74,7 +71,7 @@ jobs:
|
||||
name: ${{ inputs.config_file_path }}
|
||||
# This is the runner with no NPU for k8s controller
|
||||
runs-on: ${{ inputs.runner }}
|
||||
if: ${{ inputs.is_run }}
|
||||
if: ${{ inputs.should_run }}
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-cpu
|
||||
env:
|
||||
@@ -84,7 +81,7 @@ jobs:
|
||||
- name: Decode kubeconfig from secrets
|
||||
run: |
|
||||
# Decode and save kubeconfig
|
||||
if [ "${{ inputs.is_pr_test }}" = "true" ]; then
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo "PR test mode"
|
||||
if [ "${{ inputs.soc_version }}" = "a3" ]; then
|
||||
echo "Using A3 cached kubeconfig"
|
||||
@@ -161,7 +158,7 @@ jobs:
|
||||
image="${{ inputs.image }}"
|
||||
config_file_path="${{ inputs.config_file_path }}"
|
||||
fail_tag=FAIL_TAG_"${{ inputs.config_file_path }}"
|
||||
is_pr_test="${{ inputs.is_pr_test }}"
|
||||
is_pr_test="${{ github.event_name == 'pull_request' }}"
|
||||
vllm_version="${{ inputs.vllm_version }}"
|
||||
vllm_ascend_ref="${{ inputs.vllm_ascend_ref }}"
|
||||
vllm_ascend_remote_url="${{ inputs.vllm_ascend_remote_url }}"
|
||||
|
||||
21
.github/workflows/_e2e_nightly_single_node.yaml
vendored
21
.github/workflows/_e2e_nightly_single_node.yaml
vendored
@@ -40,10 +40,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: "v0.18.0"
|
||||
is_pr_test:
|
||||
required: true
|
||||
type: boolean
|
||||
is_run:
|
||||
should_run:
|
||||
required: true
|
||||
type: boolean
|
||||
|
||||
@@ -64,7 +61,7 @@ jobs:
|
||||
e2e-nightly:
|
||||
name: ${{ inputs.name || inputs.config_file_path || inputs.tests }}
|
||||
runs-on: ${{ inputs.runner }}
|
||||
if: ${{ inputs.is_run }}
|
||||
if: ${{ inputs.should_run }}
|
||||
timeout-minutes: 600
|
||||
container:
|
||||
image: ${{ inputs.image }}
|
||||
@@ -85,14 +82,14 @@ jobs:
|
||||
pip install uv
|
||||
|
||||
- name: uninstall vlm vllm-ascend and remove code (if pr test)
|
||||
if: ${{ inputs.is_pr_test }}
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
pip uninstall -y vllm vllm-ascend || true
|
||||
cp -r /vllm-workspace/vllm-ascend/benchmark /tmp/aisbench-backup || true
|
||||
rm -rf /vllm-workspace/vllm /vllm-workspace/vllm-ascend
|
||||
|
||||
- name: Checkout vllm-project/vllm repo
|
||||
if: ${{ inputs.is_pr_test }}
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: vllm-project/vllm
|
||||
@@ -101,27 +98,27 @@ jobs:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Checkout vllm-project/vllm-ascend repo
|
||||
if: ${{ inputs.is_pr_test }}
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
path: ./temp-vllm-ascend
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Move code to /vllm-workspace
|
||||
if: ${{ inputs.is_pr_test }}
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
mv ./temp-vllm /vllm-workspace/vllm
|
||||
mv ./temp-vllm-ascend /vllm-workspace/vllm-ascend
|
||||
ls -R /vllm-workspace
|
||||
|
||||
- name: Install vllm-project/vllm from source
|
||||
if: ${{ inputs.is_pr_test }}
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
working-directory: /vllm-workspace/vllm
|
||||
run: |
|
||||
VLLM_TARGET_DEVICE=empty uv pip install -e .
|
||||
|
||||
- name: Install vllm-project/vllm-ascend
|
||||
if: ${{ inputs.is_pr_test }}
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
working-directory: /vllm-workspace/vllm-ascend
|
||||
env:
|
||||
PIP_EXTRA_INDEX_URL: https://mirrors.huaweicloud.com/ascend/repos/pypi
|
||||
@@ -132,7 +129,7 @@ jobs:
|
||||
uv pip install -v -e .
|
||||
|
||||
- name: Install aisbench
|
||||
if: ${{ inputs.is_pr_test }}
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
shell: bash -l {0}
|
||||
run: |
|
||||
cp -r /tmp/aisbench-backup /vllm-workspace/vllm-ascend/benchmark
|
||||
|
||||
71
.github/workflows/_nightly_image_build.yaml
vendored
Normal file
71
.github/workflows/_nightly_image_build.yaml
vendored
Normal file
@@ -0,0 +1,71 @@
|
||||
#
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# This file is a part of the vllm-ascend project.
|
||||
#
|
||||
|
||||
name: 'Nightly image build'
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
target:
|
||||
required: true
|
||||
type: string
|
||||
description: "Build target: 'a2' or 'a3'"
|
||||
secrets:
|
||||
HW_USERNAME:
|
||||
required: false
|
||||
HW_TOKEN:
|
||||
required: false
|
||||
GITEE_TOKEN:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build nightly-${{ inputs.target }} image
|
||||
runs-on: ubuntu-22.04-arm
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Login to Huawei Cloud SWR
|
||||
id: login-swr
|
||||
if: ${{ env.HW_USERNAME != '' && env.HW_TOKEN != '' }}
|
||||
env:
|
||||
HW_USERNAME: ${{ secrets.HW_USERNAME }}
|
||||
HW_TOKEN: ${{ secrets.HW_TOKEN }}
|
||||
run: |
|
||||
echo "$HW_TOKEN" | docker login -u "$HW_USERNAME" --password-stdin swr.cn-southwest-2.myhuaweicloud.com
|
||||
|
||||
- name: Build nightly-${{ inputs.target }} image
|
||||
env:
|
||||
GITEE_USERNAME: ${{ vars.GITEE_USERNAME }}
|
||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
||||
run: |
|
||||
IMAGE="swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-${{ inputs.target }}"
|
||||
docker build \
|
||||
--network host \
|
||||
--platform linux/arm64 \
|
||||
-f .github/workflows/dockerfiles/Dockerfile.nightly.${{ inputs.target }} \
|
||||
--build-arg CANN_VERSION="8.5.1" \
|
||||
--build-arg UBUNTU_VERSION="22.04" \
|
||||
--build-arg PYTHON_VERSION="3.11" \
|
||||
--build-arg GITEE_USERNAME="${GITEE_USERNAME}" \
|
||||
--build-arg GITEE_TOKEN="${GITEE_TOKEN}" \
|
||||
-t "$IMAGE" .
|
||||
|
||||
- name: Push image to SWR
|
||||
if: ${{ github.repository_owner == 'vllm-project' && steps.login-swr.conclusion == 'success' }}
|
||||
run: |
|
||||
docker push swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-${{ inputs.target }}
|
||||
115
.github/workflows/_parse_trigger.yaml
vendored
Normal file
115
.github/workflows/_parse_trigger.yaml
vendored
Normal file
@@ -0,0 +1,115 @@
|
||||
#
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# This file is a part of the vllm-ascend project.
|
||||
#
|
||||
|
||||
name: 'Parse nightly trigger'
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runner:
|
||||
required: false
|
||||
type: string
|
||||
default: linux-aarch64-a2b3-0
|
||||
outputs:
|
||||
run:
|
||||
description: "Whether nightly tests should run"
|
||||
value: ${{ jobs.parse.outputs.run }}
|
||||
filter:
|
||||
description: "Comma-wrapped test name filter (e.g. ',name1,name2,'), or 'all'"
|
||||
value: ${{ jobs.parse.outputs.filter }}
|
||||
ref:
|
||||
description: "The vllm-ascend ref (commit SHA for PRs, branch/tag name otherwise)"
|
||||
value: ${{ jobs.parse.outputs.ref }}
|
||||
|
||||
jobs:
|
||||
parse:
|
||||
name: Parse trigger and determine test scope
|
||||
runs-on: ${{ inputs.runner }}
|
||||
outputs:
|
||||
run: ${{ steps.parse.outputs.run }}
|
||||
filter: ${{ steps.parse.outputs.filter }}
|
||||
ref: ${{ steps.parse.outputs.ref }}
|
||||
steps:
|
||||
- name: Parse trigger
|
||||
id: parse
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const eventName = context.eventName;
|
||||
|
||||
function parseNightlyComment(body) {
|
||||
if (!body) return null;
|
||||
const match = body.trim().match(/^\/nightly(?:\s+(.+))?$/m);
|
||||
if (!match) return null;
|
||||
const args = (match[1] || '').trim();
|
||||
if (!args || args === 'all') return 'all';
|
||||
// Wrap with commas for exact-name matching: ",name1,name2,"
|
||||
return ',' + args.split(/\s+/).join(',') + ',';
|
||||
}
|
||||
|
||||
function getRef() {
|
||||
if (eventName === 'pull_request') {
|
||||
return context.payload.pull_request.head.sha;
|
||||
}
|
||||
return (context.ref || '').replace(/^refs\/(heads|tags)\//, '') || 'main';
|
||||
}
|
||||
|
||||
core.setOutput('ref', getRef());
|
||||
|
||||
// 1. schedule / workflow_dispatch: run all tests with pre-built image
|
||||
if (eventName === 'schedule' || eventName === 'workflow_dispatch') {
|
||||
core.setOutput('run', 'true');
|
||||
core.setOutput('filter', 'all');
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. pull_request (labeled / synchronize)
|
||||
if (eventName === 'pull_request') {
|
||||
const labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
if (!labels.includes('nightly-test')) {
|
||||
core.setOutput('run', 'false');
|
||||
core.setOutput('filter', '');
|
||||
return;
|
||||
}
|
||||
// Search comments for latest /nightly command
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const comments = await github.paginate(github.rest.issues.listComments, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
per_page: 100,
|
||||
});
|
||||
let filter = null;
|
||||
for (let i = comments.length - 1; i >= 0; i--) {
|
||||
const result = parseNightlyComment(comments[i].body);
|
||||
if (result !== null) { filter = result; break; }
|
||||
}
|
||||
// No /nightly comment found: do not run any tests
|
||||
if (filter === null) {
|
||||
core.info('nightly-test label present but no /nightly comment found; skipping.');
|
||||
core.setOutput('run', 'false');
|
||||
core.setOutput('filter', '');
|
||||
return;
|
||||
}
|
||||
core.setOutput('run', 'true');
|
||||
core.setOutput('filter', filter);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback
|
||||
core.setOutput('run', 'false');
|
||||
core.setOutput('filter', '');
|
||||
44
.github/workflows/nightly_image_build.yaml
vendored
Normal file
44
.github/workflows/nightly_image_build.yaml
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
#
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# This file is a part of the vllm-ascend project.
|
||||
#
|
||||
|
||||
# This workflow builds nightly images as a layer-cache warm-up at 20:00 Beijing time.
|
||||
# The nightly test workflows (Nightly-A2, Nightly-A3) each rebuild the image fresh
|
||||
# before running tests, so this schedule only serves to pre-populate the build cache.
|
||||
name: Nightly Image Build Schedule
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
# Next step: Add more inputs here if needed, e.g. vllm version, vllm-ascend version, image tag, etc.
|
||||
|
||||
jobs:
|
||||
build-a2:
|
||||
uses: ./.github/workflows/_nightly_image_build.yaml
|
||||
with:
|
||||
target: a2
|
||||
secrets:
|
||||
HW_USERNAME: ${{ secrets.HW_USERNAME }}
|
||||
HW_TOKEN: ${{ secrets.HW_TOKEN }}
|
||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
||||
|
||||
build-a3:
|
||||
uses: ./.github/workflows/_nightly_image_build.yaml
|
||||
with:
|
||||
target: a3
|
||||
secrets:
|
||||
HW_USERNAME: ${{ secrets.HW_USERNAME }}
|
||||
HW_TOKEN: ${{ secrets.HW_TOKEN }}
|
||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
||||
@@ -1,66 +0,0 @@
|
||||
name: Nightly Image Build Schedule
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# UTC+8: 20pm, 23pm
|
||||
- cron: '0 12,15 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
# This workflow builds and pushes Docker images for nightly-ci
|
||||
# It will be built base on the quay.io/ascend/vllm-ascend:main
|
||||
# And have some customizations for nightly testing, pushing to Huawei Cloud SWR
|
||||
jobs:
|
||||
build-and-sync:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
target: ['a2', 'a3']
|
||||
|
||||
outputs:
|
||||
image-tag: ${{ steps.build-image.outputs.image-tag }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Show build target
|
||||
run: |
|
||||
echo "Building target: ${{ matrix.target }}"
|
||||
|
||||
- name: Login to Huawei Cloud SWR
|
||||
id: login-swr
|
||||
if: ${{ env.HW_USERNAME != '' && env.HW_TOKEN != '' }}
|
||||
run: |
|
||||
echo "${{ env.HW_TOKEN }}" | docker login -u "${{ env.HW_USERNAME }}" --password-stdin swr.cn-southwest-2.myhuaweicloud.com
|
||||
env:
|
||||
HW_USERNAME: ${{ secrets.HW_USERNAME }}
|
||||
HW_TOKEN: ${{ secrets.HW_TOKEN }}
|
||||
|
||||
- name: Build image
|
||||
id: build-image
|
||||
env:
|
||||
GITEE_USERNAME: ${{ vars.GITEE_USERNAME }}
|
||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
||||
run: |
|
||||
TARGET="${{ matrix.target }}"
|
||||
IMAGE_TAG="swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-${TARGET}"
|
||||
|
||||
echo "Building image: $IMAGE_TAG"
|
||||
docker build \
|
||||
--network host \
|
||||
--platform linux/arm64 \
|
||||
-f .github/workflows/dockerfiles/Dockerfile.nightly.${TARGET} \
|
||||
--build-arg CANN_VERSION="8.5.1" \
|
||||
--build-arg UBUNTU_VERSION="22.04" \
|
||||
--build-arg PYTHON_VERSION="3.11" \
|
||||
--build-arg GITEE_USERNAME="${GITEE_USERNAME}" \
|
||||
--build-arg GITEE_TOKEN="${GITEE_TOKEN}" \
|
||||
-t "$IMAGE_TAG" .
|
||||
|
||||
echo "image-tag=$IMAGE_TAG" >> $GITHUB_OUTPUT
|
||||
|
||||
# To avoid pushing images from forks, only push when the repository owner is 'vllm-project'
|
||||
- name: Push image to SWR
|
||||
if: ${{ github.repository_owner == 'vllm-project' && steps.login-swr.conclusion == 'success' }}
|
||||
run: |
|
||||
docker push ${{ steps.build-image.outputs.image-tag }}
|
||||
168
.github/workflows/schedule_nightly_test_a2.yaml
vendored
168
.github/workflows/schedule_nightly_test_a2.yaml
vendored
@@ -49,125 +49,42 @@ concurrency:
|
||||
jobs:
|
||||
parse-trigger:
|
||||
name: Parse trigger and determine test scope
|
||||
runs-on: linux-aarch64-a2b3-0
|
||||
if: >-
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
contains(github.event.pull_request.labels.*.name, 'nightly-test')
|
||||
outputs:
|
||||
should_run: ${{ steps.parse.outputs.should_run }}
|
||||
test_filter: ${{ steps.parse.outputs.test_filter }}
|
||||
is_pr_event: ${{ steps.parse.outputs.is_pr_event }}
|
||||
steps:
|
||||
- name: Parse trigger
|
||||
id: parse
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const eventName = context.eventName;
|
||||
uses: ./.github/workflows/_parse_trigger.yaml
|
||||
|
||||
function parseNightlyComment(body) {
|
||||
if (!body) return null;
|
||||
const match = body.trim().match(/^\/nightly(?:\s+(.+))?$/m);
|
||||
if (!match) return null;
|
||||
const args = (match[1] || '').trim();
|
||||
if (!args || args === 'all') return 'all';
|
||||
// Wrap with commas for exact-name matching: ",name1,name2,"
|
||||
return ',' + args.split(/\s+/).join(',') + ',';
|
||||
}
|
||||
|
||||
// schedule / workflow_dispatch: run all tests with pre-built image
|
||||
if (eventName === 'schedule' || eventName === 'workflow_dispatch') {
|
||||
core.setOutput('should_run', 'true');
|
||||
core.setOutput('test_filter', 'all');
|
||||
core.setOutput('is_pr_event', 'false');
|
||||
return;
|
||||
}
|
||||
|
||||
// pull_request (labeled / synchronize)
|
||||
if (eventName === 'pull_request') {
|
||||
const labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
if (!labels.includes('nightly-test')) {
|
||||
core.setOutput('should_run', 'false');
|
||||
core.setOutput('test_filter', '');
|
||||
core.setOutput('is_pr_event', 'true');
|
||||
return;
|
||||
}
|
||||
// Search comments for latest /nightly command
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const comments = await github.paginate(github.rest.issues.listComments, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
per_page: 100,
|
||||
});
|
||||
let testFilter = null;
|
||||
for (let i = comments.length - 1; i >= 0; i--) {
|
||||
const result = parseNightlyComment(comments[i].body);
|
||||
if (result !== null) { testFilter = result; break; }
|
||||
}
|
||||
// No /nightly comment found: do not run any tests
|
||||
if (testFilter === null) {
|
||||
core.info('nightly-test label present but no /nightly comment found; skipping.');
|
||||
core.setOutput('should_run', 'false');
|
||||
core.setOutput('test_filter', '');
|
||||
core.setOutput('is_pr_event', 'true');
|
||||
return;
|
||||
}
|
||||
core.setOutput('should_run', 'true');
|
||||
core.setOutput('test_filter', testFilter);
|
||||
core.setOutput('is_pr_event', 'true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback
|
||||
core.setOutput('should_run', 'false');
|
||||
core.setOutput('test_filter', '');
|
||||
core.setOutput('is_pr_event', 'false');
|
||||
build-image:
|
||||
name: Build nightly-a2 image
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
uses: ./.github/workflows/_nightly_image_build.yaml
|
||||
with:
|
||||
target: a2
|
||||
secrets:
|
||||
HW_USERNAME: ${{ secrets.HW_USERNAME }}
|
||||
HW_TOKEN: ${{ secrets.HW_TOKEN }}
|
||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
||||
|
||||
single-node-tests:
|
||||
name: single-node
|
||||
needs: [parse-trigger]
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
needs: [parse-trigger, build-image]
|
||||
if: >-
|
||||
always() &&
|
||||
needs.parse-trigger.outputs.run == 'true' &&
|
||||
(needs.build-image.result == 'success' || needs.build-image.result == 'skipped')
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test_config:
|
||||
# pytest-driven tests
|
||||
- name: test_custom_op
|
||||
os: linux-aarch64-a2b3-1
|
||||
tests: tests/e2e/nightly/single_node/ops/singlecard_ops
|
||||
- name: test_custom_op_multi_card
|
||||
os: linux-aarch64-a2b3-4
|
||||
tests: tests/e2e/nightly/single_node/ops/multicard_ops_a2/
|
||||
uses: ./.github/workflows/_e2e_nightly_single_node.yaml
|
||||
with:
|
||||
runner: ${{ matrix.test_config.os }}
|
||||
tests: ${{ matrix.test_config.tests }}
|
||||
name: ${{ matrix.test_config.name }}
|
||||
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a2'
|
||||
is_pr_test: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.is_pr_event == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
|
||||
single-node-yaml-tests:
|
||||
name: single-node
|
||||
needs: [parse-trigger]
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test_config:
|
||||
# YAML-driven tests
|
||||
- name: qwen3-32b
|
||||
os: linux-aarch64-a2b3-4
|
||||
config_file_path: Qwen3-32B.yaml
|
||||
@@ -181,28 +98,24 @@ jobs:
|
||||
with:
|
||||
runner: ${{ matrix.test_config.os }}
|
||||
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a2'
|
||||
tests: ${{ matrix.test_config.tests }}
|
||||
config_file_path: ${{ matrix.test_config.config_file_path }}
|
||||
name: ${{ matrix.test_config.name }}
|
||||
is_pr_test: >-
|
||||
should_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.is_pr_event == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
needs.parse-trigger.outputs.run == 'true' && (
|
||||
needs.parse-trigger.outputs.filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
|
||||
|
||||
multi-node-tests:
|
||||
name: multi-node
|
||||
needs: [parse-trigger, single-node-tests, single-node-yaml-tests]
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
needs: [parse-trigger, build-image, single-node-tests]
|
||||
if: >-
|
||||
always() &&
|
||||
needs.parse-trigger.outputs.run == 'true' &&
|
||||
(needs.build-image.result == 'success' || needs.build-image.result == 'skipped')
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
@@ -222,19 +135,12 @@ jobs:
|
||||
replicas: 1
|
||||
size: ${{ matrix.test_config.size }}
|
||||
config_file_path: ${{ matrix.test_config.config_file_path }}
|
||||
vllm_ascend_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref_name }}
|
||||
is_pr_test: >-
|
||||
vllm_ascend_ref: ${{ needs.parse-trigger.outputs.ref }}
|
||||
should_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.is_pr_event == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
needs.parse-trigger.outputs.run == 'true' && (
|
||||
needs.parse-trigger.outputs.filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
secrets:
|
||||
@@ -242,7 +148,7 @@ jobs:
|
||||
|
||||
single-node-accuracy-tests:
|
||||
needs: [parse-trigger]
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
if: always() && needs.parse-trigger.outputs.run == 'true'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -283,9 +189,9 @@ jobs:
|
||||
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.1-910b-ubuntu22.04-py3.11'
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
needs.parse-trigger.outputs.run == 'true' && (
|
||||
needs.parse-trigger.outputs.filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
upload: false
|
||||
|
||||
199
.github/workflows/schedule_nightly_test_a3.yaml
vendored
199
.github/workflows/schedule_nightly_test_a3.yaml
vendored
@@ -49,86 +49,30 @@ concurrency:
|
||||
jobs:
|
||||
parse-trigger:
|
||||
name: Parse trigger and determine test scope
|
||||
runs-on: linux-aarch64-a2b3-0
|
||||
if: >-
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
contains(github.event.pull_request.labels.*.name, 'nightly-test')
|
||||
outputs:
|
||||
should_run: ${{ steps.parse.outputs.should_run }}
|
||||
test_filter: ${{ steps.parse.outputs.test_filter }}
|
||||
is_pr_event: ${{ steps.parse.outputs.is_pr_event }}
|
||||
steps:
|
||||
- name: Parse trigger
|
||||
id: parse
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const eventName = context.eventName;
|
||||
uses: ./.github/workflows/_parse_trigger.yaml
|
||||
|
||||
function parseNightlyComment(body) {
|
||||
if (!body) return null;
|
||||
const match = body.trim().match(/^\/nightly(?:\s+(.+))?$/m);
|
||||
if (!match) return null;
|
||||
const args = (match[1] || '').trim();
|
||||
if (!args || args === 'all') return 'all';
|
||||
// Wrap with commas for exact-name matching: ",name1,name2,"
|
||||
return ',' + args.split(/\s+/).join(',') + ',';
|
||||
}
|
||||
|
||||
// schedule / workflow_dispatch: run all tests with pre-built image
|
||||
if (eventName === 'schedule' || eventName === 'workflow_dispatch') {
|
||||
core.setOutput('should_run', 'true');
|
||||
core.setOutput('test_filter', 'all');
|
||||
core.setOutput('is_pr_event', 'false');
|
||||
return;
|
||||
}
|
||||
|
||||
// pull_request (labeled / synchronize)
|
||||
if (eventName === 'pull_request') {
|
||||
const labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
if (!labels.includes('nightly-test')) {
|
||||
core.setOutput('should_run', 'false');
|
||||
core.setOutput('test_filter', '');
|
||||
core.setOutput('is_pr_event', 'true');
|
||||
return;
|
||||
}
|
||||
// Search comments for latest /nightly command
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const comments = await github.paginate(github.rest.issues.listComments, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
per_page: 100,
|
||||
});
|
||||
let testFilter = null;
|
||||
for (let i = comments.length - 1; i >= 0; i--) {
|
||||
const result = parseNightlyComment(comments[i].body);
|
||||
if (result !== null) { testFilter = result; break; }
|
||||
}
|
||||
// No /nightly comment found: do not run any tests
|
||||
if (testFilter === null) {
|
||||
core.info('nightly-test label present but no /nightly comment found; skipping.');
|
||||
core.setOutput('should_run', 'false');
|
||||
core.setOutput('test_filter', '');
|
||||
core.setOutput('is_pr_event', 'true');
|
||||
return;
|
||||
}
|
||||
core.setOutput('should_run', 'true');
|
||||
core.setOutput('test_filter', testFilter);
|
||||
core.setOutput('is_pr_event', 'true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback
|
||||
core.setOutput('should_run', 'false');
|
||||
core.setOutput('test_filter', '');
|
||||
core.setOutput('is_pr_event', 'false');
|
||||
build-image:
|
||||
name: Build nightly-a3 image
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
uses: ./.github/workflows/_nightly_image_build.yaml
|
||||
with:
|
||||
target: a3
|
||||
secrets:
|
||||
HW_USERNAME: ${{ secrets.HW_USERNAME }}
|
||||
HW_TOKEN: ${{ secrets.HW_TOKEN }}
|
||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
||||
|
||||
multi-node-tests:
|
||||
name: multi-node
|
||||
needs: [parse-trigger]
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
needs: [parse-trigger, build-image]
|
||||
if: >-
|
||||
always() &&
|
||||
needs.parse-trigger.outputs.run == 'true' &&
|
||||
(needs.build-image.result == 'success' || needs.build-image.result == 'skipped')
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
@@ -140,15 +84,9 @@ jobs:
|
||||
- name: multi-node-qwen3-dp
|
||||
config_file_path: Qwen3-235B-A22B.yaml
|
||||
size: 2
|
||||
# - name: multi-node-dpsk-4node-pd
|
||||
# config_file_path: DeepSeek-R1-W8A8.yaml
|
||||
# size: 4
|
||||
- name: multi-node-qwenw8a8-2node
|
||||
config_file_path: Qwen3-235B-W8A8.yaml
|
||||
size: 2
|
||||
# - name: multi-node-deepseek-r1-w8a8-eplb
|
||||
# config_file_path: DeepSeek-R1-W8A8-EPLB.yaml
|
||||
# size: 4
|
||||
- name: multi-node-qwenw8a8-2node-eplb
|
||||
config_file_path: Qwen3-235B-W8A8-EPLB.yaml
|
||||
size: 2
|
||||
@@ -190,19 +128,12 @@ jobs:
|
||||
replicas: 1
|
||||
size: ${{ matrix.test_config.size }}
|
||||
config_file_path: ${{ matrix.test_config.config_file_path }}
|
||||
vllm_ascend_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref_name }}
|
||||
is_pr_test: >-
|
||||
vllm_ascend_ref: ${{ needs.parse-trigger.outputs.ref }}
|
||||
should_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.is_pr_event == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
needs.parse-trigger.outputs.run == 'true' && (
|
||||
needs.parse-trigger.outputs.filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
secrets:
|
||||
@@ -210,44 +141,22 @@ jobs:
|
||||
|
||||
single-node-tests:
|
||||
name: single-node
|
||||
needs: [parse-trigger, multi-node-tests]
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
needs: [parse-trigger, build-image, multi-node-tests]
|
||||
if: >-
|
||||
always() &&
|
||||
needs.parse-trigger.outputs.run == 'true' &&
|
||||
(needs.build-image.result == 'success' || needs.build-image.result == 'skipped')
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test_config:
|
||||
# pytest-driven tests
|
||||
- name: qwen3-30b-acc
|
||||
os: linux-aarch64-a3-4
|
||||
tests: tests/e2e/weekly/single_node/models/test_qwen3_30b_acc.py
|
||||
uses: ./.github/workflows/_e2e_nightly_single_node.yaml
|
||||
with:
|
||||
runner: ${{ matrix.test_config.os }}
|
||||
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3'
|
||||
tests: ${{ matrix.test_config.tests }}
|
||||
name: ${{ matrix.test_config.name }}
|
||||
is_pr_test: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.is_pr_event == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
|
||||
single-node-yaml-tests:
|
||||
name: single-node
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
needs: [parse-trigger, multi-node-tests]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test_config:
|
||||
- name: custom-multi-ops
|
||||
os: linux-aarch64-a3-16
|
||||
tests: tests/e2e/nightly/single_node/ops/multicard_ops_a3/
|
||||
# YAML-driven tests
|
||||
- name: deepseek-r1-0528-w8a8
|
||||
os: linux-aarch64-a3-16
|
||||
@@ -316,51 +225,13 @@ jobs:
|
||||
with:
|
||||
runner: ${{ matrix.test_config.os }}
|
||||
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3'
|
||||
tests: ${{ matrix.test_config.tests }}
|
||||
config_file_path: ${{ matrix.test_config.config_file_path }}
|
||||
name: ${{ matrix.test_config.name }}
|
||||
is_pr_test: >-
|
||||
should_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.is_pr_event == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
|
||||
custom-ops-tests:
|
||||
name: test ops
|
||||
needs: [parse-trigger, multi-node-tests]
|
||||
if: always() && needs.parse-trigger.outputs.should_run == 'true'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test_config:
|
||||
- name: custom-multi-ops
|
||||
os: linux-aarch64-a3-16
|
||||
tests: tests/e2e/nightly/single_node/ops/multicard_ops_a3/
|
||||
uses: ./.github/workflows/_e2e_nightly_single_node.yaml
|
||||
with:
|
||||
runner: ${{ matrix.test_config.os }}
|
||||
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3'
|
||||
tests: ${{ matrix.test_config.tests }}
|
||||
name: ${{ matrix.test_config.name }}
|
||||
is_pr_test: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.is_pr_event == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
is_run: >-
|
||||
${{
|
||||
needs.parse-trigger.outputs.should_run == 'true' && (
|
||||
needs.parse-trigger.outputs.test_filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.test_filter, format(',{0},', matrix.test_config.name))
|
||||
needs.parse-trigger.outputs.run == 'true' && (
|
||||
needs.parse-trigger.outputs.filter == 'all' ||
|
||||
contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
|
||||
)
|
||||
}}
|
||||
|
||||
Reference in New Issue
Block a user