Add models test and add serval new models yaml (#3394)
### What this PR does / why we need it? This PR added Add accuracy CI for servals new models - `ascend test / accuracy` is for PR triggered check popluar models accuracy - `ascedn test / models` is for accuracy report, full models test, nightly model test - Add Qwen2-Audio-7B-Instruct, Qwen2-VL-7B-Instruct, Qwen3-8B, Qwen3-VL-30B-A3B-Instruct ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI passed Closes: https://github.com/vllm-project/vllm-ascend/pull/2330 Closes: https://github.com/vllm-project/vllm-ascend/pull/3362 - vLLM version: v0.11.0rc3 - vLLM main: https://github.com/vllm-project/vllm/commit/v0.11.0 --------- Signed-off-by: hfadzxy <starmoon_zhang@163.com> Signed-off-by: Yikun Jiang <yikunkero@gmail.com> Co-authored-by: hfadzxy <starmoon_zhang@163.com>
This commit is contained in:
Yikun Jiang
177
.github/workflows/vllm_ascend_test_models.yaml
vendored
Normal file
177
.github/workflows/vllm_ascend_test_models.yaml
vendored
Normal file
@@ -0,0 +1,177 @@
|
||||
#
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# This file is a part of the vllm-ascend project.
|
||||
#
|
||||
|
||||
# This test will be triggered:
|
||||
# 1. schedule
|
||||
# 2. pull_request change the related files
|
||||
# 3. workflow_dispatch with models input
|
||||
|
||||
name: ascend test / models
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Runs every 6 hours
|
||||
- cron: '0 */6 * * *'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'main'
|
||||
- '*-dev'
|
||||
paths:
|
||||
- '.github/workflows/vllm_ascend_test_models.yaml'
|
||||
- 'tests/e2e/models/test_lm_eval_correctness.py'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
vllm-ascend-version:
|
||||
description: 'vllm-ascend:'
|
||||
required: true
|
||||
type: choice
|
||||
# Current supported vLLM versions
|
||||
options:
|
||||
- latest
|
||||
- main
|
||||
default: main
|
||||
|
||||
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
|
||||
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
|
||||
# It's used to activate ascend-toolkit environment variables.
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -el {0}
|
||||
|
||||
# only cancel in-progress runs of the same workflow
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
run:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- model_name: Qwen3-8B
|
||||
runner: a2-1
|
||||
- model_name: Qwen2.5-VL-7B-Instruct
|
||||
runner: a2-1
|
||||
- model_name: Qwen2-Audio-7B-Instruct
|
||||
runner: a2-1
|
||||
- model_name: Qwen3-30B-A3B
|
||||
runner: a2-2
|
||||
- model_name: Qwen3-VL-30B-A3B-Instruct
|
||||
runner: a2-2
|
||||
- model_name: DeepSeek-V2-Lite
|
||||
runner: a2-2
|
||||
fail-fast: false
|
||||
uses: ./.github/workflows/_accuracy_test.yaml
|
||||
with:
|
||||
vllm: v0.11.0
|
||||
runner: linux-aarch64-${{ matrix.runner }}
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11
|
||||
model_name: ${{ matrix.model_name }}
|
||||
upload: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.vllm-ascend-version == 'latest' }}
|
||||
|
||||
create_pr:
|
||||
runs-on: ubuntu-latest
|
||||
needs: run
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.vllm-ascend-version == 'latest' }}
|
||||
env:
|
||||
UPSTREAM_REPO: vllm-project/vllm-ascend
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: vllm-ascend-ci/vllm-ascend
|
||||
token: ${{ secrets.PAT_TOKEN }}
|
||||
ref: main
|
||||
|
||||
- name: Add upstream remote
|
||||
run: |
|
||||
git remote add upstream https://github.com/${{ env.UPSTREAM_REPO }}.git
|
||||
git fetch upstream
|
||||
git remote -v
|
||||
|
||||
- name: Set Git user info dynamically
|
||||
run: |
|
||||
git config user.name "${{ github.actor }}"
|
||||
git config user.email "${{ github.actor }}@users.noreply.github.com"
|
||||
|
||||
- name: Create or switch to branch
|
||||
run: |
|
||||
TIMESTAMP=$(date +%Y%m%d%H%M%S)
|
||||
BRANCH_NAME="auto-pr/accuracy-report-${TIMESTAMP}"
|
||||
echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_ENV
|
||||
git checkout -B "${BRANCH_NAME}" upstream/main
|
||||
|
||||
- name: Download only current run reports
|
||||
uses: actions/download-artifact@v5
|
||||
with:
|
||||
path: ./docs/source/developer_guide/evaluation/accuracy_report
|
||||
pattern: report-*
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
run-id: ${{ github.run_id }}
|
||||
|
||||
- name: Delete old report
|
||||
run: |
|
||||
find ./docs/source/developer_guide/evaluation/accuracy_report -maxdepth 1 -type f -name '*.md' ! -name 'index.md' -delete
|
||||
find ./docs/source/developer_guide/evaluation/accuracy_report -mindepth 2 -type f -name '*.md' -exec mv -f {} ./docs/source/developer_guide/evaluation/accuracy_report \;
|
||||
find ./docs/source/developer_guide/evaluation/accuracy_report -mindepth 1 -type d -empty -delete
|
||||
|
||||
- name: Update accuracy_report/index.md
|
||||
run: |
|
||||
REPORT_DIR="./docs/source/developer_guide/evaluation/accuracy_report"
|
||||
INDEX_MD="$REPORT_DIR/index.md"
|
||||
{
|
||||
echo "# Accuracy Report"
|
||||
echo ""
|
||||
echo ":::{toctree}"
|
||||
echo ":caption: Accuracy Report"
|
||||
echo ":maxdepth: 1"
|
||||
|
||||
for report in "$REPORT_DIR"/*.md; do
|
||||
filename="$(basename "$report" .md)"
|
||||
if [ "$filename" != "index" ]; then
|
||||
echo "$filename"
|
||||
fi
|
||||
done
|
||||
echo ":::"
|
||||
} > "$INDEX_MD"
|
||||
|
||||
- name: push accuracy report
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
|
||||
run: |
|
||||
git add ./docs/source/developer_guide/evaluation/accuracy_report/*.md
|
||||
git commit -s -m "[Doc] Update accuracy reports for ${{ env.BRANCH_NAME }}"
|
||||
git push -f origin "${{ env.BRANCH_NAME }}"
|
||||
|
||||
- name: Create PR in upstream via API
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.PAT_TOKEN }}
|
||||
script: |
|
||||
const pr = await github.rest.pulls.create({
|
||||
owner: 'vllm-project',
|
||||
repo: 'vllm-ascend',
|
||||
head: `vllm-ascend-ci:${{ env.BRANCH_NAME }}`,
|
||||
base: 'main',
|
||||
title: `[Doc] Update accuracy reports for ${{ env.BRANCH_NAME }}`,
|
||||
body: `The accuracy results running on NPU Altlas A2 have changed, updating reports for: All models
|
||||
|
||||
- [Workflow run][1]
|
||||
|
||||
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`
|
||||
});
|
||||
core.info(`Created PR #${pr.data.number}`);
|
||||
Reference in New Issue
Block a user