From a95afc011e4e6b9f37a98ab44439d9abaac64467 Mon Sep 17 00:00:00 2001
From: Yikun Jiang <yikunkero@gmail.com>
Date: Sun, 22 Jun 2025 17:21:57 +0800
Subject: [PATCH] [CI] Enable merge trigger unit test and accuracy test
 schedule job (#1345)

### What this PR does / why we need it?
- Enable merge trigger unit test and accuracy test schedule job
- Pin lm-eval==0.4.8 to resovle Qwen3 8B accuracy
### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
CI passed

Signed-off-by: Yikun Jiang <yikunkero@gmail.com>
---
 .github/workflows/accuracy_test.yaml       | 14 ++++++++++----
 .github/workflows/image_310p_openeuler.yml |  2 +-
 .github/workflows/image_310p_ubuntu.yml    |  4 ++--
 .github/workflows/vllm_ascend_test.yaml    |  8 ++++++--
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/accuracy_test.yaml b/.github/workflows/accuracy_test.yaml
index 0297f62..e14cd39 100644
--- a/.github/workflows/accuracy_test.yaml
+++ b/.github/workflows/accuracy_test.yaml
@@ -22,6 +22,9 @@
 name: Benchmarks / accuracy
 
 on:
+  schedule:
+    # Runs every 6 hours
+    - cron:  '0 */6 * * *'
   pull_request:
     types: [ labeled ]
   workflow_dispatch:
@@ -43,6 +46,7 @@ on:
         options:
           - main
           - v0.7.3-dev
+          - v0.9.1-dev
       models:
         description: 'model:'
         required: true
@@ -75,7 +79,7 @@ jobs:
       contains(github.event.pull_request.labels.*.name, 'vl-accuracy-test') ||
       contains(github.event.pull_request.labels.*.name, 'dense-accuracy-test')) &&
       contains(github.event.pull_request.labels.*.name, 'ready-for-test') ||
-      github.event_name == 'workflow_dispatch'
+      github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
       }}
     runs-on: >-
       ${{
@@ -94,6 +98,8 @@ jobs:
         #   - dense-accuracy-test: Qwen/Qwen2.5-7B-Instruct
         #   - vl-accuracy-test: Qwen/Qwen2.5-VL-7B-Instruct
         model_name: ${{ fromJSON(
+          (github.event_name == 'schedule' &&
+            '["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
           (github.event.inputs.models == 'all' &&
             '["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
           (github.event.inputs.models == 'Qwen/Qwen2.5-7B-Instruct' &&
@@ -103,9 +109,9 @@ jobs:
           (github.event.inputs.models == 'Qwen/Qwen3-8B-Base' &&
             '["Qwen/Qwen3-8B-Base"]') ||
           contains(github.event.pull_request.labels.*.name, 'accuracy-test') &&
-            '["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct"]' ||
+            '["Qwen/Qwen3-8B-Base","Qwen/Qwen2.5-VL-7B-Instruct"]' ||
           contains(github.event.pull_request.labels.*.name, 'dense-accuracy-test') &&
-            '["Qwen/Qwen2.5-7B-Instruct"]' ||
+            '["Qwen/Qwen3-8B-Base"]' ||
           contains(github.event.pull_request.labels.*.name, 'vl-accuracy-test') &&
             '["Qwen/Qwen2.5-VL-7B-Instruct"]'
          ) }}
@@ -181,7 +187,7 @@ jobs:
 
       - name: Install lm-eval, ray, and datasets
         run: |
-            pip install lm-eval
+            pip install lm-eval==0.4.8
 
       - name: Collect version info
         run: |
diff --git a/.github/workflows/image_310p_openeuler.yml b/.github/workflows/image_310p_openeuler.yml
index 5b84a24..2626a77 100644
--- a/.github/workflows/image_310p_openeuler.yml
+++ b/.github/workflows/image_310p_openeuler.yml
@@ -1,4 +1,4 @@
-name: 'image / openEuler'
+name: 'image / openEuler / 310p'
 # This is a docker build check and publish job:
 # 1. PR Triggered docker image build check
 #   - is for image build check
diff --git a/.github/workflows/image_310p_ubuntu.yml b/.github/workflows/image_310p_ubuntu.yml
index 02a59fb..638c0e3 100644
--- a/.github/workflows/image_310p_ubuntu.yml
+++ b/.github/workflows/image_310p_ubuntu.yml
@@ -1,4 +1,4 @@
-name: 'image / Ubuntu'
+name: 'image / Ubuntu / 310p'
 # This is a docker build check and publish job:
 # 1. PR Triggered docker image build check
 #   - is for image build check
@@ -16,7 +16,7 @@ on:
       - 'main'
       - '*-dev'
     paths:
-      - '.github/workflows/image_310p.ubuntu.yml'
+      - '.github/workflows/image_310p_ubuntu.yml'
       - 'Dockerfile.310p'
       - 'vllm_ascend/**'
       - 'setup.py'
diff --git a/.github/workflows/vllm_ascend_test.yaml b/.github/workflows/vllm_ascend_test.yaml
index 73b4635..f2bb8a4 100644
--- a/.github/workflows/vllm_ascend_test.yaml
+++ b/.github/workflows/vllm_ascend_test.yaml
@@ -66,6 +66,8 @@ concurrency:
 
 jobs:
   lint:
+    # Only trigger lint on pull request
+    if: ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -131,7 +133,8 @@ jobs:
   ut:
     needs: [lint]
     name: unit test
-    if: ${{ needs.lint.result == 'success' }}
+    # only trigger e2e test on [pull request after lint passed] and [merged commit]
+    if: ${{ needs.lint.result == 'success' || github.event_name == 'push' }}
     runs-on: ubuntu-latest
     container:
       image: m.daocloud.io/quay.io/ascend/cann:8.1.rc1-910b-ubuntu22.04-py3.10
@@ -191,7 +194,8 @@ jobs:
 
   e2e:
     needs: [lint]
-    if: ${{ needs.lint.result == 'success' }}
+    # only trigger e2e test on pull request after lint passed
+    if: ${{ needs.lint.result == 'success' && github.event_name == 'pull_request' }}
     strategy:
       max-parallel: 2
       matrix: