[accuracy test]Update cann version and huggingface-hub version for Qwen3 (#823)
### What this PR does / why we need it? 1. update cann version to 8.1.0 for multimodal 2. fix huggingface-hub version to adapt to qwen3 3. change Qwen3-8B to Qwen-8B-Base, Signed-off-by: hfadzxy <starmoon_zhang@163.com>
This commit is contained in:
hfadzxy
18
.github/workflows/accuracy_report.yaml
vendored
18
.github/workflows/accuracy_report.yaml
vendored
@@ -70,12 +70,12 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Query artifact run id for Qwen3-8B V0 latest artifact
|
||||
id: get_Qwen3_8B_latest_run_id_V0
|
||||
- name: Query artifact run id for Qwen3-8B-Base V0 latest artifact
|
||||
id: get_Qwen3_8B_Base_latest_run_id_V0
|
||||
run: |
|
||||
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
|
||||
RUN_ID=$(echo "$ARTIFACT_JSON" | \
|
||||
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
|
||||
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
|
||||
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -107,14 +107,14 @@ jobs:
|
||||
repository: vllm-project/vllm-ascend
|
||||
run-id: ${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }}
|
||||
|
||||
- name: Download Qwen/Qwen3-8B Artifact
|
||||
- name: Download Qwen/Qwen3-8B-Base Artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-V0-report
|
||||
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report
|
||||
path: ./docs/source/developer_guide/evaluation/accuracy_report
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
repository: vllm-project/vllm-ascend
|
||||
run-id: ${{ steps.get_Qwen3_8B_latest_run_id_V0.outputs.runid }}
|
||||
run-id: ${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }}
|
||||
|
||||
- name: Display Files
|
||||
working-directory: ./docs/source/developer_guide/evaluation/accuracy_report
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
cat ./Qwen2.5-VL-7B-Instruct.md
|
||||
cat ./Llama-3.1-8B-Instruct.md
|
||||
cat ./Qwen2.5-7B-Instruct.md
|
||||
cat ./Qwen3-8B.md
|
||||
cat ./Qwen3-8B-Base.md
|
||||
|
||||
- name: Create Pull Request for markdown update
|
||||
uses: peter-evans/create-pull-request@v7
|
||||
@@ -141,10 +141,10 @@ jobs:
|
||||
- [Qwen2.5-7B-Instruct accuracy report][2]
|
||||
- [Llama-3.1-8B-Instruct accuracy report][3]
|
||||
- [Qwen2.5-VL-7B-Instruct accuracy report][4]
|
||||
- [Qwen3-8B accuracy report][5]
|
||||
- [Qwen3-8B-Base accuracy report][5]
|
||||
|
||||
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
[2]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id_V0.outputs.runid }}
|
||||
[3]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }}
|
||||
[4]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0.outputs.runid }}
|
||||
[5]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen3_8B_latest_run_id_V0.outputs.runid }}
|
||||
[5]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }}
|
||||
18
.github/workflows/accuracy_test.yaml
vendored
18
.github/workflows/accuracy_test.yaml
vendored
@@ -29,7 +29,7 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
models:
|
||||
description: 'choose model(all/Qwen2.5-7B-Instruct/Llama-3.1-8B-Instruct/Qwen2.5-VL-7B-Instruct/Qwen3-8B)'
|
||||
description: 'choose model(all/Qwen2.5-7B-Instruct/Llama-3.1-8B-Instruct/Qwen2.5-VL-7B-Instruct/Qwen3-8B-Base)'
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
@@ -37,7 +37,7 @@ on:
|
||||
- Qwen/Qwen2.5-7B-Instruct
|
||||
- meta-llama/Llama-3.1-8B-Instruct
|
||||
- Qwen/Qwen2.5-VL-7B-Instruct
|
||||
- Qwen/Qwen3-8B
|
||||
- Qwen/Qwen3-8B-Base
|
||||
default: 'all'
|
||||
|
||||
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
|
||||
@@ -54,16 +54,16 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
include: ${{ fromJSON(
|
||||
(github.event.inputs.models == 'all' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"},{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"},{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}, {"model_name":"Qwen/Qwen3-8B","output_file":"Qwen3-8B"}]') ||
|
||||
(github.event.inputs.models == 'all' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"},{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"},{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}, {"model_name":"Qwen/Qwen3-8B-Base","output_file":"Qwen3-8B-Base"}]') ||
|
||||
(github.event.inputs.models == 'Qwen/Qwen2.5-7B-Instruct' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"}]') ||
|
||||
(github.event.inputs.models == 'meta-llama/Llama-3.1-8B-Instruct' && '[{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"}]') ||
|
||||
(github.event.inputs.models == 'Qwen/Qwen2.5-VL-7B-Instruct' && '[{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}]') ||
|
||||
(github.event.inputs.models == 'Qwen/Qwen3-8B' && '[{"model_name":"Qwen/Qwen3-8B","output_file":"Qwen3-8B"}]')
|
||||
(github.event.inputs.models == 'Qwen/Qwen3-8B-Base' && '[{"model_name":"Qwen/Qwen3-8B-Base","output_file":"Qwen3-8B-Base"}]')
|
||||
) }}
|
||||
fail-fast: false
|
||||
|
||||
container:
|
||||
image: quay.io/ascend/cann:8.0.0-910b-ubuntu22.04-py3.10
|
||||
image: m.daocloud.io/quay.io/ascend/cann:8.1.rc1-910b-ubuntu22.04-py3.10
|
||||
env:
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
@@ -86,12 +86,6 @@ jobs:
|
||||
apt install git -y
|
||||
git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
apt-get -y install `cat packages.txt`
|
||||
apt-get -y install gcc g++ cmake libnuma-dev
|
||||
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
apt-get -y install `cat packages.txt`
|
||||
@@ -147,7 +141,7 @@ jobs:
|
||||
working-directory: ./lm-eval
|
||||
run: |
|
||||
pip install -e .
|
||||
pip install ray datasets==2.16.0 transformers==4.50.3 huggingface-hub==0.29.3
|
||||
pip install ray datasets==2.16.0
|
||||
|
||||
- name: Collect version info
|
||||
run: |
|
||||
|
||||
Reference in New Issue
Block a user