From 4a2505f81f5b5f3eedeb58f08ab522e2c69aa4a5 Mon Sep 17 00:00:00 2001 From: hfadzxy <59153331+hfadzxy@users.noreply.github.com> Date: Mon, 12 May 2025 19:12:48 +0800 Subject: [PATCH] [accuracy test]Update cann version and huggingface-hub version for Qwen3 (#823) ### What this PR does / why we need it? 1. update cann version to 8.1.0 for multimodal 2. fix huggingface-hub version to adapt to qwen3 3. change Qwen3-8B to Qwen-8B-Base, Signed-off-by: hfadzxy --- .github/workflows/accuracy_report.yaml | 18 +++++++++--------- .github/workflows/accuracy_test.yaml | 18 ++++++------------ 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.github/workflows/accuracy_report.yaml b/.github/workflows/accuracy_report.yaml index 6da1f84..57960b7 100644 --- a/.github/workflows/accuracy_report.yaml +++ b/.github/workflows/accuracy_report.yaml @@ -70,12 +70,12 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Query artifact run id for Qwen3-8B V0 latest artifact - id: get_Qwen3_8B_latest_run_id_V0 + - name: Query artifact run id for Qwen3-8B-Base V0 latest artifact + id: get_Qwen3_8B_Base_latest_run_id_V0 run: | ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts") RUN_ID=$(echo "$ARTIFACT_JSON" | \ - jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-V0-report")] | sort_by(.created_at) | last | .workflow_run.id') + jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report")] | sort_by(.created_at) | last | .workflow_run.id') echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -107,14 +107,14 @@ jobs: repository: vllm-project/vllm-ascend run-id: ${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }} - - name: Download Qwen/Qwen3-8B Artifact + - name: Download Qwen/Qwen3-8B-Base Artifact uses: actions/download-artifact@v4 with: - name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-V0-report + name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report path: ./docs/source/developer_guide/evaluation/accuracy_report github-token: ${{ secrets.GITHUB_TOKEN }} repository: vllm-project/vllm-ascend - run-id: ${{ steps.get_Qwen3_8B_latest_run_id_V0.outputs.runid }} + run-id: ${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }} - name: Display Files working-directory: ./docs/source/developer_guide/evaluation/accuracy_report @@ -122,7 +122,7 @@ jobs: cat ./Qwen2.5-VL-7B-Instruct.md cat ./Llama-3.1-8B-Instruct.md cat ./Qwen2.5-7B-Instruct.md - cat ./Qwen3-8B.md + cat ./Qwen3-8B-Base.md - name: Create Pull Request for markdown update uses: peter-evans/create-pull-request@v7 @@ -141,10 +141,10 @@ jobs: - [Qwen2.5-7B-Instruct accuracy report][2] - [Llama-3.1-8B-Instruct accuracy report][3] - [Qwen2.5-VL-7B-Instruct accuracy report][4] - - [Qwen3-8B accuracy report][5] + - [Qwen3-8B-Base accuracy report][5] [1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} [2]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id_V0.outputs.runid }} [3]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }} [4]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0.outputs.runid }} - [5]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen3_8B_latest_run_id_V0.outputs.runid }} \ No newline at end of file + [5]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }} \ No newline at end of file diff --git a/.github/workflows/accuracy_test.yaml b/.github/workflows/accuracy_test.yaml index 33f3be7..f5f8d32 100644 --- a/.github/workflows/accuracy_test.yaml +++ b/.github/workflows/accuracy_test.yaml @@ -29,7 +29,7 @@ on: required: true type: string models: - description: 'choose model(all/Qwen2.5-7B-Instruct/Llama-3.1-8B-Instruct/Qwen2.5-VL-7B-Instruct/Qwen3-8B)' + description: 'choose model(all/Qwen2.5-7B-Instruct/Llama-3.1-8B-Instruct/Qwen2.5-VL-7B-Instruct/Qwen3-8B-Base)' required: true type: choice options: @@ -37,7 +37,7 @@ on: - Qwen/Qwen2.5-7B-Instruct - meta-llama/Llama-3.1-8B-Instruct - Qwen/Qwen2.5-VL-7B-Instruct - - Qwen/Qwen3-8B + - Qwen/Qwen3-8B-Base default: 'all' # Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly @@ -54,16 +54,16 @@ jobs: strategy: matrix: include: ${{ fromJSON( - (github.event.inputs.models == 'all' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"},{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"},{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}, {"model_name":"Qwen/Qwen3-8B","output_file":"Qwen3-8B"}]') || + (github.event.inputs.models == 'all' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"},{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"},{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}, {"model_name":"Qwen/Qwen3-8B-Base","output_file":"Qwen3-8B-Base"}]') || (github.event.inputs.models == 'Qwen/Qwen2.5-7B-Instruct' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"}]') || (github.event.inputs.models == 'meta-llama/Llama-3.1-8B-Instruct' && '[{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"}]') || (github.event.inputs.models == 'Qwen/Qwen2.5-VL-7B-Instruct' && '[{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}]') || - (github.event.inputs.models == 'Qwen/Qwen3-8B' && '[{"model_name":"Qwen/Qwen3-8B","output_file":"Qwen3-8B"}]') + (github.event.inputs.models == 'Qwen/Qwen3-8B-Base' && '[{"model_name":"Qwen/Qwen3-8B-Base","output_file":"Qwen3-8B-Base"}]') ) }} fail-fast: false container: - image: quay.io/ascend/cann:8.0.0-910b-ubuntu22.04-py3.10 + image: m.daocloud.io/quay.io/ascend/cann:8.1.rc1-910b-ubuntu22.04-py3.10 env: HF_ENDPOINT: https://hf-mirror.com HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -86,12 +86,6 @@ jobs: apt install git -y git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/ - - name: Install system dependencies - run: | - apt-get -y install `cat packages.txt` - apt-get -y install gcc g++ cmake libnuma-dev - - - name: Install system dependencies run: | apt-get -y install `cat packages.txt` @@ -147,7 +141,7 @@ jobs: working-directory: ./lm-eval run: | pip install -e . - pip install ray datasets==2.16.0 transformers==4.50.3 huggingface-hub==0.29.3 + pip install ray datasets==2.16.0 - name: Collect version info run: |