[CI] optimize lint term (#5986)
### What this PR does / why we need it?
This patch purpose to optimize the lint check term. The main idea is to
reduce unnecessary installation time.
1. The installation of vllm is not must, only append the path of vllm
src to the `PATHONPATH` is effective
2. This installation of `requirements-dev.txt` is not must, we have a
pre-built image `quay.io/ascend-ci/vllm-ascend:lint` with all the
requirements installed in advance.
**NOTE**: the conditions for triggering image builds are: 1).Daily
scheduled build; 2) Build when requirements are modified; 3) Manual
build. This ensures that the dependencies in our image are up-to-date to
the greatest extent possible.
3. The `mypy` was separated from the `pre-commit` hook for performance
reasons; we found that integrating `mypy` into the `pre-commit` hook
resulted in poor performance.
4. Reduce the CPU core consumption from 16 -> 8
### Does this PR introduce _any_ user-facing change?
The end-to-end lint time was optimized from 20min/per PR to 8min/per PR
### How was this patch tested?
- vLLM version: v0.13.0
- vLLM main:
2c24bc6996
---------
Signed-off-by: wangli <wangli858794774@gmail.com>
This commit is contained in:
39
.github/workflows/_pre_commit.yml
vendored
39
.github/workflows/_pre_commit.yml
vendored
@@ -12,19 +12,16 @@ permissions:
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: linux-amd64-cpu-16-hk
|
||||
runs-on: linux-amd64-cpu-8-hk
|
||||
container:
|
||||
# Build it from https://github.com/nv-action/vllm-benchmarks/blob/main/Dockerfile
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-cpu
|
||||
env:
|
||||
GOPROXY: https://goproxy.io,direct
|
||||
GITHUB_WORKSPACE: /__w/vllm-ascend/vllm-ascend
|
||||
image: quay.io/ascend-ci/vllm-ascend:lint
|
||||
steps:
|
||||
- name: Checkout vllm-project/vllm-ascend repo
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# With problem matchers in a container, the output of $GITHUB_WORKSPACE and ${{ github.workspace }} are different.
|
||||
# So we will just copy it into the path ${{ github.workspace }}. see https://github.com/actions/runner/issues/2058
|
||||
# So we will just copy it into a temp path. see https://github.com/actions/runner/issues/2058
|
||||
- name: cp problem matchers
|
||||
run: |
|
||||
cp .github/workflows/matchers/actionlint.json "$RUNNER_TEMP/actionlint.json"
|
||||
@@ -41,14 +38,21 @@ jobs:
|
||||
repository: vllm-project/vllm
|
||||
path: ./vllm-empty
|
||||
ref: ${{ inputs.vllm }}
|
||||
- name: Install vllm
|
||||
working-directory: vllm-empty
|
||||
run: |
|
||||
VLLM_TARGET_DEVICE=empty python3 -m pip install . --extra-index https://download.pytorch.org/whl/cpu/
|
||||
- name: Install vllm-ascend dev
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
lint_tracker:
|
||||
- 'requirements.txt'
|
||||
- 'requirements-dev.txt'
|
||||
- 'requirements-lint.txt'
|
||||
|
||||
- name: Install vllm-ascend dev (conditional)
|
||||
if: steps.filter.outputs.lint_tracker == 'true'
|
||||
run: |
|
||||
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
|
||||
pip install -r requirements-dev.txt --extra-index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
- name: Run pre-commit
|
||||
env:
|
||||
PRE_COMMIT_COLOR: always
|
||||
@@ -56,4 +60,17 @@ jobs:
|
||||
TERM: xterm-256color
|
||||
SHELLCHECK_OPTS: "--exclude=SC2046,SC2006,SC2086" # Exclude SC2046, SC2006, SC2086 for actionlint
|
||||
run: |
|
||||
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
|
||||
pre-commit run --all-files --hook-stage manual --show-diff-on-failure
|
||||
- name: Run mypy
|
||||
run: |
|
||||
PYTHONPATH="$PYTHONPATH:$(pwd)/vllm-empty"
|
||||
export PYTHONPATH
|
||||
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
|
||||
# Run mypy for Python 3.10, 3.11, 3.12 manually
|
||||
# Note: We are now separating mypy from pre-commit hooks for performance reasons.
|
||||
for python_version in "3.10" "3.11" "3.12"; do
|
||||
echo "============================"
|
||||
tools/mypy.sh 1 "$python_version"
|
||||
echo "============================"
|
||||
done
|
||||
|
||||
46
.github/workflows/dockerfiles/Dockerfile.lint
vendored
Normal file
46
.github/workflows/dockerfiles/Dockerfile.lint
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
#
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# This file is a part of the vllm-ascend project.
|
||||
#
|
||||
|
||||
FROM ascendai/python:3.11-ubuntu22.04
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apt-get update -y && \
|
||||
apt-get install -y curl git gcc g++ cmake libnuma-dev jq && \
|
||||
rm -rf /var/cache/apt/* && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
|
||||
# For lint purpose, actually we need make a main2main matching.
|
||||
ARG VLLM_COMMIT=d68209402ddab3f54a09bc1f4de9a9495a283b60
|
||||
RUN git clone $VLLM_REPO /vllm-workspace/vllm && \
|
||||
cd /vllm-workspace/vllm && \
|
||||
git checkout $VLLM_COMMIT
|
||||
|
||||
# # Install vLLM common dependencies
|
||||
RUN python3 -m pip install -r /vllm-workspace/vllm/requirements/common.txt --extra-index https://download.pytorch.org/whl/cpu/ && \
|
||||
python3 -m pip uninstall -y triton && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
COPY . /vllm-workspace/vllm-ascend/
|
||||
|
||||
RUN pip install -r /vllm-workspace/vllm-ascend/requirements-dev.txt --extra-index-url https://download.pytorch.org/whl/cpu && \
|
||||
pip cache purge && \
|
||||
rm -fr /vllm-workspace/
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
67
.github/workflows/schedule_lint_image_build.yaml
vendored
Normal file
67
.github/workflows/schedule_lint_image_build.yaml
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
name: 'Image build lint'
|
||||
on:
|
||||
schedule:
|
||||
# Runs at 00:00 UTC+8 every day
|
||||
- cron: '0 20 * * *'
|
||||
workflow_dispatch:
|
||||
push:
|
||||
paths:
|
||||
- 'Dockerfile.lint'
|
||||
- 'requirements-lint.txt'
|
||||
- 'requirements-dev.txt'
|
||||
- 'requirements.txt'
|
||||
|
||||
# only cancel in-progress runs of the same workflow
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: vllm-ascend lint image build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Print
|
||||
run: |
|
||||
lscpu
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
quay.io/ascend-ci/vllm-ascend
|
||||
tags: lint
|
||||
flavor:
|
||||
latest=false
|
||||
|
||||
- name: Build - Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Build - Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Publish - Login to Quay Container Registry
|
||||
if: ${{ github.repository_owner == 'vllm-project' }}
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: quay.io
|
||||
username: ${{ vars.QUAY_CI_USERNAME }}
|
||||
password: ${{ secrets.QUAY_CI_PASSWORD }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
# For now, we only build amd64 lint image
|
||||
platforms: 'linux/amd64'
|
||||
context: .
|
||||
file: .github/workflows/dockerfiles/Dockerfile.lint
|
||||
push: true
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
provenance: false
|
||||
Reference in New Issue
Block a user