EngineX/xc-llm-ascend
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[CI] Add daily images build for nightly ci (#3989)

Browse Source 
### What this PR does / why we need it?
Given the current excessively long build time of our nightly-ci, I
recommend installing necessary, confirmed versions of packages in the
Docker image to reduce the time required for integration testing.
Including Mooncake vllm with fixed tags, This is expected to reduce
nightly-ci duration by 2 hours.

- vLLM version: v0.11.0
- vLLM main:
2918c1b49c

---------

Signed-off-by: wangli <wangli858794774@gmail.com>
This commit is contained in:
Li Wang
2025-11-13 20:10:12 +08:00
committed by GitHub
parent f7d1f73b98
commit 7294f89e43
11 changed files with 285 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
tests/e2e/nightly/multi_node/scripts/build_mooncake.sh
View File

@@ -1,112 +0,0 @@
#!/bin/bash
set -e
set -o pipefail
GREEN="\033[0;32m"
BLUE="\033[0;34m"
YELLOW="\033[0;33m"
RED="\033[0;31m"
NC="\033[0m" # No Color
branch=${1:-v0.3.7.post2}
repo_url="https://github.com/kvcache-ai/Mooncake"
repo_name="Mooncake"
state_file=".build_state"
echo "[INFO] Branch: $branch"
echo "-------------------------------------------"
mark_done() { echo "$1" >> "$state_file"; }
is_done() { grep -Fxq "$1" "$state_file" 2>/dev/null; }
if ! is_done "clone"; then
echo "[STEP] Clone repository..."
if [ -d "$repo_name" ]; then
echo "[WARN] Directory $repo_name already exists, skipping clone."
else
git clone --branch "$branch" --depth 1 "$repo_url" "$repo_name"
fi
mark_done "clone"
else
echo "[SKIP] Clone step already done."
fi
init_ascend_env() {
cann_in_sys_path=/usr/local/Ascend/ascend-toolkit; \
cann_in_user_path=$HOME/Ascend/ascend-toolkit; \
uname_m=$(uname -m) && \
if [ -f "${cann_in_sys_path}/set_env.sh" ]; then \
source ${cann_in_sys_path}/set_env.sh; \
export LD_LIBRARY_PATH=${cann_in_sys_path}/latest/lib64:${cann_in_sys_path}/latest/${uname_m}-linux/devlib:${LD_LIBRARY_PATH} ; \
elif [ -f "${cann_in_user_path}/set_env.sh" ]; then \
source "$HOME/Ascend/ascend-toolkit/set_env.sh"; \
export LD_LIBRARY_PATH=${cann_in_user_path}/latest/lib64:${cann_in_user_path}/latest/${uname_m}-linux/devlib:${LD_LIBRARY_PATH}; \
else \
echo "No Ascend Toolkit found"; \
exit 1; \
fi
}
init_ascend_env
if ! is_done "deps"; then
cd "$repo_name"
echo "[STEP]Installing dependencies..."
sed -i 's|https://go.dev/dl/|https://golang.google.cn/dl/|g' dependencies.sh
bash dependencies.sh -y
cd ..
mark_done "deps"
else
echo "[SKIP] Dependencies already installed."
fi
if ! is_done "mpi"; then
echo "[STEP] Install MPI..."
apt purge -y mpich libmpich-dev openmpi-bin libopenmpi-dev || true
apt install -y mpich libmpich-dev
export CPATH=/usr/lib/aarch64-linux-gnu/mpich/include/:${CPATH:-}
export CPATH=/usr/lib/aarch64-linux-gnu/openmpi/lib:${CPATH:-}
mark_done "mpi"
else
echo "[SKIP] MPI installation already done."
fi
if ! is_done "build"; then
echo "[STEP] Compile and install..."
cd "$repo_name"
if [ -d "build" ]; then
echo "[INFO] Removing existing build directory..."
rm -rf build
fi
mkdir build && cd build
cmake .. -USE_ASCEND_DIRECT=ON || { echo "[ERROR] cmake failed."; exit 1; }
make -j || { echo "[ERROR] make failed."; exit 1; }
make install || { echo "[ERROR] make install failed."; exit 1; }
mark_done "build"
else
echo "[SKIP] Build already done."
fi
if ! grep -q "export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/python/site-packages:$LD_LIBRARY_PATH" ~/.bashrc; then
echo -e "${YELLOW}Adding LD_LIBRARY_PATH to your PATH in ~/.bashrc${NC}"
echo 'export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/python/site-packages:$LD_LIBRARY_PATH' >> ~/.bashrc
echo -e "${YELLOW}Please run 'source ~/.bashrc' or start a new terminal${NC}"
fi
export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/python/site-packages:$LD_LIBRARY_PATH
echo "=========================================="
echo -e "${GREEN}[SUCCESS] Mooncake build completed!"
echo "You can rerun this script anytime — it will resume from the last step."
echo "=========================================="
echo "Example startup command:"
echo "mooncake_master --eviction_high_watermark_ratio 0.8 --eviction_ratio 0.05 --port 50088"

12
tests/e2e/nightly/multi_node/scripts/lws.yaml.jinja2
View File

@@ -15,15 +15,13 @@ spec:
spec:
containers:
- name: vllm-leader
image: {{ image | default("m.daocloud.io/quay.io/ascend/cann:8.3.rc1-a3-ubuntu22.04-py3.11") }}
image: {{ image | default("swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3") }}
env:
- name: CONFIG_YAML_PATH
value: {{ config_file_path | default("DeepSeek-V3.yaml") }}
- name: WORKSPACE
value: "/root/workspace"
value: "/vllm-workspace"
# Set vLLM version and vLLM-Ascend version here, once there is a new release, update here.
- name: VLLM_VERSION
value: "v0.11.0"
- name: VLLM_ASCEND_VERSION
value: {{ vllm_ascend_ref | default("main") }}
- name: VLLM_ASCEND_REMOTE_URL
@@ -75,15 +73,13 @@ spec:
spec:
containers:
- name: vllm-worker
image: {{ image | default("m.daocloud.io/quay.io/ascend/cann:8.3.rc1-a3-ubuntu22.04-py3.11") }}
image: {{ image | default("swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-a3") }}
env:
- name: CONFIG_YAML_PATH
value: {{ config_file_path | default("DeepSeek-V3.yaml") }}
- name: WORKSPACE
value: "/root/workspace"
value: "/vllm-workspace"
# Set vLLM version and vLLM-Ascend version here, once there is a new release, update here.
- name: VLLM_VERSION
value: "v0.11.0"
- name: VLLM_ASCEND_VERSION
value: {{ vllm_ascend_ref | default("main") }}
- name: VLLM_ASCEND_REMOTE_URL

139
tests/e2e/nightly/multi_node/scripts/run.sh
View File

@@ -11,8 +11,8 @@ NC="\033[0m" # No Color
# Configuration
LOG_DIR="/root/.cache/tests/logs"
OVERWRITE_LOGS=true
SRC_DIR="$WORKSPACE/source_code"
export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/python/site-packages:$LD_LIBRARY_PATH
export BENCHMARK_HOME=${WORKSPACE}/vllm-ascend/benchmark
# Function to print section headers
print_section() {
@@ -35,18 +35,50 @@ print_error() {
exit 1
}
# Function to check command success
check_success() {
if [ $? -ne 0 ]; then
print_error "$1"
show_vllm_info() {
cd "$WORKSPACE"
echo "Installed vLLM-related Python packages:"
pip list | grep vllm || echo "No vllm packages found."
echo ""
echo "============================"
echo "vLLM Git information"
echo "============================"
cd vllm
if [ -d .git ]; then
echo "Branch: $(git rev-parse --abbrev-ref HEAD)"
echo "Commit hash: $(git rev-parse HEAD)"
echo "Author: $(git log -1 --pretty=format:'%an <%ae>')"
echo "Date: $(git log -1 --pretty=format:'%ad' --date=iso)"
echo "Message: $(git log -1 --pretty=format:'%s')"
echo "Tags: $(git tag --points-at HEAD || echo 'None')"
echo "Remote: $(git remote -v | head -n1)"
echo ""
else
echo "No .git directory found in vllm"
fi
cd ..
echo ""
echo "============================"
echo "vLLM-Ascend Git information"
echo "============================"
cd vllm-ascend
if [ -d .git ]; then
echo "Branch: $(git rev-parse --abbrev-ref HEAD)"
echo "Commit hash: $(git rev-parse HEAD)"
echo "Author: $(git log -1 --pretty=format:'%an <%ae>')"
echo "Date: $(git log -1 --pretty=format:'%ad' --date=iso)"
echo "Message: $(git log -1 --pretty=format:'%s')"
echo "Tags: $(git tag --points-at HEAD || echo 'None')"
echo "Remote: $(git remote -v | head -n1)"
echo ""
else
echo "No .git directory found in vllm-ascend"
fi
cd ..
}
if [ $(id -u) -ne 0 ]; then
print_error "Require root permission, try sudo ./dependencies.sh"
fi
check_npu_info() {
echo "====> Check NPU info"
npu-smi info
@@ -60,79 +92,6 @@ check_and_config() {
export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
}
checkout_src() {
echo "====> Checkout source code"
mkdir -p "$SRC_DIR"
# vllm-ascend
if [ ! -d "$SRC_DIR/vllm-ascend" ]; then
git clone --depth 1 -b $VLLM_ASCEND_VERSION $VLLM_ASCEND_REMOTE_URL "$SRC_DIR/vllm-ascend"
fi
# vllm
if [ ! -d "$SRC_DIR/vllm" ]; then
git clone -b $VLLM_VERSION https://github.com/vllm-project/vllm.git "$SRC_DIR/vllm"
fi
}
install_sys_dependencies() {
echo "====> Install system dependencies"
apt-get update -y
DEP_LIST=()
while IFS= read -r line; do
[[ -n "$line" && ! "$line" =~ ^# ]] && DEP_LIST+=("$line")
done < "$SRC_DIR/vllm-ascend/packages.txt"
apt-get install -y "${DEP_LIST[@]}" gcc g++ cmake libnuma-dev iproute2
}
install_vllm() {
echo "====> Install vllm and vllm-ascend"
VLLM_TARGET_DEVICE=empty pip install -e "$SRC_DIR/vllm"
pip install -e "$SRC_DIR/vllm-ascend"
pip install modelscope
# Install for pytest
pip install -r "$SRC_DIR/vllm-ascend/requirements-dev.txt"
}
install_ais_bench() {
local AIS_BENCH="$SRC_DIR/vllm-ascend/benchmark"
git clone https://gitee.com/aisbench/benchmark.git $AIS_BENCH
cd $AIS_BENCH
git checkout v3.0-20250930-master
pip3 install -e ./
pip3 install -r requirements/api.txt
pip3 install -r requirements/extra.txt
cd -
}
install_extra_components() {
echo "====> Installing extra components for DeepSeek-v3.2-exp-bf16"
if ! wget -q https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/a3/CANN-custom_ops-sfa-linux.aarch64.run; then
echo "Failed to download CANN-custom_ops-sfa-linux.aarch64.run"
return 1
fi
chmod +x ./CANN-custom_ops-sfa-linux.aarch64.run
./CANN-custom_ops-sfa-linux.aarch64.run --quiet
if ! wget -q https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/a3/custom_ops-1.0-cp311-cp311-linux_aarch64.whl; then
echo "Failed to download custom_ops wheel"
return 1
fi
pip install custom_ops-1.0-cp311-cp311-linux_aarch64.whl
export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize:${ASCEND_CUSTOM_OPP_PATH}
export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH}
source /usr/local/Ascend/ascend-toolkit/set_env.sh
EOF
rm -f CANN-custom_ops-sfa-linux.aarch64.run \
custom_ops-1.0-cp311-cp311-linux_aarch64.whl
echo "====> Extra components installation completed"
}
kill_npu_processes() {
pgrep python3 | xargs -r kill -9
pgrep VLLM | xargs -r kill -9
@@ -163,17 +122,9 @@ run_tests_with_log() {
main() {
check_npu_info
check_and_config
checkout_src
install_sys_dependencies
install_vllm
if [[ "$CONFIG_YAML_PATH" == *"DeepSeek-V3_2-Exp-bf16.yaml" ]]; then
install_extra_components
fi
install_ais_bench
cd "$WORKSPACE/source_code"
. $SRC_DIR/vllm-ascend/tests/e2e/nightly/multi_node/scripts/build_mooncake.sh
cd "$WORKSPACE/source_code/vllm-ascend"
show_vllm_info
cd "$WORKSPACE/vllm-ascend"
run_tests_with_log
}
main "$@"
main "$@"