diff --git a/CMakeLists.txt b/CMakeLists.txt index b44d89d..561c99a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,7 +77,8 @@ message("TORCH_NPU_PATH is ${TORCH_NPU_PATH}") if(SOC_VERSION STREQUAL "ASCEND310P3") file(GLOB VLLM_ASCEND_SRC - ${CMAKE_CURRENT_SOURCE_DIR}/csrc/*.cpp) + ${CMAKE_CURRENT_SOURCE_DIR}/csrc/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/csrc/idle_offload/shm_worker.cpp) else() file(GLOB VLLM_ASCEND_SRC ${CMAKE_CURRENT_SOURCE_DIR}/csrc/*.cpp diff --git a/Dockerfile.310p b/Dockerfile.310p index 93ec8ab..710db19 100644 --- a/Dockerfile.310p +++ b/Dockerfile.310p @@ -51,9 +51,15 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi source /usr/local/Ascend/nnal/atb/set_env.sh && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ export SOC_VERSION=ASCEND310P3 && \ + cd /vllm-workspace/vllm-ascend/csrc/idle_offload && \ + make install && make clean && \ python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ python3 -m pip cache purge +ENV VLLM_ASCEND_ENABLE_NZ=0 \ + VLLM_WORKER_MULTIPROC_METHOD=spawn \ + VLLM_ASCEND_ENABLE_IDLE_OFFLOAD=1 + # Install modelscope (for fast download) and ray (for multinode) RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \ python3 -m pip cache purge diff --git a/Dockerfile.310p.openEuler b/Dockerfile.310p.openEuler index 9e7727a..bebed5d 100644 --- a/Dockerfile.310p.openEuler +++ b/Dockerfile.310p.openEuler @@ -49,9 +49,15 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/`uname -i`-openEuler-linux && \ export SOC_VERSION=ASCEND310P3 && \ + cd /vllm-workspace/vllm-ascend/csrc/idle_offload && \ + make install && make clean && \ python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ python3 -m pip cache purge +ENV VLLM_ASCEND_ENABLE_NZ=0 \ + VLLM_WORKER_MULTIPROC_METHOD=spawn \ + VLLM_ASCEND_ENABLE_IDLE_OFFLOAD=1 + # Install modelscope (for fast download) and ray (for multinode) RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \ python3 -m pip cache purge diff --git a/Dockerfile.a3 b/Dockerfile.a3 index 4328e72..1037ce0 100644 --- a/Dockerfile.a3 +++ b/Dockerfile.a3 @@ -20,11 +20,13 @@ FROM quay.io/ascend/cann:8.3.rc2-a3-ubuntu22.04-py3.11 ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" ARG COMPILE_CUSTOM_KERNELS=1 ARG MOONCAKE_TAG=v0.3.7.post2 +ARG SOC_VERSION="ascend910_9391" COPY . /vllm-workspace/vllm-ascend/ # Define environments ENV DEBIAN_FRONTEND=noninteractive -ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} +ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} \ + SOC_VERSION=$SOC_VERSION RUN pip config set global.index-url ${PIP_INDEX_URL} @@ -58,9 +60,15 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ source /usr/local/Ascend/nnal/atb/set_env.sh && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + cd /vllm-workspace/vllm-ascend/csrc/idle_offload && \ + make install && make clean && \ python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ python3 -m pip cache purge +ENV VLLM_ASCEND_ENABLE_NZ=0 \ + VLLM_WORKER_MULTIPROC_METHOD=spawn \ + VLLM_ASCEND_ENABLE_IDLE_OFFLOAD=1 + # Install modelscope (for fast download) and ray (for multinode) RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \ python3 -m pip cache purge diff --git a/Dockerfile.a3.openEuler b/Dockerfile.a3.openEuler index 8faae39..21c0631 100644 --- a/Dockerfile.a3.openEuler +++ b/Dockerfile.a3.openEuler @@ -20,8 +20,10 @@ FROM quay.io/ascend/cann:8.3.rc2-a3-openeuler24.03-py3.11 ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" ARG COMPILE_CUSTOM_KERNELS=1 ARG MOONCAKE_TAG="v0.3.7.post2" +ARG SOC_VERSION="ascend910_9391" -ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} +ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} \ + SOC_VERSION=${SOC_VERSION} RUN pip config set global.index-url ${PIP_INDEX_URL} @@ -61,9 +63,15 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi source /usr/local/Ascend/nnal/atb/set_env.sh && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/`uname -i`-openEuler-linux && \ + cd /vllm-workspace/vllm-ascend/csrc/idle_offload && \ + make install && make clean && \ python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ python3 -m pip cache purge +ENV VLLM_ASCEND_ENABLE_NZ=0 \ + VLLM_WORKER_MULTIPROC_METHOD=spawn \ + VLLM_ASCEND_ENABLE_IDLE_OFFLOAD=1 + # Install modelscope (for fast download) and ray (for multinode) RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \ python3 -m pip cache purge diff --git a/Dockerfile.openEuler b/Dockerfile.openEuler index 7f82994..d13855d 100644 --- a/Dockerfile.openEuler +++ b/Dockerfile.openEuler @@ -62,9 +62,15 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi source /usr/local/Ascend/nnal/atb/set_env.sh && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/`uname -i`-openEuler-linux && \ + cd /vllm-workspace/vllm-ascend/csrc/idle_offload && \ + make install && make clean && \ python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ python3 -m pip cache purge +ENV VLLM_ASCEND_ENABLE_NZ=0 \ + VLLM_WORKER_MULTIPROC_METHOD=spawn \ + VLLM_ASCEND_ENABLE_IDLE_OFFLOAD=1 + # Install modelscope (for fast download) and ray (for multinode) RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \ python3 -m pip cache purge diff --git a/Dockerfile.backup b/Dockerfile_backup/Dockerfile similarity index 100% rename from Dockerfile.backup rename to Dockerfile_backup/Dockerfile diff --git a/Dockerfile_backup/Dockerfile.310p b/Dockerfile_backup/Dockerfile.310p new file mode 100644 index 0000000..93ec8ab --- /dev/null +++ b/Dockerfile_backup/Dockerfile.310p @@ -0,0 +1,61 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is a part of the vllm-ascend project. +# + +FROM quay.io/ascend/cann:8.3.rc2-310p-ubuntu22.04-py3.11 + +ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +ARG COMPILE_CUSTOM_KERNELS=1 + +# Define environments +ENV DEBIAN_FRONTEND=noninteractive +ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} + +RUN apt-get update -y && \ + apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \ + rm -rf /var/cache/apt/* && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +COPY . /vllm-workspace/vllm-ascend/ + +RUN pip config set global.index-url ${PIP_INDEX_URL} + +# Install vLLM +ARG VLLM_REPO=https://github.com/vllm-project/vllm.git +ARG VLLM_TAG=v0.11.0 +RUN git clone --depth 1 $VLLM_REPO --branch $VLLM_TAG /vllm-workspace/vllm +# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it. +RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -v -e /vllm-workspace/vllm/[audio] --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip uninstall -y triton && \ + python3 -m pip cache purge + +# Install vllm-ascend +# Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH +RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + export SOC_VERSION=ASCEND310P3 && \ + python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip cache purge + +# Install modelscope (for fast download) and ray (for multinode) +RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \ + python3 -m pip cache purge + +CMD ["/bin/bash"] diff --git a/Dockerfile_backup/Dockerfile.310p.openEuler b/Dockerfile_backup/Dockerfile.310p.openEuler new file mode 100644 index 0000000..9e7727a --- /dev/null +++ b/Dockerfile_backup/Dockerfile.310p.openEuler @@ -0,0 +1,59 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is a part of the vllm-ascend project. +# + +FROM quay.io/ascend/cann:8.3.rc2-310p-openeuler24.03-py3.11 + +ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +ARG COMPILE_CUSTOM_KERNELS=1 + +ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} + +RUN yum update -y && \ + yum install -y python3-pip git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \ + rm -rf /var/cache/yum + +RUN pip config set global.index-url ${PIP_INDEX_URL} + +WORKDIR /workspace + +COPY . /vllm-workspace/vllm-ascend/ + +# Install vLLM +ARG VLLM_REPO=https://github.com/vllm-project/vllm.git +ARG VLLM_TAG=v0.11.0 + +RUN git clone --depth 1 $VLLM_REPO --branch $VLLM_TAG /vllm-workspace/vllm +# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it. +RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip uninstall -y triton && \ + python3 -m pip cache purge + +# Install vllm-ascend +RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/`uname -i`-openEuler-linux && \ + export SOC_VERSION=ASCEND310P3 && \ + python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip cache purge + +# Install modelscope (for fast download) and ray (for multinode) +RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \ + python3 -m pip cache purge + +CMD ["/bin/bash"] diff --git a/Dockerfile_backup/Dockerfile.a3 b/Dockerfile_backup/Dockerfile.a3 new file mode 100644 index 0000000..4328e72 --- /dev/null +++ b/Dockerfile_backup/Dockerfile.a3 @@ -0,0 +1,68 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is a part of the vllm-ascend project. +# + +FROM quay.io/ascend/cann:8.3.rc2-a3-ubuntu22.04-py3.11 + +ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +ARG COMPILE_CUSTOM_KERNELS=1 +ARG MOONCAKE_TAG=v0.3.7.post2 + +COPY . /vllm-workspace/vllm-ascend/ +# Define environments +ENV DEBIAN_FRONTEND=noninteractive +ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} + +RUN pip config set global.index-url ${PIP_INDEX_URL} + +WORKDIR /workspace + +# Install Mooncake dependencies +RUN apt-get update -y && \ + apt-get install -y git vim wget net-tools gcc g++ cmake libnuma-dev && \ + git clone --depth 1 --branch ${MOONCAKE_TAG} https://github.com/kvcache-ai/Mooncake /vllm-workspace/Mooncake && \ + cp /vllm-workspace/vllm-ascend/tools/mooncake_installer.sh /vllm-workspace/Mooncake/ && \ + cd /vllm-workspace/Mooncake && bash mooncake_installer.sh -y && \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/lib64 && \ + mkdir -p build && cd build && cmake .. -DUSE_ASCEND_DIRECT=ON && \ + make -j$(nproc) && make install && \ + rm -fr /vllm-workspace/Mooncake/build && \ + rm -rf /var/cache/apt/* && \ + rm -rf /var/lib/apt/lists/* + +# Install vLLM +ARG VLLM_REPO=https://github.com/vllm-project/vllm.git +ARG VLLM_TAG=v0.11.0 +RUN git clone --depth 1 $VLLM_REPO --branch $VLLM_TAG /vllm-workspace/vllm +# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it. +RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -v -e /vllm-workspace/vllm/[audio] --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip uninstall -y triton && \ + python3 -m pip cache purge + +# Install vllm-ascend +# Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH +RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip cache purge + +# Install modelscope (for fast download) and ray (for multinode) +RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \ + python3 -m pip cache purge + +CMD ["/bin/bash"] diff --git a/Dockerfile_backup/Dockerfile.a3.openEuler b/Dockerfile_backup/Dockerfile.a3.openEuler new file mode 100644 index 0000000..8faae39 --- /dev/null +++ b/Dockerfile_backup/Dockerfile.a3.openEuler @@ -0,0 +1,71 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is a part of the vllm-ascend project. +# + +FROM quay.io/ascend/cann:8.3.rc2-a3-openeuler24.03-py3.11 + +ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +ARG COMPILE_CUSTOM_KERNELS=1 +ARG MOONCAKE_TAG="v0.3.7.post2" + +ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} + +RUN pip config set global.index-url ${PIP_INDEX_URL} + +WORKDIR /workspace + +COPY . /vllm-workspace/vllm-ascend/ + +SHELL ["/bin/bash", "-c"] + +RUN yum update -y && \ + yum install -y git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \ + git clone --depth 1 --branch ${MOONCAKE_TAG} https://github.com/kvcache-ai/Mooncake /vllm-workspace/Mooncake && \ + cp /vllm-workspace/vllm-ascend/tools/mooncake_installer.sh /vllm-workspace/Mooncake/ && \ + ARCH=$(uname -m) && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/devlib:/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/lib64:$LD_LIBRARY_PATH && \ + export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/${ARCH}-openEuler-linux && \ + cd /vllm-workspace/Mooncake && \ + bash mooncake_installer.sh -y && \ + mkdir -p build && cd build && cmake .. -DUSE_ASCEND_DIRECT=ON && \ + make -j$(nproc) && make install && \ + rm -fr /vllm-workspace/Mooncake/build && \ + rm -rf /var/cache/yum/* + +# Install vLLM +ARG VLLM_REPO=https://github.com/vllm-project/vllm.git +ARG VLLM_TAG=v0.11.0 +RUN git clone --depth 1 $VLLM_REPO --branch $VLLM_TAG /vllm-workspace/vllm +# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it. +RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip uninstall -y triton && \ + python3 -m pip cache purge + +# Install vllm-ascend +RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/`uname -i`-openEuler-linux && \ + python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip cache purge + +# Install modelscope (for fast download) and ray (for multinode) +RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \ + python3 -m pip cache purge + +CMD ["/bin/bash"] diff --git a/Dockerfile_backup/Dockerfile.openEuler b/Dockerfile_backup/Dockerfile.openEuler new file mode 100644 index 0000000..7f82994 --- /dev/null +++ b/Dockerfile_backup/Dockerfile.openEuler @@ -0,0 +1,72 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is a part of the vllm-ascend project. +# + +FROM quay.io/ascend/cann:8.3.rc2-910b-openeuler24.03-py3.11 + +ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +ARG COMPILE_CUSTOM_KERNELS=1 +ARG MOONCAKE_TAG="v0.3.7.post2" + +ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS} + + +RUN pip config set global.index-url ${PIP_INDEX_URL} + +WORKDIR /workspace + +COPY . /vllm-workspace/vllm-ascend/ + +SHELL ["/bin/bash", "-c"] + +RUN yum update -y && \ + yum install -y git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \ + git clone --depth 1 --branch ${MOONCAKE_TAG} https://github.com/kvcache-ai/Mooncake /vllm-workspace/Mooncake && \ + cp /vllm-workspace/vllm-ascend/tools/mooncake_installer.sh /vllm-workspace/Mooncake/ && \ + ARCH=$(uname -m) && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/devlib:/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/lib64:$LD_LIBRARY_PATH && \ + export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/${ARCH}-openEuler-linux && \ + cd /vllm-workspace/Mooncake && \ + bash mooncake_installer.sh -y && \ + mkdir -p build && cd build && cmake .. -DUSE_ASCEND_DIRECT=ON && \ + make -j$(nproc) && make install && \ + rm -fr /vllm-workspace/Mooncake/build && \ + rm -rf /var/cache/yum/* + +# Install vLLM +ARG VLLM_REPO=https://github.com/vllm-project/vllm.git +ARG VLLM_TAG=v0.11.0 +RUN git clone --depth 1 $VLLM_REPO --branch $VLLM_TAG /vllm-workspace/vllm +# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it. +RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip uninstall -y triton && \ + python3 -m pip cache purge + +# Install vllm-ascend +RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/`uname -i`-openEuler-linux && \ + python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \ + python3 -m pip cache purge + +# Install modelscope (for fast download) and ray (for multinode) +RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \ + python3 -m pip cache purge + +CMD ["/bin/bash"] diff --git a/csrc/idle_offload/Makefile b/csrc/idle_offload/Makefile index 9edd05f..3cea2c3 100644 --- a/csrc/idle_offload/Makefile +++ b/csrc/idle_offload/Makefile @@ -6,7 +6,7 @@ ASCEND_HOME := /usr/local/Ascend/ascend-toolkit/latest INCLUDES := -I$(ASCEND_HOME)/include -Iinclude LIBS := -L$(ASCEND_HOME)/lib64 -lascendcl -CXXFLAGS := $(INCLUDES) +CXXFLAGS := $(INCLUDES) -O2 LDFLAGS := $(LIBS) PREFIX ?= /usr/local diff --git a/vllm_ascend/worker/worker_v1.py b/vllm_ascend/worker/worker_v1.py index da6c7d1..1c1ab34 100644 --- a/vllm_ascend/worker/worker_v1.py +++ b/vllm_ascend/worker/worker_v1.py @@ -358,7 +358,7 @@ class NPUWorker(WorkerBase): def try_reload_vram(self) -> tuple[bool, bool]: allocator = CaMemAllocator.get_instance() return allocator.try_reload_vram(tags=None) - + def vnpu_unlock_gpu(self) -> None: allocator = CaMemAllocator.get_instance() allocator.vnpu_unlock_gpu()