diff --git a/README.md b/README.md index 90822b176..39e537022 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,16 @@ pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ ### Method 3: Using docker The docker images are available on Docker Hub as [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags). +```bash +docker run --gpus all \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HUGGING_FACE_HUB_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B --host 0.0.0.0 --port 30000 +``` + ### Common Notes - If you see errors from the Triton compiler, please install the [Triton Nightly](https://triton-lang.org/main/getting-started/installation.html) by ``` diff --git a/docker/Dockerfile b/docker/Dockerfile index 65abd11b6..f519d48ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,7 +16,9 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv python3-pip \ && if [ "${PYTHON_VERSION}" != "3" ]; then update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1; fi \ && python3 --version \ - && python3 -m pip --version + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean RUN apt-get update -y \ && apt-get install -y python3-pip git curl sudo @@ -29,9 +31,10 @@ RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/ WORKDIR /sgl-workspace -RUN pip3 install --upgrade pip -RUN pip3 install "sglang[all]" -RUN pip3 uninstall -y triton triton-nightly && pip3 install --no-deps --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/ triton-nightly -RUN pip3 install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ +RUN pip3 --no-cache-dir install --upgrade pip \ + && pip3 --no-cache-dir install "sglang[all]" \ + && pip3 --no-cache-dir uninstall -y triton triton-nightly \ + && pip3 --no-cache-dir install --no-deps --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/ triton-nightly \ + && pip3 --no-cache-dir install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ ENV DEBIAN_FRONTEND=interactive