32 lines
1013 B
Bash
Executable File
32 lines
1013 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
IMAGE=${IMAGE:-harbor.4pd.io/modelhubxc/zhoushasha/sage-a1.5-nv:v0.1.0}
|
|
MODEL=${MODEL:-/mnt/disk0/models/Sage-A1.5}
|
|
test -f "$MODEL/config.json"
|
|
docker run -d \
|
|
--name sage-a15-h100-test \
|
|
--gpus '"device=0,1"' \
|
|
--device /dev/nvidia-uvm:/dev/nvidia-uvm \
|
|
--device /dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools \
|
|
--shm-size=16g \
|
|
-p 127.0.0.1:9099:8000 \
|
|
-e NVIDIA_DRIVER_CAPABILITIES=compute,utility \
|
|
-e HF_HUB_OFFLINE=1 \
|
|
-e TRANSFORMERS_OFFLINE=1 \
|
|
-e VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
|
--mount "type=bind,src=$MODEL,dst=/model,readonly" \
|
|
--entrypoint python3 \
|
|
"$IMAGE" \
|
|
-u -m vllm.entrypoints.openai.api_server \
|
|
--model /model \
|
|
--served-model-name Sage-A1.5 \
|
|
--host 0.0.0.0 --port 8000 \
|
|
--tensor-parallel-size 2 \
|
|
--distributed-executor-backend mp \
|
|
--dtype bfloat16 \
|
|
--max-model-len 4096 \
|
|
--max-num-batched-tokens 4096 \
|
|
--max-num-seqs 1 \
|
|
--gpu-memory-utilization 0.90 \
|
|
--enforce-eager
|