26 lines
833 B
Bash
26 lines
833 B
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
export XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||
|
|
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||
|
|
# Avoid the incomplete MLA interface in the installed Kunlun plugin.
|
||
|
|
export VLLM_MLA_DISABLE=1
|
||
|
|
# kv_b_proj remains BF16; keep INT8 BMM disabled.
|
||
|
|
export VLLM_KUNLUN_ENABLE_INT8_BMM=0
|
||
|
|
exec /opt/vllm_kunlun/bin/vllm serve /mnt/data1/deepseek-r1-INT8 \
|
||
|
|
--trust-remote-code \
|
||
|
|
--dtype bfloat16 \
|
||
|
|
--quantization compressed-tensors \
|
||
|
|
--tensor-parallel-size 8 \
|
||
|
|
--distributed-executor-backend mp \
|
||
|
|
--max-model-len 4096 \
|
||
|
|
--max-num-seqs 1 \
|
||
|
|
--max-num-batched-tokens 4096 \
|
||
|
|
--kv-cache-memory-bytes 4294967296 \
|
||
|
|
--enforce-eager \
|
||
|
|
--no-enable-prefix-caching \
|
||
|
|
--no-enable-chunked-prefill \
|
||
|
|
--enable-auto-tool-choice \
|
||
|
|
--tool-call-parser deepseek_v3 \
|
||
|
|
--reasoning-parser deepseek_r1 \
|
||
|
|
"$@"
|