24 lines
736 B
Bash
24 lines
736 B
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
export XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
|
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
|
# kv_b_proj is BF16: use the MLA BF16 BMM path.
|
|
export VLLM_KUNLUN_ENABLE_INT8_BMM=0
|
|
exec /opt/vllm_kunlun/bin/vllm serve /mnt/data1/deepseek-r1-INT8 \
|
|
--trust-remote-code \
|
|
--dtype bfloat16 \
|
|
--quantization compressed-tensors \
|
|
--tensor-parallel-size 8 \
|
|
--distributed-executor-backend mp \
|
|
--max-model-len 4096 \
|
|
--max-num-seqs 1 \
|
|
--max-num-batched-tokens 4096 \
|
|
--kv-cache-memory-bytes 4294967296 \
|
|
--enforce-eager \
|
|
--no-enable-prefix-caching \
|
|
--no-enable-chunked-prefill \
|
|
--enable-auto-tool-choice \
|
|
--tool-call-parser deepseek_v3 \
|
|
--reasoning-parser deepseek_r1 \
|
|
"$@"
|