first commit

serve_int8_no_mla.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+export XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+# Avoid the incomplete MLA interface in the installed Kunlun plugin.
+export VLLM_MLA_DISABLE=1
+# kv_b_proj remains BF16; keep INT8 BMM disabled.
+export VLLM_KUNLUN_ENABLE_INT8_BMM=0
+exec /opt/vllm_kunlun/bin/vllm serve /mnt/data1/deepseek-r1-INT8 \
+  --trust-remote-code \
+  --dtype bfloat16 \
+  --quantization compressed-tensors \
+  --tensor-parallel-size 8 \
+  --distributed-executor-backend mp \
+  --max-model-len 4096 \
+  --max-num-seqs 1 \
+  --max-num-batched-tokens 4096 \
+  --kv-cache-memory-bytes 4294967296 \
+  --enforce-eager \
+  --no-enable-prefix-caching \
+  --no-enable-chunked-prefill \
+  --enable-auto-tool-choice \
+  --tool-call-parser deepseek_v3 \
+  --reasoning-parser deepseek_r1 \
+  "$@"