first commit

serve_int8.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Run inside srx_test after conversion finishes.
+set -euo pipefail
+export XPU_VISIBLE_DEVICES=0,1,2,3
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+exec /opt/vllm_kunlun/bin/vllm serve \
+  /mnt/data0/models/MiniMax/MiniMax-M2.7-INT8 \
+  --trust-remote-code \
+  --quantization compressed-tensors \
+  --dtype bfloat16 \
+  --tensor-parallel-size 4 \
+  --distributed-executor-backend mp \
+  --max-model-len 8192 \
+  --max-num-seqs 4 \
+  --max-num-batched-tokens 8192 \
+  --block-size 128 \
+  --enforce-eager \
+  --no-enable-prefix-caching \
+  --no-enable-chunked-prefill \
+  --tool-call-parser minimax_m2 \
+  --enable-auto-tool-choice \
+  --reasoning-parser minimax_m2 \
+  "$@"