24 lines
684 B
Bash
24 lines
684 B
Bash
#!/usr/bin/env bash
|
|
# Run inside srx_test after conversion finishes.
|
|
set -euo pipefail
|
|
export XPU_VISIBLE_DEVICES=0,1,2,3
|
|
export CUDA_VISIBLE_DEVICES=0,1,2,3
|
|
exec /opt/vllm_kunlun/bin/vllm serve \
|
|
/mnt/data0/models/MiniMax/MiniMax-M2.7-INT8 \
|
|
--trust-remote-code \
|
|
--quantization compressed-tensors \
|
|
--dtype bfloat16 \
|
|
--tensor-parallel-size 4 \
|
|
--distributed-executor-backend mp \
|
|
--max-model-len 8192 \
|
|
--max-num-seqs 4 \
|
|
--max-num-batched-tokens 8192 \
|
|
--block-size 128 \
|
|
--enforce-eager \
|
|
--no-enable-prefix-caching \
|
|
--no-enable-chunked-prefill \
|
|
--tool-call-parser minimax_m2 \
|
|
--enable-auto-tool-choice \
|
|
--reasoning-parser minimax_m2 \
|
|
"$@"
|