132 lines
3.2 KiB
Markdown
132 lines
3.2 KiB
Markdown
# Multi XPU (Qwen3-Coder-480B-A35B(W8A8))
|
|
|
|
## Run vllm-kunlun on Multi XPU
|
|
|
|
Setup environment using container:
|
|
|
|
```bash
|
|
# !/bin/bash
|
|
# rundocker.sh
|
|
XPU_NUM=8
|
|
DOCKER_DEVICE_CONFIG=""
|
|
if [ $XPU_NUM -gt 0 ]; then
|
|
for idx in $(seq 0 $((XPU_NUM-1))); do
|
|
DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpu${idx}:/dev/xpu${idx}"
|
|
done
|
|
DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpuctrl:/dev/xpuctrl"
|
|
fi
|
|
|
|
export build_image="xxxxxxxxxxxxxxxxx"
|
|
|
|
docker run -itd ${DOCKER_DEVICE_CONFIG} \
|
|
--net=host \
|
|
--cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
|
|
--tmpfs /dev/shm:rw,nosuid,nodev,exec,size=32g \
|
|
--cap-add=SYS_PTRACE \
|
|
-v /home/users/vllm-kunlun:/home/vllm-kunlun \
|
|
-v /usr/local/bin/xpu-smi:/usr/local/bin/xpu-smi \
|
|
--name "$1" \
|
|
-w /workspace \
|
|
"$build_image" /bin/bash
|
|
```
|
|
|
|
### Preparation Weight
|
|
|
|
* Pull Qwen3-Coder-480B-A35B-Instruct bf16 weights
|
|
* Modify the weights configuration.json file and add the fields quantization_config and compression_config.
|
|
|
|
```json
|
|
{
|
|
"architectures": [
|
|
"Qwen3MoeForCausalLM"
|
|
],
|
|
"attention_dropout": 0.0,
|
|
"decoder_sparse_step": 1,
|
|
"eos_token_id": 151645,
|
|
"head_dim": 128,
|
|
"hidden_act": "silu",
|
|
"hidden_size": 6144,
|
|
"initializer_range": 0.02,
|
|
"intermediate_size": 8192,
|
|
"max_position_embeddings": 262144,
|
|
"max_window_layers": 62,
|
|
"mlp_only_layers": [],
|
|
"model_type": "qwen3_moe",
|
|
"moe_intermediate_size": 2560,
|
|
"norm_topk_prob": true,
|
|
"num_attention_heads": 96,
|
|
"num_experts": 160,
|
|
"num_experts_per_tok": 8,
|
|
"num_hidden_layers": 62,
|
|
"num_key_value_heads": 8,
|
|
"output_router_logits": false,
|
|
"qkv_bias": false,
|
|
"rms_norm_eps": 1e-06,
|
|
"rope_scaling": null,
|
|
"rope_theta": 10000000,
|
|
"router_aux_loss_coef": 0.0,
|
|
"shared_expert_intermediate_size": 0,
|
|
"sliding_window": null,
|
|
"tie_word_embeddings": false,
|
|
"torch_dtype": "bfloat16",
|
|
"transformers_version": "4.51.0",
|
|
"use_cache": true,
|
|
"use_qk_norm": true,
|
|
"use_sliding_window": false,
|
|
"vocab_size": 151936,
|
|
"quantization_config": {
|
|
"quant_method": "compressed-tensors"
|
|
},
|
|
"compression_config": {
|
|
"format": "pack_quantized",
|
|
"config_groups": {
|
|
"linear_w8a8": {
|
|
"targets": ["Linear"],
|
|
"weights": {
|
|
"type": "int",
|
|
"num_bits": 8,
|
|
"strategy": "channel",
|
|
"group_size": null,
|
|
"symmetric": true,
|
|
"dynamic": false
|
|
},
|
|
"input_activations": {
|
|
"type": "int",
|
|
"num_bits": 8,
|
|
"strategy": "token",
|
|
"group_size": null,
|
|
"symmetric": true,
|
|
"dynamic": true
|
|
}
|
|
}
|
|
},
|
|
"ignore": [],
|
|
"sparsity_config": null
|
|
}
|
|
}
|
|
|
|
```
|
|
|
|
### Online Serving on Multi XPU
|
|
|
|
Start the vLLM server on multi XPU:
|
|
|
|
```bash
|
|
python3 -m vllm.entrypoints.openai.api_server \
|
|
--host 0.0.0.0 \
|
|
--port 8898 \
|
|
--model /Qwen/Qwen3-Coder-480B-A35B-Instruct \
|
|
--dtype float16 \
|
|
--trust-remote-code \
|
|
--tensor-parallel-size 8 \
|
|
--block-size 128 \
|
|
--max-model-len 40960 \
|
|
--max-num-seqs 512 \
|
|
--max-num-batched-tokens 40960 \
|
|
--max-seq-len-to-capture 40960 \
|
|
--distributed-executor-backend mp \
|
|
--enable-chunked-prefill=False \
|
|
--no-enable-prefix-caching \
|
|
--disable-log-requests \
|
|
--gpu-memory-utilization 0.85
|
|
``` |