41 lines
907 B
JSON
41 lines
907 B
JSON
|
{
|
||
|
|
"args": [
|
||
|
|
"vllm",
|
||
|
|
"serve",
|
||
|
|
"/model",
|
||
|
|
"--trust-remote-code",
|
||
|
|
"--kv-cache-dtype",
|
||
|
|
"fp8",
|
||
|
|
"--block-size",
|
||
|
|
"256",
|
||
|
|
"--tensor-parallel-size",
|
||
|
|
"8",
|
||
|
|
"--data-parallel-size",
|
||
|
|
"2",
|
||
|
|
"--enable-expert-parallel",
|
||
|
|
"--no-enable-flashinfer-autotune",
|
||
|
|
"--tokenizer-mode",
|
||
|
|
"deepseek_v4",
|
||
|
|
"--tool-call-parser",
|
||
|
|
"deepseek_v4",
|
||
|
|
"--enable-auto-tool-choice",
|
||
|
|
"--reasoning-parser",
|
||
|
|
"deepseek_v4",
|
||
|
|
"--max-model-len",
|
||
|
|
"4096",
|
||
|
|
"--max-num-batched-tokens",
|
||
|
|
"1024",
|
||
|
|
"--enforce-eager",
|
||
|
|
"--port",
|
||
|
|
"18000"
|
||
|
|
],
|
||
|
|
"env": {
|
||
|
|
"VLLM_DISABLED_KERNELS": "PPUCutlassFp8BlockScaledMMKernel",
|
||
|
|
"VLLM_PPU_BF16_O_PROJ": "1",
|
||
|
|
"CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15",
|
||
|
|
"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": "0",
|
||
|
|
"VLLM_PPU_SERIAL_ATTENTION": "1",
|
||
|
|
"VLLM_ENGINE_READY_TIMEOUT_S": "3600"
|
||
|
|
}
|
||
|
|
}
|