EngineX-Ascend/enginex-ascend-910-vllm
11
0
Fork 1
Code Issues Pull Requests Actions Projects Releases Wiki Activity

init v0.23.0

Browse Source 
Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>
This commit is contained in:
Sun Ruoxi
2026-08-27 15:11:51 +08:00
parent b582a8e7d1
commit 7f8a1b1f7a
2849 changed files with 712887 additions and 22001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
tests/e2e/nightly/multi_node/internal_dp/config/DeepSeek-R1-W8A8-EPLB.yaml Normal file
View File

@@ -0,0 +1,196 @@
test_name: "test DeepSeek-R1-W8A8 disaggregated_prefill"
model: "vllm-ascend/DeepSeek-R1-0528-W8A8"
num_nodes: 4
npu_per_node: 16
env_common: &env_common
VLLM_USE_MODELSCOPE: true
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
OMP_PROC_BIND: false
OMP_NUM_THREADS: 10
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
HCCL_DETERMINISTIC: True
TASK_QUEUE_ENABLE: 1
HCCL_OP_RETRY_ENABLE: "L0:0, L1:0, L2:0"
DYNAMIC_EPLB: true
VLLM_ENGINE_READY_TIMEOUT_S: 3000
disaggregated_prefill:
enabled: true
prefiller_host_index: [0, 1]
decoder_host_index: [2, 3]
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-R1-0528-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--tensor-parallel-size 8
--enforce-eager
--enable-expert-parallel
--seed 1024
--quantization ascend
--max-num-seqs 4
--max-model-len 36864
--max-num-batched-tokens 16384
--trust-remote-code
--gpu-memory-utilization 0.9
--speculative-config '{"num_speculative_tokens": 1, "method":"mtp"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 32,
"tp_size": 1
}
}
}'
--additional-config
'{"enable_prefill_optimizations":true,"enable_weight_nz_layout":true,"eplb_config": {"dynamic_eplb":true,"expert_heat_collection_interval":2048,"algorithm_execution_interval":200}}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-R1-0528-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--tensor-parallel-size 8
--enforce-eager
--enable-expert-parallel
--seed 1024
--quantization ascend
--max-num-seqs 4
--max-model-len 36864
--max-num-batched-tokens 16384
--trust-remote-code
--gpu-memory-utilization 0.9
--speculative-config '{"num_speculative_tokens": 1, "method":"mtp"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30100",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 32,
"tp_size": 1
}
}
}'
--additional-config
'{"enable_prefill_optimizations":true,"enable_weight_nz_layout":true,"eplb_config": {"dynamic_eplb":true,"expert_heat_collection_interval":2048,"algorithm_execution_interval":200}}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-R1-0528-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 32
--data-parallel-size-local 16
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 1
--enable-expert-parallel
--seed 1024
--quantization ascend
--max-num-seqs 28
--max-model-len 36864
--max-num-batched-tokens 256
--trust-remote-code
--gpu-memory-utilization 0.9
--speculative-config '{"num_speculative_tokens": 1, "method":"mtp"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30200",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 32,
"tp_size": 1
}
}
}'
--additional-config
'{"multistream_overlap_shared_expert":true,"dynamic_eplb":true,"expert_heat_collection_interval":2048,"algorithm_execution_interval":200}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-R1-0528-W8A8
--headless
--data-parallel-size 32
--data-parallel-size-local 16
--data-parallel-start-rank 16
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 1
--enable-expert-parallel
--seed 1024
--quantization ascend
--max-num-seqs 28
--max-model-len 36864
--max-num-batched-tokens 256
--trust-remote-code
--gpu-memory-utilization 0.9
--speculative-config '{"num_speculative_tokens": 1, "method":"mtp"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30200",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 32,
"tp_size": 1
}
}
}'
--additional-config
'{"multistream_overlap_shared_expert":true,"eplb_config": {"dynamic_eplb":true,"expert_heat_collection_interval":2048,"algorithm_execution_interval":200}}'
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 2800
max_out_len: 1500
batch_size: 700
request_rate: 11.2
baseline: 1
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
max_out_len: 32768
batch_size: 512
baseline: 95
threshold: 5

114
tests/e2e/nightly/multi_node/internal_dp/config/DeepSeek-R1-W8A8-longseq.yaml Normal file
View File

@@ -0,0 +1,114 @@
test_name: "test DeepSeek-R1-W8A8-longseq disaggregated_prefill"
model: "vllm-ascend/DeepSeek-R1-0528-W8A8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: AIV
VLLM_USE_MODELSCOPE: true
HCCL_BUFFSIZE: 768
SERVER_PORT: 8080
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
HCCL_DETERMINISTIC: True
TASK_QUEUE_ENABLE: 1
HCCL_OP_RETRY_ENABLE: "L0:0, L1:0"
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: 480
VLLM_ENGINE_READY_TIMEOUT_S: 3000
disaggregated_prefill:
enabled: true
prefiller_host_index: [0]
decoder_host_index: [1]
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-R1-0528-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 1
--decode-context-parallel-size 8
--prefill-context-parallel-size 2
--tensor-parallel-size 8
--cp-kv-cache-interleave-size 128
--enforce-eager
--enable-expert-parallel
--seed 1024
--quantization ascend
--max-num-seqs 32
--max-model-len 32768
--max-num-batched-tokens 16384
--trust-remote-code
--gpu-memory-utilization 0.85
--enable-chunked-prefill
--speculative-config '{"num_speculative_tokens": 3, "method":"mtp"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 1,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-R1-0528-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--decode-context-parallel-size 2
--prefill-context-parallel-size 1
--tensor-parallel-size 8
--cp-kv-cache-interleave-size 128
--enable-expert-parallel
--seed 1024
--quantization ascend
--max-num-seqs 32
--max-model-len 32768
--max-num-batched-tokens 256
--trust-remote-code
--gpu-memory-utilization 0.85
--compilation_config '{"cudagraph_capture_sizes":[4,8,16,32],"cudagraph_mode": "FULL_DECODE_ONLY"}'
--enable-chunked-prefill
--speculative-config '{"num_speculative_tokens": 3, "method":"mtp"}'
--additional-config '{"recompute_scheduler_enable":true}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30100",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 1,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
benchmarks:
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
num_prompts: 360
max_out_len: 4096
batch_size: 32
baseline: 95
threshold: 5

85
tests/e2e/nightly/multi_node/internal_dp/config/DeepSeek-V3.1-BF16.yaml Normal file
View File

@@ -0,0 +1,85 @@
test_name: "test DeepSeek-V3.1-BF16 on A3"
model: "unsloth/DeepSeek-V3.1-BF16"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
VLLM_USE_MODELSCOPE: true
HCCL_BUFFSIZE: 2048
SERVER_PORT: 8080
OMP_PROC_BIND: false
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
OMP_NUM_THREADS: 1
VLLM_ASCEND_ENABLE_MLAPO: 1
VLLM_ASCEND_BALANCE_SCHEDULING: 1
HCCL_INTRA_PCIE_ENABLE: 1
HCCL_INTRA_ROCE_ENABLE: 0
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve unsloth/DeepSeek-V3.1-BF16
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 4
--tensor-parallel-size 8
--data-parallel-size-local 2
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13399
--no-enable-prefix-caching
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 4096
--enable-expert-parallel
--trust-remote-code
--gpu-memory-utilization 0.95
--speculative-config '{"num_speculative_tokens": 1, "method":"mtp"}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[2, 4, 8, 16, 32]}'
--additional_config '{"enable_multistream_moe": true}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve unsloth/DeepSeek-V3.1-BF16
--headless
--data-parallel-size 4
--tensor-parallel-size 8
--data-parallel-size-local 2
--data-parallel-start-rank 2
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13399
--no-enable-prefix-caching
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 4096
--enable-expert-parallel
--trust-remote-code
--gpu-memory-utilization 0.95
--speculative-config '{"num_speculative_tokens": 1, "method":"mtp"}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[2, 4, 8, 16, 32]}'
--additional_config '{"enable_multistream_moe": true}'
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 512
max_out_len: 512
batch_size: 700
request_rate: 11.2
baseline: 1
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k-lite
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
max_out_len: 4096
batch_size: 512
baseline: 95
threshold: 10

127
tests/e2e/nightly/multi_node/internal_dp/config/DeepSeek-V3_2-W8A8-A3-dual-nodes.yaml Normal file
View File

@@ -0,0 +1,127 @@
test_name: "test DeepSeek-V3.2-W8A8 on A3"
model: "vllm-ascend/DeepSeek-V3.2-W8A8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: "AIV"
VLLM_USE_MODELSCOPE: true
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
VLLM_ASCEND_ENABLE_MLAPO: 1
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
ASCEND_A3_EBA_ENABLE: 1
VLLM_ENGINE_READY_TIMEOUT_S: 3000
# TODO: need to identify why TP and mtp+1 divisibility rules break on dual-node case
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-V3.2-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 4
--data-parallel-size-local 2
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13399
--tensor-parallel-size 8
--quantization ascend
--seed 1024
--enable-expert-parallel
--max-num-seqs 128
--max-model-len 90000
--max-num-batched-tokens 4096
--no-enable-prefix-caching
--gpu-memory-utilization 0.85
--trust-remote-code
--speculative-config '{"num_speculative_tokens": 3, "method":"deepseek_mtp"}'
--compilation-config '{"cudagraph_capture_sizes": [8, 16, 24, 32, 40, 48], "cudagraph_mode": "FULL_DECODE_ONLY"}'
--tokenizer-mode deepseek_v32
--reasoning-parser deepseek_v3
-
envs:
<<: *env_common
server_cmd: >
vllm serve vllm-ascend/DeepSeek-V3.2-W8A8
--headless
--data-parallel-size 4
--data-parallel-rpc-port 13399
--data-parallel-size-local 2
--data-parallel-start-rank 2
--data-parallel-address $MASTER_IP
--tensor-parallel-size 8
--quantization ascend
--seed 1024
--enable-expert-parallel
--max-num-seqs 128
--max-model-len 90000
--max-num-batched-tokens 4096
--no-enable-prefix-caching
--gpu-memory-utilization 0.85
--trust-remote-code
--speculative-config '{"num_speculative_tokens": 3, "method":"deepseek_mtp"}'
--compilation-config '{"cudagraph_capture_sizes": [8, 16, 24, 32, 40, 48], "cudagraph_mode": "FULL_DECODE_ONLY"}'
--tokenizer-mode deepseek_v32
--reasoning-parser deepseek_v3
benchmarks:
perf_short_warmup:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 1
max_out_len: 3000
batch_size: 512
request_rate: 11.2
baseline: 1
threshold: 0.97
perf_long_warmup:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in64000-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 1
max_out_len: 3000
batch_size: 1
request_rate: 11.2
baseline: 1
threshold: 0.97
perf_short:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 512
max_out_len: 3000
batch_size: 256
request_rate: 11.2
baseline: 305.2903
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k-lite
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
max_out_len: 4096
batch_size: 128
baseline: 95
threshold: 10
acc_aime2025:
case_type: accuracy
dataset_path: vllm-ascend/aime2025
request_conf: vllm_api_general_chat
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
max_out_len: 80000
batch_size: 32
baseline: 57
threshold: 10

268
tests/e2e/nightly/multi_node/internal_dp/config/DeepSeek-V3_2-W8A8-EP.yaml Normal file
View File

@@ -0,0 +1,268 @@
test_name: "test DeepSeek-V3.2-W8A8-EP disaggregated_prefill"
model: "vllm-ascend/DeepSeek-V3.2-W8A8"
num_nodes: 4
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: "AIV"
VLLM_USE_MODELSCOPE: true
SERVER_PORT: 8080
OMP_PROC_BIND: false
OMP_NUM_THREADS: 10
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
HCCL_BUFFSIZE: 360
VLLM_TORCH_PROFILER_WITH_STACK: 0
ASCEND_AGGREGATE_ENABLE: 1
ASCEND_TRANSPORT_PRINT: 1
ACL_OP_INIT_MODE: 1
ASCEND_A3_ENABLE: 1
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: 480
VLLM_ENGINE_READY_TIMEOUT_S: 3000
HCCL_CONNECT_TIMEOUT: 1200
disaggregated_prefill:
enabled: true
prefiller_host_index: [0, 1]
decoder_host_index: [2, 3]
deployment:
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
server_cmd: >
vllm serve vllm-ascend/DeepSeek-V3.2-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-start-rank 0
--data-parallel-size-local 1
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 16
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 2, "method":"deepseek_mtp"}'
--seed 1024
--quantization ascend
--max-num-seqs 16
--max-model-len 133000
--max-num-batched-tokens 8192
--trust-remote-code
--gpu-memory-utilization 0.90
--enforce-eager
--no-enable-prefix-caching
--additional-config '{"enable_cpu_binding" : false, "enable_sfa_cp":false,"layer_sharding": ["q_b_proj", "o_proj"]}'
--tokenizer-mode deepseek_v32
--reasoning-parser deepseek_v3
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 16
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
server_cmd: >
vllm serve vllm-ascend/DeepSeek-V3.2-W8A8
--host 0.0.0.0
--headless
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-start-rank 1
--data-parallel-size-local 1
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 16
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 2, "method":"deepseek_mtp"}'
--seed 1024
--quantization ascend
--max-num-seqs 16
--max-model-len 133000
--max-num-batched-tokens 8192
--trust-remote-code
--gpu-memory-utilization 0.90
--enforce-eager
--no-enable-prefix-caching
--additional-config '{"enable_cpu_binding" : false, "enable_sfa_cp":false,"layer_sharding": ["q_b_proj", "o_proj"]}'
--tokenizer-mode deepseek_v32
--reasoning-parser deepseek_v3
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 16
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_MLAPO: 1
TASK_QUEUE_ENABLE: 1
HCCL_BUFFSIZE: 1100
server_cmd: >
vllm serve vllm-ascend/DeepSeek-V3.2-W8A8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 8
--data-parallel-size-local 4
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 4
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 2, "method":"deepseek_mtp"}'
--seed 1024
--quantization ascend
--max-model-len 133000
--max-num-batched-tokens 42
--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42]}'
--trust-remote-code
--max-num-seqs 14
--gpu-memory-utilization 0.90
--no-enable-prefix-caching
--additional-config '{"enable_cpu_binding" : false,"recompute_scheduler_enable" : true}'
--tokenizer-mode deepseek_v32
--reasoning-parser deepseek_v3
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30200",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 16
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_MLAPO: 1
TASK_QUEUE_ENABLE: 1
HCCL_BUFFSIZE: 1100
server_cmd: >
vllm serve vllm-ascend/DeepSeek-V3.2-W8A8
--host 0.0.0.0
--headless
--port $SERVER_PORT
--data-parallel-size 8
--data-parallel-size-local 4
--data-parallel-start-rank 4
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 4
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 2, "method":"deepseek_mtp"}'
--seed 1024
--quantization ascend
--max-model-len 133000
--max-num-batched-tokens 42
--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42]}'
--trust-remote-code
--max-num-seqs 14
--gpu-memory-utilization 0.90
--no-enable-prefix-caching
--additional-config '{"enable_cpu_binding" : false,"recompute_scheduler_enable" : true}'
--tokenizer-mode deepseek_v32
--reasoning-parser deepseek_v3
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30200",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 16
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
benchmarks:
perf_short_warmup:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 1
max_out_len: 1500
batch_size: 1
request_rate: 11.2
baseline: 1
threshold: 0.97
perf_long_warmup:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in64000-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 1
max_out_len: 1024
batch_size: 1
request_rate: 11.2
baseline: 1
threshold: 0.97
perf_short:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 128
max_out_len: 1500
batch_size: 32
request_rate: 0
baseline: 1
threshold: 0.97
perf_long:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in64000-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 16
max_out_len: 1024
batch_size: 4
request_rate: 1
baseline: 1
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k-lite
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
max_out_len: 4096
batch_size: 64
baseline: 96.88
threshold: 10

102
tests/e2e/nightly/multi_node/internal_dp/config/GLM-5.1-W8A8C8-A3_128k_90_50.yaml Normal file
View File

@@ -0,0 +1,102 @@
test_name: "multi-node-GLM-5.1-W8A8C8-MTP-A3_64k/128k"
model: "Eco-Tech/GLM-5.1-w8a8c8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: "AIV"
OMP_PROC_BIND: "false"
VLLM_USE_MODELSCOPE: "true"
OMP_NUM_THREADS: "1"
HCCL_BUFFSIZE: "400"
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
VLLM_ASCEND_ENABLE_MLAPO: "1"
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
SERVER_PORT: 8077
deployment:
- envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8c8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 8
--data-parallel-size-local 4
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--enable-expert-parallel
--data-parallel-rpc-port 12981
--tensor-parallel-size 4
--hf-overrides '{"use_index_cache": true, "index_topk_freq": 4}'
--seed 1024
--tool-call-parser glm47
--reasoning-parser glm45
--enable-auto-tool-choice
--max-num-seqs 6
--max-model-len 133120
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.92
--quantization ascend
--enable-chunked-prefill
--enable-prefix-caching
--async-scheduling
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
--additional-config '{"enable_dsa_cp": true, "enable_sparse_sfa_c8": false, "enable_sparse_li_c8": true, "enable_balance_scheduling": true, "fuse_muls_add": true, "multistream_overlap_shared_expert": true}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp","enforce_eager":true}'
- envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8c8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 8
--data-parallel-size-local 4
--data-parallel-start-rank 4
--headless
--data-parallel-address $MASTER_IP
--enable-expert-parallel
--data-parallel-rpc-port 12981
--tensor-parallel-size 4
--hf-overrides '{"use_index_cache": true, "index_topk_freq": 4}'
--seed 1024
--tool-call-parser glm47
--reasoning-parser glm45
--enable-auto-tool-choice
--max-num-seqs 6
--max-model-len 133120
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.9
--quantization ascend
--enable-chunked-prefill
--enable-prefix-caching
--async-scheduling
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
--additional-config '{"enable_dsa_cp": true, "enable_sparse_sfa_c8": false, "enable_sparse_li_c8": true, "enable_balance_scheduling": true, "fuse_muls_add": true, "multistream_overlap_shared_expert": true}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp","enforce_eager":true}'
benchmarks:
perf_128k_warmup:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in131072-bs500-prefix90-glm51
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 1
max_out_len: 1
batch_size: 1
request_rate: 0
baseline: 0
threshold: 0.97
perf_128k:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in131072-bs500-prefix90-glm51
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 256
max_out_len: 1024
batch_size: 64
request_rate: 0
baseline: 290.8308
threshold: 0.97

84
tests/e2e/nightly/multi_node/internal_dp/config/GLM5_1-W8A8-A2-dual-nodes.yaml Normal file
View File

@@ -0,0 +1,84 @@
test_name: "multi-node-GLM-5.1-w8a8-A2"
model: "Eco-Tech/GLM-5.1-w8a8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: "AIV"
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
HCCL_BUFFSIZE: 200
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
VLLM_ASCEND_BALANCE_SCHEDULING: 0
VLLM_ASCEND_ENABLE_MLAPO: 1
VLLM_USE_MODELSCOPE: true
VLLM_ENGINE_READY_TIMEOUT_S: 3000
VLLM_RPC_TIMEOUT: 600
SERVER_PORT: 8078
special_dependencies:
transformers: "5.2.0"
# TODO: need to identify why TP and mtp+1 divisibility rules break on dual-node case
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--tensor-parallel-size 8
--data-parallel-rpc-port 13389
--data-parallel-size-local 1
--data-parallel-address $LOCAL_IP
--enable-expert-parallel
--seed 1024
--max-num-seqs 64
--max-model-len 38000
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.95
--quantization ascend
--no-enable-prefix-caching
--additional-config '{"multistream_overlap_shared_expert": true, "ascend_compilation_config": {"fuse_qknorm_rope": false}}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY","cudagraph_capture_sizes": [1,4,8,12,16,20,24,28,32,36,48,60,72]}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--tensor-parallel-size 8
--data-parallel-size-local 1
--data-parallel-start-rank 1
--headless
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13389
--enable-expert-parallel
--seed 1024
--max-num-seqs 64
--max-model-len 38000
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.95
--quantization ascend
--no-enable-prefix-caching
--additional-config '{"multistream_overlap_shared_expert": true, "ascend_compilation_config": {"fuse_qknorm_rope": false}}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY","cudagraph_capture_sizes": [1,4,8,12,16,20,24,28,32,36,48,60,72]}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}'
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 72
max_out_len: 1500
batch_size: 18
request_rate: 0
baseline: 1
threshold: 0.97

98
tests/e2e/nightly/multi_node/internal_dp/config/GLM5_1-W8A8-A3-dual-nodes.yaml Normal file
View File

@@ -0,0 +1,98 @@
test_name: "multi-node-GLM-5.1-w8a8-A3"
model: "Eco-Tech/GLM-5.1-w8a8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: "AIV"
OMP_PROC_BIND: false
VLLM_USE_MODELSCOPE: true
OMP_NUM_THREADS: 1
HCCL_BUFFSIZE: 200
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
VLLM_ASCEND_BALANCE_SCHEDULING: 0
VLLM_ASCEND_ENABLE_MLAPO: 1
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
VLLM_RPC_TIMEOUT: "600"
SERVER_PORT: 8080
special_dependencies:
transformers: "5.2.0"
# TODO: need to identify why TP and mtp+1 divisibility rules break on dual-node case
deployment:
- envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--tensor-parallel-size 16
--data-parallel-rpc-port 13389
--data-parallel-size-local 1
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--enable-expert-parallel
--seed 1024
--max-num-seqs 16
--max-model-len 133120
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.95
--quantization ascend
--enable-chunked-prefill
--enable-prefix-caching
--async-scheduling
--additional-config '{"multistream_overlap_shared_expert":true}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--tensor-parallel-size 16
--data-parallel-size-local 1
--data-parallel-start-rank 1
--data-parallel-rpc-port 13389
--headless
--data-parallel-address $MASTER_IP
--enable-expert-parallel
--seed 1024
--max-num-seqs 16
--max-model-len 133120
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.95
--quantization ascend
--enable-chunked-prefill
--enable-prefix-caching
--async-scheduling
--additional-config '{"multistream_overlap_shared_expert":true}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}'
benchmarks:
acc_aime2025:
case_type: accuracy
dataset_path: vllm-ascend/aime2025
request_conf: vllm_api_general_chat
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
max_out_len: 72348
batch_size: 32
baseline: 90
threshold: 10
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_glm
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 12
max_out_len: 1024
batch_size: 3
request_rate: 0
baseline: 1
threshold: 0.97

240
tests/e2e/nightly/multi_node/internal_dp/config/GLM5_1-W8A8-EP.yaml Normal file
View File

@@ -0,0 +1,240 @@
test_name: "multi-node-GLM-5.1-w8a8-EP"
model: "Eco-Tech/GLM-5.1-w8a8"
num_nodes: 4
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: "AIV"
VLLM_USE_MODELSCOPE: true
SERVER_PORT: 8080
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
HCCL_BUFFSIZE: 1024
VLLM_TORCH_PROFILER_WITH_STACK: 0
ASCEND_AGGREGATE_ENABLE: 1
ASCEND_TRANSPORT_PRINT: 1
ACL_OP_INIT_MODE: 1
ASCEND_A3_ENABLE: 1
VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300000"
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
HCCL_CONNECT_TIMEOUT: "1200"
HCCL_INTRA_PCIE_ENABLE: 1
HCCL_INTRA_ROCE_ENABLE: 0
VLLM_ASCEND_ENABLE_FUSED_MC2: 1
special_dependencies:
transformers: "5.2.0"
disaggregated_prefill:
enabled: true
prefiller_host_index: [0, 1]
decoder_host_index: [2, 3]
# TODO: need to identify why TP and mtp+1 divisibility rules break on dual-node case
deployment:
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 4
--data-parallel-size-local 2
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--tensor-parallel-size 8
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 3, "method":"deepseek_mtp"}'
--profiler-config
'{"profiler": "torch",
"torch_profiler_dir": "./vllm_profile",
"torch_profiler_with_stack": false}'
--seed 1024
--max-model-len 131072
--additional-config '{"ascend_compilation_config": {"enable_npugraph_ex": true}}'
--max-num-batched-tokens 4096
--trust-remote-code
--max-num-seqs 64
--quantization ascend
--gpu-memory-utilization 0.95
--enforce-eager
--enable-auto-tool-choice
--tool-call-parser glm47
--reasoning-parser glm45
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"use_ascend_direct": true,
"prefill": {
"dp_size": 4,
"tp_size": 8
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--headless
--port $SERVER_PORT
--data-parallel-size 4
--data-parallel-size-local 2
--data-parallel-start-rank 2
--data-parallel-address $MASTER_IP
--tensor-parallel-size 8
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 3, "method":"deepseek_mtp"}'
--profiler-config
'{"profiler": "torch",
"torch_profiler_dir": "./vllm_profile",
"torch_profiler_with_stack": false}'
--seed 1024
--max-model-len 131072
--additional-config '{"ascend_compilation_config": {"enable_npugraph_ex": true}}'
--max-num-batched-tokens 4096
--trust-remote-code
--max-num-seqs 64
--quantization ascend
--gpu-memory-utilization 0.95
--enforce-eager
--enable-auto-tool-choice
--tool-call-parser glm47
--reasoning-parser glm45
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"use_ascend_direct": true,
"prefill": {
"dp_size": 4,
"tp_size": 8
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_MLAPO: 1
TASK_QUEUE_ENABLE: 1
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 8
--data-parallel-size-local 4
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 10543
--tensor-parallel-size 4
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 3, "method":"deepseek_mtp"}'
--profiler-config
'{"profiler": "torch",
"torch_profiler_dir": "./vllm_profile",
"torch_profiler_with_stack": false}'
--seed 1024
--max-model-len 202752
--max-num-batched-tokens 32
--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[4, 8, 12, 16,20,24,28, 32]}'
--additional-config '{"recompute_scheduler_enable": true, "ascend_compilation_config": {"enable_npugraph_ex": true}}'
--trust-remote-code
--max-num-seqs 8
--gpu-memory-utilization 0.92
--quantization ascend
--enable-auto-tool-choice
--tool-call-parser glm47
--reasoning-parser glm45
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30100",
"kv_connector_extra_config": {
"use_ascend_direct": true,
"prefill": {
"dp_size": 4,
"tp_size": 8
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
VLLM_ASCEND_ENABLE_MLAPO: 1
TASK_QUEUE_ENABLE: 1
server_cmd: >
vllm serve Eco-Tech/GLM-5.1-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 8
--data-parallel-size-local 4
--data-parallel-start-rank 4
--headless
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 10543
--tensor-parallel-size 4
--enable-expert-parallel
--speculative-config '{"num_speculative_tokens": 3, "method":"deepseek_mtp"}'
--profiler-config
'{"profiler": "torch",
"torch_profiler_dir": "./vllm_profile",
"torch_profiler_with_stack": false}'
--seed 1024
--max-model-len 202752
--max-num-batched-tokens 32
--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[4, 8, 12, 16,20,24,28, 32]}'
--additional-config '{"recompute_scheduler_enable": true, "ascend_compilation_config": {"enable_npugraph_ex": true}}'
--trust-remote-code
--max-num-seqs 8
--gpu-memory-utilization 0.92
--quantization ascend
--enable-auto-tool-choice
--tool-call-parser glm47
--reasoning-parser glm45
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30100",
"kv_connector_extra_config": {
"use_ascend_direct": true,
"prefill": {
"dp_size": 4,
"tp_size": 8
},
"decode": {
"dp_size": 8,
"tp_size": 4
}
}
}'
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 160
max_out_len: 1500
batch_size: 40
request_rate: 0
baseline: 1
threshold: 0.97

91
tests/e2e/nightly/multi_node/internal_dp/config/GLM5_2-W8A8-A3-dual-nodes.yaml Normal file
View File

@@ -0,0 +1,91 @@
test_name: "multi-node-GLM-5.2-w8a8-A3"
model: "Eco-Tech/GLM-5.2-w8a8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: "AIV"
OMP_PROC_BIND: false
VLLM_USE_MODELSCOPE: true
OMP_NUM_THREADS: 1
HCCL_BUFFSIZE: 200
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
VLLM_ASCEND_ENABLE_MLAPO: 1
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
VLLM_RPC_TIMEOUT: "600"
SERVER_PORT: 8080
special_dependencies:
transformers: "5.12.0"
# TODO: need to identify why TP and mtp+1 divisibility rules break on dual-node case
deployment:
- envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.2-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--tensor-parallel-size 16
--data-parallel-rpc-port 13389
--data-parallel-size-local 1
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--enable-expert-parallel
--seed 1024
--max-num-seqs 16
--max-model-len 133120
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.95
--quantization ascend
--additional-config '{"multistream_overlap_shared_expert":true}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp", "enforce_eager":true}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/GLM-5.2-w8a8
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--tensor-parallel-size 16
--data-parallel-size-local 1
--data-parallel-start-rank 1
--data-parallel-rpc-port 13389
--headless
--data-parallel-address $MASTER_IP
--enable-expert-parallel
--seed 1024
--max-num-seqs 16
--max-model-len 133120
--max-num-batched-tokens 4096
--trust-remote-code
--gpu-memory-utilization 0.95
--quantization ascend
--additional-config '{"multistream_overlap_shared_expert":true}'
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
--speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp", "enforce_eager":true}'
benchmarks:
acc_aime2025:
case_type: accuracy
dataset_path: vllm-ascend/aime2025
request_conf: vllm_api_general_chat
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
max_out_len: 72348
batch_size: 32
baseline: 90
threshold: 10
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_glm
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 12
max_out_len: 1024
batch_size: 3
request_rate: 0
baseline: 1
threshold: 0.97

90
tests/e2e/nightly/multi_node/internal_dp/config/Kimi-K2_5-W4A8-A2-dual-nodes.yaml Normal file
View File

@@ -0,0 +1,90 @@
test_name: "test Kimi-K2.5-W4A8 A2 dual nodes"
model: "Eco-Tech/Kimi-K2.5-W4A8"
num_nodes: 2
npu_per_node: 8
env_common: &env_common
HCCL_OP_EXPANSION_MODE: AIV
HCCL_INTRA_PCIE_ENABLE: 1
HCCL_INTRA_ROCE_ENABLE: 0
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
TASK_QUEUE_ENABLE: 1
HCCL_BUFFSIZE: 512
VLLM_ASCEND_ENABLE_MLAPO: 1
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
VLLM_USE_MODELSCOPE: true
SERVER_PORT: 8080
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/Kimi-K2.5-W4A8
--host 0.0.0.0
--port $SERVER_PORT
--quantization ascend
--allowed-local-media-path /
--trust-remote-code
--no-enable-prefix-caching
--seed 42
--data-parallel-size 2
--data-parallel-size-local 1
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 8
--enable-expert-parallel
--max-num-seqs 64
--max-model-len 51200
--max-num-batched-tokens 8192
--gpu-memory-utilization 0.9
--compilation-config '{"cudagraph_capture_sizes":[16,32,128,160,256], "cudagraph_mode":"FULL_DECODE_ONLY"}'
--speculative-config '{"method":"eagle3", "model":"lightseekorg/kimi-k2.5-eagle3", "num_speculative_tokens":3}'
--mm-processor-cache-gb 0
--mm-encoder-tp-mode data
-
envs:
<<: *env_common
server_cmd: >
vllm serve Eco-Tech/Kimi-K2.5-W4A8
--host 0.0.0.0
--headless
--port $SERVER_PORT
--quantization ascend
--allowed-local-media-path /
--trust-remote-code
--no-enable-prefix-caching
--seed 42
--data-parallel-size 2
--data-parallel-size-local 1
--data-parallel-start-rank 1
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 8
--enable-expert-parallel
--max-num-seqs 64
--max-model-len 51200
--max-num-batched-tokens 8192
--gpu-memory-utilization 0.9
--compilation-config '{"cudagraph_capture_sizes":[16,32,128,160,256], "cudagraph_mode":"FULL_DECODE_ONLY"}'
--speculative-config '{"method":"eagle3", "model":"lightseekorg/kimi-k2.5-eagle3", "num_speculative_tokens":3}'
--mm-processor-cache-gb 0
--mm-encoder-tp-mode data
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 320
max_out_len: 1500
batch_size: 80
trust_remote_code: True
request_rate: 0
baseline: 1
threshold: 0.97

74
tests/e2e/nightly/multi_node/internal_dp/config/Qwen3-235B-A22B-A2.yaml Normal file
View File

@@ -0,0 +1,74 @@
test_name: "test Qwen3-235B-A22B multi-dp on A2"
model: "Qwen/Qwen3-235B-A22B"
num_nodes: 2
npu_per_node: 8
env_common: &env_common
VLLM_USE_MODELSCOPE: true
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
NUMEXPR_MAX_THREADS: 128
TASK_QUEUE_ENABLE: 1
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-235B-A22B"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 1
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 8
--seed 1024
--enable-expert-parallel
--max-num-seqs 128
--max-model-len 40960
--max-num-batched-tokens 2048
--trust-remote-code
--gpu-memory-utilization 0.9
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-235B-A22B"
--headless
--data-parallel-size 2
--data-parallel-size-local 1
--data-parallel-start-rank 1
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 8
--seed 1024
--max-num-seqs 128
--max-model-len 40960
--max-num-batched-tokens 2048
--enable-expert-parallel
--trust-remote-code
--gpu-memory-utilization 0.9
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 2800
max_out_len: 1500
batch_size: 256
request_rate: 4.8
baseline: 1
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k-lite
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
max_out_len: 7680
batch_size: 256
baseline: 96
threshold: 10

77
tests/e2e/nightly/multi_node/internal_dp/config/Qwen3-235B-A22B.yaml Normal file
View File

@@ -0,0 +1,77 @@
test_name: "test Qwen3-235B-A22B multi-dp"
model: "Qwen/Qwen3-235B-A22B"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: AIV
TASK_QUEUE_ENABLE: 1
VLLM_USE_MODELSCOPE: true
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
NUMEXPR_MAX_THREADS: 128
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-235B-A22B"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 4
--data-parallel-size-local 2
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 8
--seed 1024
--enable-expert-parallel
--max-num-seqs 32
--max-model-len 8192
--max-num-batched-tokens 8192
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-235B-A22B"
--headless
--data-parallel-size 4
--data-parallel-size-local 2
--data-parallel-start-rank 2
--data-parallel-address $MASTER_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 8
--seed 1024
--max-num-seqs 32
--max-model-len 8192
--max-num-batched-tokens 8192
--enable-expert-parallel
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 2800
max_out_len: 1500
batch_size: 700
request_rate: 11.2
baseline: 1
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
max_out_len: 7680
batch_size: 512
baseline: 95
threshold: 3

93
tests/e2e/nightly/multi_node/internal_dp/config/Qwen3-235B-W8A8-EPLB.yaml Normal file
View File

@@ -0,0 +1,93 @@
test_name: "test Qwen3-235B-A22B-W8A8 EPLB"
model: "vllm-ascend/Qwen3-235B-A22B-W8A8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: AIV
VLLM_USE_MODELSCOPE: true
TASK_QUEUE_ENABLE: 1
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
DYNAMIC_EPLB: true
disaggregated_prefill:
enabled: true
prefiller_host_index: [0]
decoder_host_index: [1]
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve "vllm-ascend/Qwen3-235B-A22B-W8A8"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--tensor-parallel-size 8
--seed 1024
--enable-expert-parallel
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 8192
--quantization ascend
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
--additional-config
'{"eplb_config": {"dynamic_eplb":true,"expert_heat_collection_interval":50,"algorithm_execution_interval":5}}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve "vllm-ascend/Qwen3-235B-A22B-W8A8"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--tensor-parallel-size 8
--seed 1024
--quantization ascend
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 8192
--enable-expert-parallel
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30200",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
--additional-config
'{"eplb_config": {"dynamic_eplb":true,"expert_heat_collection_interval":600,"algorithm_execution_interval":50}}'
benchmarks:

100
tests/e2e/nightly/multi_node/internal_dp/config/Qwen3-235B-W8A8-longseq.yaml Normal file
View File

@@ -0,0 +1,100 @@
test_name: "test Qwen3-235B-A22B-W8A8-longseq disaggregated_prefill"
model: "vllm-ascend/Qwen3-235B-A22B-W8A8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: AIV
VLLM_USE_MODELSCOPE: true
TASK_QUEUE_ENABLE: 1
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
NUMEXPR_MAX_THREADS: 128
DYNAMIC_EPLB: true
disaggregated_prefill:
enabled: true
prefiller_host_index: [0]
decoder_host_index: [1]
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve "vllm-ascend/Qwen3-235B-A22B-W8A8"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 1
--decode-context-parallel-size 2
--prefill-context-parallel-size 2
--tensor-parallel-size 8
--cp-kv-cache-interleave-size 128
--seed 1024
--enforce-eager
--enable-expert-parallel
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 8192
--quantization ascend
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 1,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
--additional-config
'{"dynamic_eplb":true}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve "vllm-ascend/Qwen3-235B-A22B-W8A8"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--decode-context-parallel-size 2
--prefill-context-parallel-size 1
--tensor-parallel-size 8
--cp-kv-cache-interleave-size 128
--seed 1024
--quantization ascend
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 8192
--enable-expert-parallel
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--compilation_config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30100",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 1,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
--additional-config
'{"dynamic_eplb":true}'
benchmarks:

89
tests/e2e/nightly/multi_node/internal_dp/config/Qwen3-235B-W8A8.yaml Normal file
View File

@@ -0,0 +1,89 @@
test_name: "test Qwen3-235B-A22B-W8A8 disaggregated_prefill"
model: "vllm-ascend/Qwen3-235B-A22B-W8A8"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
HCCL_OP_EXPANSION_MODE: AIV
VLLM_USE_MODELSCOPE: true
TASK_QUEUE_ENABLE: 1
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
NUMEXPR_MAX_THREADS: 128
disaggregated_prefill:
enabled: true
prefiller_host_index: [0]
decoder_host_index: [1]
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve "vllm-ascend/Qwen3-235B-A22B-W8A8"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--tensor-parallel-size 8
--seed 1024
--enable-expert-parallel
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 8192
--quantization ascend
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve "vllm-ascend/Qwen3-235B-A22B-W8A8"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--tensor-parallel-size 8
--seed 1024
--quantization ascend
--max-num-seqs 16
--max-model-len 8192
--max-num-batched-tokens 8192
--enable-expert-parallel
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30200",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 2,
"tp_size": 8
}
}
}'
benchmarks:

118
tests/e2e/nightly/multi_node/internal_dp/config/Qwen3-235B-disagg-pd.yaml Normal file
View File

@@ -0,0 +1,118 @@
test_name: "test Qwen3-235B-A22B disaggregated_prefill"
model: "Qwen/Qwen3-235B-A22B"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
VLLM_USE_MODELSCOPE: true
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
HCCL_BUFFSIZE: 1024
HCCL_OP_EXPANSION_MODE: "AIV"
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
VLLM_ASCEND_ENABLE_FLASHCOMM1: 1
VLLM_ASCEND_ENABLE_FUSED_MC2: 2
TASK_QUEUE_ENABLE: 1
SERVER_PORT: 8080
disaggregated_prefill:
enabled: true
prefiller_host_index: [0]
decoder_host_index: [1]
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-235B-A22B"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 8
--seed 1024
--max-num-seqs 32
--max-model-len 8192
--max-num-batched-tokens 8192
--enable-expert-parallel
--trust-remote-code
--gpu-memory-utilization 0.9
--no-enable-prefix-caching
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 4,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-235B-A22B"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 4
--data-parallel-size-local 4
--data-parallel-start-rank 0
--data-parallel-address $LOCAL_IP
--data-parallel-rpc-port 13389
--tensor-parallel-size 4
--seed 1024
--max-num-seqs 32
--max-model-len 8192
--max-num-batched-tokens 8192
--enable-expert-parallel
--trust-remote-code
--gpu-memory-utilization 0.9
--no-enable-prefix-caching
--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30100",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 4,
"tp_size": 4
}
}
}'
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
request_conf: vllm_api_stream_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
num_prompts: 2800
max_out_len: 1500
batch_size: 700
request_rate: 11.2
baseline: 1
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/gsm8k-lite
request_conf: vllm_api_general_chat
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
max_out_len: 7680
batch_size: 512
baseline: 97
threshold: 10

108
tests/e2e/nightly/multi_node/internal_dp/config/Qwen3-VL-235B-disagg-pd.yaml Normal file
View File

@@ -0,0 +1,108 @@
test_name: "test Qwen3-VL-235B-A22B disaggregated_prefill"
model: "Qwen/Qwen3-VL-235B-A22B-Instruct"
num_nodes: 2
npu_per_node: 16
env_common: &env_common
VLLM_USE_MODELSCOPE: true
HCCL_BUFFSIZE: 1024
SERVER_PORT: 8080
OMP_PROC_BIND: false
OMP_NUM_THREADS: 1
HCCL_OP_EXPANSION_MODE: "AIV"
TASK_QUEUE_ENABLE: 1
PYTORCH_NPU_ALLOC_CONF: expandable_segments:True
disaggregated_prefill:
enabled: true
prefiller_host_index: [0]
decoder_host_index: [1]
deployment:
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-VL-235B-A22B-Instruct"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 2
--data-parallel-size-local 2
--tensor-parallel-size 8
--seed 1024
--enable-expert-parallel
--max-num-seqs 32
--max-model-len 8192
--max-num-batched-tokens 8192
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_producer",
"kv_port": "30000",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 4,
"tp_size": 4
}
}
}'
-
envs:
<<: *env_common
server_cmd: >
vllm serve "Qwen/Qwen3-VL-235B-A22B-Instruct"
--host 0.0.0.0
--port $SERVER_PORT
--data-parallel-size 4
--data-parallel-size-local 4
--tensor-parallel-size 4
--seed 1024
--enable-expert-parallel
--max-num-seqs 32
--max-model-len 8192
--max-num-batched-tokens 8192
--trust-remote-code
--no-enable-prefix-caching
--gpu-memory-utilization 0.9
--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
--kv-transfer-config
'{"kv_connector": "MooncakeConnectorV1",
"kv_role": "kv_consumer",
"kv_port": "30200",
"kv_connector_extra_config": {
"prefill": {
"dp_size": 2,
"tp_size": 8
},
"decode": {
"dp_size": 4,
"tp_size": 4
}
}
}'
benchmarks:
perf:
case_type: performance
dataset_path: vllm-ascend/textvqa-perf-1080p
request_conf: vllm_api_stream_chat
dataset_conf: textvqa/textvqa_gen_base64
num_prompts: 2800
max_out_len: 1500
batch_size: 64
request_rate: 11.2
baseline: 1
threshold: 0.97
acc:
case_type: accuracy
dataset_path: vllm-ascend/textvqa-lite
request_conf: vllm_api_stream_chat
dataset_conf: textvqa/textvqa_gen_base64
max_out_len: 7680
batch_size: 64
baseline: 85
threshold: 5

0
tests/e2e/nightly/multi_node/internal_dp/scripts/__init__.py Normal file
View File

331
tests/e2e/nightly/multi_node/internal_dp/scripts/multi_node_config.py Normal file
View File

@@ -0,0 +1,331 @@
import logging
import os
import subprocess
from dataclasses import dataclass
from typing import Any
import regex as re
from tests.e2e.nightly.multi_node.scripts.utils import (
get_available_port,
get_net_interface,
load_yaml_mapping,
resolve_cluster_ips,
resolve_current_node_index,
setup_logger,
)
setup_logger()
logger = logging.getLogger(__name__)
DEFAULT_CONFIG_BASE_PATH = "tests/e2e/nightly/multi_node/internal_dp/config/"
DEFAULT_SERVER_PORT = 8080
@dataclass(frozen=True)
class NodeInfo:
index: int
ip: str
server_cmd: str
envs: dict[str, Any] | None = None
headless: bool = False
def __post_init__(self):
if not self.ip:
raise ValueError("NodeInfo.ip must not be empty")
def __str__(self) -> str:
return f"NodeInfo(\n index={self.index},\n ip={self.ip},\n headless={self.headless},\n)"
class DisaggregatedPrefillCfg:
def __init__(self, raw_cfg: dict, num_nodes: int):
self.prefiller_indices: list[int] = raw_cfg.get("prefiller_host_index", [])
self.decoder_indices: list[int] = raw_cfg.get("decoder_host_index", [])
if not self.decoder_indices:
raise RuntimeError("decoder_host_index must be provided")
self._validate(num_nodes)
self.decode_start_index = self.decoder_indices[0]
self.num_prefillers = len(self.prefiller_indices)
self.num_decoders = len(self.decoder_indices)
def _validate(self, num_nodes: int):
overlap = set(self.prefiller_indices) & set(self.decoder_indices)
if overlap:
raise AssertionError(f"Prefiller and decoder overlap: {overlap}")
all_indices = self.prefiller_indices + self.decoder_indices
if any(i >= num_nodes for i in all_indices):
raise ValueError("Disaggregated prefill index out of range")
def is_prefiller(self, index: int) -> bool:
return index in self.prefiller_indices
def is_decoder(self, index: int) -> bool:
return index in self.decoder_indices
def master_ip_for_node(self, index: int, nodes: list[NodeInfo]) -> str:
if self.is_prefiller(index):
return nodes[0].ip
return nodes[self.decode_start_index].ip
class DistEnvBuilder:
def __init__(
self,
*,
cur_node: NodeInfo,
master_ip: str,
):
self.cur_ip = cur_node.ip
self.nic_name = get_net_interface(self.cur_ip)
self.master_ip = master_ip
self.base_envs = dict(cur_node.envs or {})
def build(self) -> dict:
envs = dict(self.base_envs)
envs.update(
{
"HCCL_IF_IP": self.cur_ip,
"HCCL_SOCKET_IFNAME": self.nic_name,
"GLOO_SOCKET_IFNAME": self.nic_name,
"TP_SOCKET_IFNAME": self.nic_name,
"LOCAL_IP": self.cur_ip,
"NIC_NAME": self.nic_name,
"MASTER_IP": self.master_ip,
}
)
return {k: str(v) for k, v in envs.items()}
class ProxyLauncher:
def __init__(
self,
*,
nodes: list[NodeInfo],
envs: dict,
proxy_port: int,
cur_index: int,
disagg_cfg: DisaggregatedPrefillCfg | None = None,
):
self.nodes = nodes
self.cfg = disagg_cfg
self.server_port = envs.get("SERVER_PORT", DEFAULT_SERVER_PORT)
self.proxy_port = proxy_port
self.proxy_script = envs.get(
"DISAGGREGATED_PREFILL_PROXY_SCRIPT",
"examples/disaggregated_prefill_v1/load_balance_proxy_server_example.py",
)
self.envs = envs
self.is_master = cur_index == 0
self.cur_ip = nodes[cur_index].ip
self.process: subprocess.Popen[bytes] | None = None
def __enter__(self):
if not self.is_master or self.cfg is None:
logger.info("Not launching proxy on non-master node")
return self
prefiller_ips = [self.nodes[i].ip for i in self.cfg.prefiller_indices if not self.nodes[i].headless]
decoder_ips = [self.nodes[i].ip for i in self.cfg.decoder_indices if not self.nodes[i].headless]
cmd = [
"python",
self.proxy_script,
"--host",
self.cur_ip,
"--port",
str(self.proxy_port),
"--prefiller-hosts",
*prefiller_ips,
"--prefiller-ports",
*[str(self.server_port)] * len(prefiller_ips),
"--decoder-hosts",
*decoder_ips,
"--decoder-ports",
*[str(self.server_port)] * len(decoder_ips),
]
logger.info("Launching proxy: %s", " ".join(cmd))
self.process = subprocess.Popen(cmd, env={**os.environ, **self.envs})
return self
def __exit__(self, exc_type, exc, tb):
if not self.process:
return
logger.info("Stopping proxy server...")
self.process.terminate()
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
self.process.kill()
class MultiNodeConfig:
def __init__(
self,
*,
model: str,
test_name: str,
nodes: list[NodeInfo],
npu_per_node: int,
disaggregated_prefill: dict | None,
benchmark_cases: list[dict],
special_dependencies: dict,
):
self.model = model
self.test_name = test_name
self.nodes = nodes
self.npu_per_node = npu_per_node
self.benchmark_cases = benchmark_cases
self.cur_index = self._resolve_cur_index()
self.cur_node = self.nodes[self.cur_index]
self.special_dependencies = special_dependencies
self.disagg_cfg = DisaggregatedPrefillCfg(disaggregated_prefill, len(nodes)) if disaggregated_prefill else None
master_ip = (
self.disagg_cfg.master_ip_for_node(self.cur_index, self.nodes) if self.disagg_cfg else self.nodes[0].ip
)
self.proxy_port = get_available_port()
self.envs = DistEnvBuilder(
cur_node=self.cur_node,
master_ip=master_ip,
).build()
logger.info("Node %d envs: %s", self.cur_index, self.envs)
self.server_cmd = self._expand_env(self.cur_node.server_cmd)
def _resolve_cur_index(self) -> int:
return resolve_current_node_index([node.ip for node in self.nodes])
def _expand_env(self, cmd: str) -> str:
pattern = re.compile(r"\$(\w+)|\$\{(\w+)\}")
def repl(m):
key = m.group(1) or m.group(2)
return self.envs.get(key, m.group(0))
return pattern.sub(repl, cmd)
@property
def world_size(self) -> int:
return len(self.nodes) * self.npu_per_node
@property
def is_master(self) -> bool:
return self.cur_index == 0
@property
def server_port(self) -> int:
return self.envs.get("SERVER_PORT", DEFAULT_SERVER_PORT)
@property
def master_ip(self) -> str:
return self.nodes[0].ip
@property
def benchmark_endpoint(self) -> tuple[str, int]:
"""
Endpoint used by benchmark clients.
"""
master_ip = self.nodes[0].ip
server_port = self.envs.get("SERVER_PORT", DEFAULT_SERVER_PORT)
if self.disagg_cfg:
return master_ip, self.proxy_port
return master_ip, server_port
class MultiNodeConfigLoader:
"""Load MultiNodeConfig from yaml file."""
DEFAULT_CONFIG_NAME = "DeepSeek-V3.yaml"
@classmethod
def from_yaml(cls, yaml_path: str | None = None) -> MultiNodeConfig:
config = cls._load_yaml(yaml_path)
cls._validate_root(config)
nodes = cls._parse_nodes(config)
benchmarks = cls._parse_benchmarks(config)
return MultiNodeConfig(
model=config["model"],
test_name=config.get("test_name", "untitled_test"),
nodes=nodes,
npu_per_node=config.get("npu_per_node", 16),
disaggregated_prefill=config.get("disaggregated_prefill"),
special_dependencies=config.get("special_dependencies", {}),
benchmark_cases=list(benchmarks.values()),
)
@classmethod
def _load_yaml(cls, yaml_path: str | None) -> dict:
return load_yaml_mapping(
yaml_path,
default_name=cls.DEFAULT_CONFIG_NAME,
default_base_path=DEFAULT_CONFIG_BASE_PATH,
description="config",
)
@staticmethod
def _validate_root(cfg: dict):
required = ["model", "deployment", "num_nodes", "npu_per_node", "benchmarks"]
missing = [k for k in required if k not in cfg]
if missing:
raise KeyError(f"Missing required config fields: {missing}")
@classmethod
def _parse_nodes(cls, cfg: dict) -> list[NodeInfo]:
num_nodes = cfg["num_nodes"]
deployments = cfg["deployment"]
if len(deployments) != num_nodes:
raise AssertionError(f"deployment size ({len(deployments)}) != num_nodes ({num_nodes})")
for idx, deploy in enumerate(deployments):
if deploy.get("envs") is None:
raise KeyError(f"deployment[{idx}].envs is required for multi-node configs")
cluster_ips = cls._resolve_cluster_ips(cfg, num_nodes)
nodes: list[NodeInfo] = []
for idx, deploy in enumerate(deployments):
cmd = deploy.get("server_cmd", "")
envs = deploy["envs"]
nodes.append(
NodeInfo(
index=idx,
ip=cluster_ips[idx],
server_cmd=cmd,
envs=envs,
headless="--headless" in cmd,
)
)
return nodes
@staticmethod
def _parse_benchmarks(cfg: dict) -> dict:
benchmarks = cfg.get("benchmarks") or {}
for name, case in benchmarks.items():
case["case_name"] = name
return benchmarks
@staticmethod
def _resolve_cluster_ips(cfg: dict, num_nodes: int) -> list[str]:
return resolve_cluster_ips(
cfg,
num_nodes,
cluster_hosts_log_message=(
"Using cluster_hosts from config. This typically indicates that your current environment is a "
"non-Kubernetes environment."
),
dns_log_message="Resolving cluster IPs via DNS...",
)

207
tests/e2e/nightly/multi_node/internal_dp/scripts/test_multi_node.py Normal file
View File

@@ -0,0 +1,207 @@
import json
import logging
import os
import shlex
import subprocess
import sys
from typing import Any
import pytest
import vllm
from tests.e2e.conftest import RemoteOpenAIServer
from tests.e2e.nightly.multi_node.internal_dp.scripts.multi_node_config import (
MultiNodeConfig,
MultiNodeConfigLoader,
ProxyLauncher,
)
from tests.e2e.nightly.multi_node.scripts.benchmark_results import (
build_task_entry,
extract_hardware,
filter_environment,
write_results_json,
)
from tools.aisbench import run_aisbench_cases
logger = logging.getLogger(__name__)
_FEATURE_ENVS: dict[str, str] = {
"VLLM_ASCEND_ENABLE_FLASHCOMM": "flashcomm",
"VLLM_ASCEND_ENABLE_FLASHCOMM1": "flashcomm1",
"VLLM_ASCEND_ENABLE_TOPK_OPTIMIZE": "topk_optimize",
"VLLM_ASCEND_ENABLE_MATMUL_ALLREDUCE": "matmul_allreduce",
"VLLM_ASCEND_ENABLE_MLAPO": "mlapo",
"VLLM_ASCEND_ENABLE_FUSED_MC2": "fused_mc2",
}
def _extract_dtype(config: MultiNodeConfig) -> str:
"""Determine weight dtype: w8a8 if model name contains 'w8a8' and any node uses --quantization ascend."""
has_w8a8 = "w8a8" in config.model.lower()
has_quant_ascend = any("--quantization ascend" in node.server_cmd for node in config.nodes)
return "w8a8" if (has_w8a8 and has_quant_ascend) else "bf16"
def _cmd_to_list(server_cmd: list[str] | str) -> list[str]:
"""Normalize server_cmd to a list of argument strings."""
if isinstance(server_cmd, str):
try:
return shlex.split(server_cmd)
except ValueError:
return server_cmd.split()
return list(server_cmd)
def _extract_server_cmd_value(cmd_list: list[str], flag: str) -> str | None:
"""Return the value following `flag` in a command list, or None."""
try:
idx = cmd_list.index(flag)
return cmd_list[idx + 1]
except (ValueError, IndexError):
return None
def _parse_json_flag(cmd_list: list[str], flag: str) -> dict[str, Any]:
"""Extract and JSON-parse the value following `flag` in a command list."""
val = _extract_server_cmd_value(cmd_list, flag)
if not val:
return {}
try:
return json.loads(val)
except (json.JSONDecodeError, ValueError):
return {}
def _extract_features(server_cmd: list[str] | str, envs: dict[str, Any]) -> list[str]:
"""Extract enabled feature names from server_cmd and environment variables."""
cmd_list = _cmd_to_list(server_cmd)
features: list[str] = []
# Features from --additional-config JSON
additional = _parse_json_flag(cmd_list, "--additional-config")
if additional.get("enable_weight_nz_layout"):
features.append("weight_nz_layout")
wp = additional.get("weight_prefetch_config") or {}
if isinstance(wp, dict) and wp.get("enabled"):
features.append("weight_prefetch")
tc = additional.get("torchair_graph_config") or {}
if isinstance(tc, dict) and tc.get("enabled"):
features.append("torchair_graph")
asc = additional.get("ascend_scheduler_config") or {}
if isinstance(asc, dict) and asc.get("enabled"):
features.append("ascend_scheduler")
# Features from --compilation-config JSON
compilation = _parse_json_flag(cmd_list, "--compilation-config")
if compilation.get("cudagraph_mode"):
features.append("aclgraph")
# Features from --speculative-config JSON
speculative = _parse_json_flag(cmd_list, "--speculative-config")
if speculative:
features.append(speculative.get("method", "speculative"))
# Features from direct flags
if "--enable-expert-parallel" in cmd_list:
features.append("expert_parallel")
# Features from environment variables
for env_key, feature_name in _FEATURE_ENVS.items():
val = str(envs.get(env_key, "0"))
if val not in ("0", "", "false", "False"):
features.append(feature_name)
if int(envs.get("VLLM_ASCEND_FLASHCOMM2_PARALLEL_SIZE", 0)) > 0:
features.append("flashcomm2")
return features
def _build_serve_cmd(config: MultiNodeConfig) -> dict[str, Any]:
"""Build serve_cmd dict: pd format for disaggregated, dp format for multi-node."""
if config.disagg_cfg:
pd: dict[str, str] = {}
for node in config.nodes:
idx = node.index
if config.disagg_cfg.is_prefiller(idx):
n = config.disagg_cfg.prefiller_indices.index(idx)
pd[f"prefill-{n}"] = node.server_cmd
elif config.disagg_cfg.is_decoder(idx):
n = config.disagg_cfg.decoder_indices.index(idx)
pd[f"decode-{n}"] = node.server_cmd
return {"pd": pd}
return {"dp": {f"node{node.index}": node.server_cmd for node in config.nodes}}
def _save_benchmark_results_json(config: MultiNodeConfig, results: list[Any]) -> None:
"""Serialize acc & perf benchmark results to a JSON file under benchmark_results/."""
runner = os.environ.get("VLLM_CI_RUNNER", "")
# Filter out None benchmark cases; results align with the non-None ones in order
valid_items = [(case["case_name"], case) for case in config.benchmark_cases]
tasks = [build_task_entry(key, case_cfg, result) for (key, case_cfg), result in zip(valid_items, results)]
output: dict[str, Any] = {
"model_name": config.model,
"hardware": extract_hardware(runner),
"dtype": _extract_dtype(config),
"feature": _extract_features(config.nodes[0].server_cmd, config.envs),
"vllm_version": vllm.__version__,
"vllm_ascend_version": os.environ.get("VLLM_ASCEND_REF", ""),
"tasks": tasks,
"serve_cmd": _build_serve_cmd(config),
"environment": filter_environment(config.envs),
}
job_name = os.environ.get("BENCHMARK_JOB_NAME", "")
write_results_json(output, job_name=job_name)
@pytest.mark.asyncio
async def test_multi_node() -> None:
config = MultiNodeConfigLoader.from_yaml()
if config.special_dependencies:
for k, v in config.special_dependencies.items():
command = [
sys.executable,
"-m",
"pip",
"install",
f"{k}=={v}",
]
subprocess.call(command)
with (
ProxyLauncher(
nodes=config.nodes,
disagg_cfg=config.disagg_cfg,
envs=config.envs,
proxy_port=config.proxy_port,
cur_index=config.cur_index,
) as proxy,
RemoteOpenAIServer(
model=config.model,
vllm_serve_args=config.server_cmd,
server_port=config.server_port,
server_host=config.master_ip,
env_dict=config.envs,
auto_port=False,
proxy_port=proxy.proxy_port,
disaggregated_prefill=config.disagg_cfg,
nodes_info=config.nodes,
max_wait_seconds=2800,
) as server,
):
host, port = config.benchmark_endpoint
if config.is_master:
results = run_aisbench_cases(
model=config.model,
port=port,
aisbench_cases=config.benchmark_cases,
host_ip=host,
)
_save_benchmark_results_json(config, results)
else:
# We should keep listening on the master node's server url determining when to exit.
server.hang_until_terminated(f"http://{host}:{config.server_port}/health")

28
tests/e2e/nightly/multi_node/internal_dp/scripts/utils.py Normal file
View File

@@ -0,0 +1,28 @@
import os
from tests.e2e.nightly.multi_node.scripts.utils import (
get_all_ipv4,
get_available_port,
get_cluster_ips,
get_net_interface,
setup_logger,
temp_env,
)
DISAGGEGATED_PREFILL_PORT = 5333
DEFAULT_CONFIG_BASE_PATH = "tests/e2e/nightly/multi_node/internal_dp/config/"
CONFIG_BASE_PATH = os.getenv("CONFIG_BASE_PATH") or DEFAULT_CONFIG_BASE_PATH
DEFAULT_SERVER_PORT = 8080
__all__ = [
"CONFIG_BASE_PATH",
"DEFAULT_CONFIG_BASE_PATH",
"DEFAULT_SERVER_PORT",
"DISAGGEGATED_PREFILL_PORT",
"get_all_ipv4",
"get_available_port",
"get_cluster_ips",
"get_net_interface",
"setup_logger",
"temp_env",
]