@@ -0,0 +1,149 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
_envs: &envs
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
VLLM_USE_V1: "1"
|
||||
HCCL_BUFFSIZE: "256"
|
||||
ASCEND_AGGREGATE_ENABLE: "1"
|
||||
ASCEND_TRANSPORT_PRINT: "1"
|
||||
ACL_OP_INIT_MODE: "1"
|
||||
ASCEND_A3_ENABLE: "1"
|
||||
VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300000"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--enable-expert-parallel"
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--max-model-len"
|
||||
- "67000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "4096"
|
||||
- "--max-num-seqs"
|
||||
- "8"
|
||||
- "--trust-remote-code"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.95"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,2,4,8,16,24,32,40,48], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--speculative-config"
|
||||
- '{"num_speculative_tokens": 3, "method":"deepseek_mtp"}'
|
||||
- "--reasoning-parser"
|
||||
- "deepseek_v3"
|
||||
- "--tokenizer-mode"
|
||||
- "deepseek_v32"
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
perf_16384_bs4:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs1200_deepseek
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 26.10
|
||||
threshold: 0.97
|
||||
perf_16384_bs16:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs1200_deepseek
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 1024
|
||||
batch_size: 4
|
||||
request_rate: 0.5
|
||||
baseline: 60.53
|
||||
threshold: 0.97
|
||||
perf_32768_bs4:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs1000_deepseek
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 512
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 22.68
|
||||
threshold: 0.97
|
||||
perf_32768_bs8:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs1000_deepseek
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 512
|
||||
batch_size: 2
|
||||
request_rate: 0.2
|
||||
baseline: 32.73
|
||||
threshold: 0.97
|
||||
perf_65536_bs4:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs400_deepseek
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 22.77
|
||||
threshold: 0.97
|
||||
perf_65536_bs8:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs400_deepseek
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 1024
|
||||
batch_size: 2
|
||||
request_rate: 1
|
||||
baseline: 33.80
|
||||
threshold: 0.97
|
||||
_benchmarks_2048: &benchmarks_2048
|
||||
perf_16384_bs4:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in2048-bs4000-ds3.2
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 128
|
||||
max_out_len: 2048
|
||||
batch_size: 32
|
||||
request_rate: 0
|
||||
baseline: 220
|
||||
threshold: 0.97
|
||||
|
||||
test_cases:
|
||||
- name: "DeepSeek-V3.2-W8A8-weekly"
|
||||
model: "vllm-ascend/DeepSeek-V3.2-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
- name: "DeepSeek-V3.2-W8A8-mix-perf"
|
||||
model: "vllm-ascend/DeepSeek-V3.2-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--async-scheduling"
|
||||
benchmarks:
|
||||
<<: *benchmarks_2048
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
LCCL_DETERMINISTIC: "1"
|
||||
VLLM_USE_V1: "1"
|
||||
HCCL_DETERMINISTIC: "true"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
ATB_MATMUL_SHUFFLE_K_ENABLE: "0"
|
||||
ATB_LLM_LCOC_ENABLE: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--enable-expert-parallel"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--max-num-batched-tokens"
|
||||
- "4096"
|
||||
- "--max-num-seqs"
|
||||
- "16"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--async-scheduling"
|
||||
- "--speculative-config"
|
||||
- '{"num_speculative_tokens": 1, "method":"deepseek_mtp"}'
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--disable-log-stats"
|
||||
- "--compilation_config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 64
|
||||
max_out_len: 1500
|
||||
batch_size: 16
|
||||
request_rate: 0
|
||||
baseline: 411
|
||||
threshold: 0.95
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Deepseek_R1_W8A8_Performance_0006"
|
||||
model: "Eco-Tech/DeepSeek-R1-0528-w8a8-mtp-QuaRot"
|
||||
envs:
|
||||
<<: *envs
|
||||
OMP_PROC_BIND: "1"
|
||||
OMP_NUM_THREADS: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "5500"
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
|
||||
- name: "chunked_prefill_func_test_005"
|
||||
model: "Eco-Tech/DeepSeek-R1-0528-w8a8-mtp-QuaRot"
|
||||
envs:
|
||||
<<: *envs
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "10"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "32768"
|
||||
@@ -0,0 +1,71 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
LCCL_DETERMINISTIC: "1"
|
||||
VLLM_USE_V1: "1"
|
||||
HCCL_DETERMINISTIC: "true"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
ATB_MATMUL_SHUFFLE_K_ENABLE: "0"
|
||||
ATB_LLM_LCOC_ENABLE: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "100"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--data-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size-local"
|
||||
- "8"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--max-num-batched-tokens"
|
||||
- "6000"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--trust-remote-code"
|
||||
- "--enforce-eager"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.92"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--reasoning-parser"
|
||||
- "deepseek_r1"
|
||||
- "--additional-config"
|
||||
- '{"ascend_scheduler_config":{"enabled":false},"torchair_graph_config":{"enabled":false,"enable_multistream_shared_expert":false}}'
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k_lite
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 4096
|
||||
num_prompts: 32
|
||||
batch_size: 16
|
||||
baseline: 100
|
||||
threshold: 10
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Deepseek_R1_W8A8_reasoning_output_012"
|
||||
model: "vllm-ascend/DeepSeek-R1-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
|
||||
75
tests/e2e/weekly/single_node/configs/GLM-5.yaml
Normal file
75
tests/e2e/weekly/single_node/configs/GLM-5.yaml
Normal file
@@ -0,0 +1,75 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "0"
|
||||
_server_cmd: &server_cmd
|
||||
- "--enable-expert-parallel"
|
||||
- "--tensor-parallel-size"
|
||||
- "16"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "16384"
|
||||
- "--max-num-batched-tokens"
|
||||
- "4096"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.95"
|
||||
- "--max-num-seqs"
|
||||
- "8"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--additional-config"
|
||||
- '{"multistream_overlap_shared_expert": false, "ascend_compilation_config": {"enable_npugraph_ex": true}}'
|
||||
- "--speculative-config"
|
||||
- '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}'
|
||||
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k-lite
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 8192
|
||||
batch_size: 8
|
||||
baseline: 95
|
||||
threshold: 10
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs400
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 1500
|
||||
batch_size: 8
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "GLM-5-TP16-DP1-decodegraph"
|
||||
model: "Eco-Tech/GLM-5-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes": [4,8,16,32,64,128,256,512], "cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
|
||||
158
tests/e2e/weekly/single_node/configs/GLM-5_1-W8A8_A3_weekly.yaml
Normal file
158
tests/e2e/weekly/single_node/configs/GLM-5_1-W8A8_A3_weekly.yaml
Normal file
@@ -0,0 +1,158 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
HCCL_BUFFSIZE: "200"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--enable-expert-parallel"
|
||||
- "--tensor-parallel-size"
|
||||
- "16"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--max-num-batched-tokens"
|
||||
- "4096"
|
||||
- "--trust-remote-code"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--enable-chunked-prefill"
|
||||
- "--additional-config"
|
||||
- '{"multistream_overlap_shared_expert":true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"num_speculative_tokens": 3, "method": "deepseek_mtp","enforce_eager": true}'
|
||||
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs100_glm
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 20
|
||||
max_out_len: 1024
|
||||
batch_size: 5
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs8: &benchmarks_bs8
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs100_glm
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 512
|
||||
batch_size: 2
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs32: &benchmarks_bs32
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in32768_bs100_glm
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 32
|
||||
max_out_len: 512
|
||||
batch_size: 8
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs10: &benchmarks_bs10
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs100_glm
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 20
|
||||
max_out_len: 1024
|
||||
batch_size: 5
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
# 16384 1024 0% 20
|
||||
- name: "GLM-5_1-w8a8-High—Throughput-bs20"
|
||||
model: "Eco-Tech/GLM-5.1-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--max-model-len"
|
||||
- "20480"
|
||||
- "--max-num-seqs"
|
||||
- "48"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.95"
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
#32768 512 0% 8
|
||||
- name: "GLM-5_1-w8a8-High—Throughput-bs8"
|
||||
model: "Eco-Tech/GLM-5.1-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--max-model-len"
|
||||
- "33792"
|
||||
- "--max-num-seqs"
|
||||
- "48"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.95"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs8
|
||||
|
||||
#32768 512 90% 32
|
||||
- name: "GLM-5_1-w8a8-High—Throughput-bs32"
|
||||
model: "Eco-Tech/GLM-5.1-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "33792"
|
||||
- "--max-num-seqs"
|
||||
- "48"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.95"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs32
|
||||
#65536 1024 90% 10
|
||||
- name: "GLM-5_1-w8a8-High—Throughput-bs10"
|
||||
model: "Eco-Tech/GLM-5.1-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--max-num-seqs"
|
||||
- "8"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.97"
|
||||
- "--enable-prefix-caching"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs10
|
||||
191
tests/e2e/weekly/single_node/configs/Kimi-K2.6-32k-512.yaml
Normal file
191
tests/e2e/weekly/single_node/configs/Kimi-K2.6-32k-512.yaml
Normal file
@@ -0,0 +1,191 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
HCCL_BUFFSIZE: "512"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_ENABLE_NZ: "1"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--enable-expert-parallel"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--data-parallel-size"
|
||||
- "4"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
|
||||
_benchmark_TPOT50_32K_0_5K: &_benchmark_TPOT50_32K_0_5K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs200_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 512
|
||||
batch_size: 4
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT50_32K_0_5K_pc90: &_benchmark_TPOT50_32K_0_5K_pc90
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in32768_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 40
|
||||
max_out_len: 512
|
||||
batch_size: 10
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT20_32K_0_5K: &_benchmark_TPOT20_32K_0_5K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs200_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 512
|
||||
batch_size: 1
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT20_32K_0_5K_pc90: &_benchmark_TPOT20_32K_0_5K_pc90
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in32768_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 512
|
||||
batch_size: 1
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Kimi-K2.6-W4A8-TOPT50-32k-0.5k"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
HCCL_BUFFSIZE: "450"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--max-model-len"
|
||||
- "33792"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[16,32,48,64,80,96,112,128], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_32K_0_5K
|
||||
- name: "Kimi-K2.6-W4A8-TOPT50-32k-0.5k-prefix-cache90"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
HCCL_BUFFSIZE: "450"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-prefix-caching"
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--max-model-len"
|
||||
- "33792"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[16,32,48,64,80,96,112,128], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_32K_0_5K_pc90
|
||||
- name: "Kimi-K2.6-W4A8-TOPT20-32k-0.5k"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
HCCL_BUFFSIZE: "450"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--max-model-len"
|
||||
- "33792"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[16,32,48,64,80,96,112,128], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_32K_0_5K
|
||||
- name: "Kimi-K2.6-W4A8-TOPT20-32k-0.5k-pc90"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
HCCL_BUFFSIZE: "450"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-prefix-caching"
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--max-model-len"
|
||||
- "33792"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[16,32,48,64,80,96,112,128], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_32K_0_5K_pc90
|
||||
70
tests/e2e/weekly/single_node/configs/Kimi-K2.6-w4a8-A3.yaml
Normal file
70
tests/e2e/weekly/single_node/configs/Kimi-K2.6-w4a8-A3.yaml
Normal file
@@ -0,0 +1,70 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
test_cases:
|
||||
- name: "Kimi-K2.6-W4A8-in3.5k-out1.5k-TPOT50-0-128-32"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_BUFFSIZE: "800"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
DYNAMIC_EPLB: "true"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--safetensors-load-strategy"
|
||||
- 'prefetch'
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "24"
|
||||
- "--max-model-len"
|
||||
- "6144"
|
||||
- "--max-num-batched-tokens"
|
||||
- "4096"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.85"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--additional-config"
|
||||
- '{"eplb_config": {"dynamic_eplb": true},"ascend_compilation_config": {"enable_static_kernel": false}}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch", "torch_profiler_dir": "./vllm_profile", "torch_profiler_with_stack": true}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
- "--speculative-config"
|
||||
- '{"method": "dflash","model": "z-lab/Kimi-K2.5-DFlash", "num_speculative_tokens": 7}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs4096-prefix0-kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 128
|
||||
max_out_len: 1500
|
||||
batch_size: 32
|
||||
request_rate: 0
|
||||
baseline: 1433.4454
|
||||
threshold: 0.97
|
||||
331
tests/e2e/weekly/single_node/configs/Kimi-K2.6.yaml
Normal file
331
tests/e2e/weekly/single_node/configs/Kimi-K2.6.yaml
Normal file
@@ -0,0 +1,331 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
HCCL_BUFFSIZE: "512"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_ENABLE_NZ: "1"
|
||||
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--enable-expert-parallel"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--data-parallel-size"
|
||||
- "4"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
|
||||
_benchmark_acc: &_benchmark_acc
|
||||
acc_aime2025:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/aime2025
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
|
||||
max_out_len: 32768
|
||||
batch_size: 32
|
||||
baseline: 90
|
||||
threshold: 10
|
||||
|
||||
_benchmark_TPOT50_3_5k_1_5k: &_benchmark_TPOT50_3_5k_1_5k
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 392
|
||||
max_out_len: 1500
|
||||
batch_size: 98
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT50_16K_1K: &_benchmark_TPOT50_16K_1K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 56
|
||||
max_out_len: 1024
|
||||
batch_size: 14
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT20_16K_1K: &_benchmark_TPOT20_16K_1K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 32
|
||||
max_out_len: 1024
|
||||
batch_size: 8
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT50_64K_1K_pc90: &_benchmark_TPOT50_64K_1K_pc90
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 56
|
||||
max_out_len: 1024
|
||||
batch_size: 14
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT20_64K_1K_pc90: &_benchmark_TPOT20_64K_1K_pc90
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT50_128K_1K_pc90: &_benchmark_TPOT50_128K_1K_pc90
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 1024
|
||||
batch_size: 2
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
_benchmark_TPOT20_128K_1K_pc90: &_benchmark_TPOT20_128K_1K_pc90
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
trust_remote_code: true
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Kimi-K2.6-W4A8-Case"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "32768"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "500"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[48,64,80,96,112,128], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp":true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_3_5k_1_5k
|
||||
- name: "Kimi-K2.6-W4A8-TOPT50-16k-1k"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "32768"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--max-num-seqs"
|
||||
- "16"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[4,8,12,16,20], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp":true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_16K_1K
|
||||
- name: "Kimi-K2.6-W4A8-TOPT20-16k-1k"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "18000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--max-num-seqs"
|
||||
- "1"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[8], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":7}'
|
||||
- "--tool-call-parser"
|
||||
- "kimi_k2"
|
||||
- "--reasoning-parser"
|
||||
- "kimi_k2"
|
||||
- "--enable-auto-tool-choice"
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_16K_1K
|
||||
- name: "Kimi-K2.6-W4A8-TOPT50-64k-1k-pc90"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
HCCL_BUFFSIZE: "400"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--tool-call-parser"
|
||||
- "kimi_k2"
|
||||
- "--reasoning-parser"
|
||||
- "kimi_k2"
|
||||
- "--enable-auto-tool-choice"
|
||||
- "--api-server-count"
|
||||
- "1"
|
||||
- "--aggregate-engine-logging"
|
||||
- "--max-num-seqs"
|
||||
- "3"
|
||||
- "--max-model-len"
|
||||
- "67000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--enable-prefix-caching"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[8,12], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_64K_1K_pc90
|
||||
- name: "Kimi-K2.6-W4A8-TOPT20-64k-1k-pc90"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
HCCL_BUFFSIZE: "400"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
server_cmd:
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--tool-call-parser"
|
||||
- "kimi_k2"
|
||||
- "--reasoning-parser"
|
||||
- "kimi_k2"
|
||||
- "--enable-auto-tool-choice"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "16"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "8"
|
||||
- "--max-model-len"
|
||||
- "67000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--enable-prefix-caching"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[4,8,16], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_64K_1K_pc90
|
||||
- name: "Kimi-K2.6-W4A8-TOPT50-128k-1k-pc90"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-seqs"
|
||||
- "16"
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[4,8,12,16,32], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp":true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_128K_1K_pc90
|
||||
- name: "Kimi-K2.6-W4A8-TOPT20-128k-1k-pc90"
|
||||
model: "Eco-Tech/Kimi-K2.6-w4a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-chunked-prefill"
|
||||
- "--max-num-seqs"
|
||||
- "16"
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[4,8,12,16,32], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method":"eagle3", "model":"lightseekorg/kimi-k2.6-eagle3", "num_speculative_tokens":3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_shared_expert_dp":true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_128K_1K_pc90
|
||||
<<: *_benchmark_acc
|
||||
330
tests/e2e/weekly/single_node/configs/Kimi-K2.7-w4a8-A3.yaml
Normal file
330
tests/e2e/weekly/single_node/configs/Kimi-K2.7-w4a8-A3.yaml
Normal file
@@ -0,0 +1,330 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
test_cases:
|
||||
- name: "Kimi-K2.7-W4A8-in3.5k-out1.5k-TPOT50-0-128-32"
|
||||
model: "Eco-Tech/Kimi-K2.7-Code-w4a8"
|
||||
envs:
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_BUFFSIZE: "800"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
DYNAMIC_EPLB: "true"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--safetensors-load-strategy"
|
||||
- 'prefetch'
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "24"
|
||||
- "--max-model-len"
|
||||
- "6144"
|
||||
- "--max-num-batched-tokens"
|
||||
- "4096"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.85"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--additional-config"
|
||||
- '{"eplb_config": {"dynamic_eplb": true},"ascend_compilation_config": {"enable_static_kernel": false}}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch", "torch_profiler_dir": "./vllm_profile", "torch_profiler_with_stack": true}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
- "--speculative-config"
|
||||
- '{"method": "dflash","model": "z-lab/Kimi-K2.5-DFlash", "num_speculative_tokens": 7}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs4096-prefix0-kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 128
|
||||
max_out_len: 1500
|
||||
batch_size: 32
|
||||
request_rate: 0
|
||||
baseline: 1585.1248
|
||||
threshold: 0.97
|
||||
- name: "Kimi-K2.7-W4A8-in3.5k-out1.5k-TPOT20-0-128-32"
|
||||
model: "Eco-Tech/Kimi-K2.7-Code-w4a8"
|
||||
envs:
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_BUFFSIZE: "800"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
DYNAMIC_EPLB: "true"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--safetensors-load-strategy"
|
||||
- 'prefetch'
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "24"
|
||||
- "--max-model-len"
|
||||
- "6144"
|
||||
- "--max-num-batched-tokens"
|
||||
- "4096"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.85"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--additional-config"
|
||||
- '{"eplb_config": {"dynamic_eplb": true},"ascend_compilation_config": {"enable_static_kernel": false}}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch", "torch_profiler_dir": "./vllm_profile", "torch_profiler_with_stack": true}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
- "--speculative-config"
|
||||
- '{"method": "dflash","model": "z-lab/Kimi-K2.5-DFlash", "num_speculative_tokens": 7}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs4096-prefix0-kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 128
|
||||
max_out_len: 1500
|
||||
batch_size: 32
|
||||
request_rate: 0
|
||||
baseline: 1585.1248
|
||||
threshold: 0.97
|
||||
- name: "Kimi-K2.7-W4A8-in64k-out1k-TPOT50-90-24-6"
|
||||
model: "Eco-Tech/Kimi-K2.7-Code-w4a8"
|
||||
envs:
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_BUFFSIZE: "1300"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
DYNAMIC_EPLB: "true"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--safetensors-load-strategy"
|
||||
- 'prefetch'
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--enable-prefix-caching"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "4"
|
||||
- "--max-model-len"
|
||||
- "67000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.87"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--additional-config"
|
||||
- '{"eplb_config": {"dynamic_eplb": true},"ascend_compilation_config": {"enable_static_kernel": false}}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch", "torch_profiler_dir": "./vllm_profile", "torch_profiler_with_stack": true}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
- "--speculative-config"
|
||||
- '{"method": "dflash","model": "z-lab/Kimi-K2.5-DFlash", "num_speculative_tokens": 7}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 24
|
||||
max_out_len: 1024
|
||||
batch_size: 6
|
||||
request_rate: 0
|
||||
baseline: 167.6767
|
||||
threshold: 0.97
|
||||
- name: "Kimi-K2.7-W4A8-in128k-out1k-TPOT50-90-24-6"
|
||||
model: "Eco-Tech/Kimi-K2.7-Code-w4a8"
|
||||
envs:
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
HCCL_BUFFSIZE: "1500"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "0"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "10"
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.92"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch", "torch_profiler_dir": "./vllm_profile", "torch_profiler_with_stack": true}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
- "--speculative-config"
|
||||
- '{"method": "dflash","model": "z-lab/Kimi-K2.5-DFlash", "num_speculative_tokens": 15}'
|
||||
benchmarks:
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/aime2025
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
|
||||
max_out_len: 32768
|
||||
batch_size: 8
|
||||
baseline: 96.67
|
||||
threshold: 10
|
||||
temperature: 1.0
|
||||
top_p: 1
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs1000_kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 24
|
||||
max_out_len: 1024
|
||||
batch_size: 6
|
||||
request_rate: 0
|
||||
baseline: 95.3761
|
||||
threshold: 0.97
|
||||
- name: "Kimi-K2.7-W4A8-in254k-out1k"
|
||||
model: "Eco-Tech/Kimi-K2.7-Code-w4a8"
|
||||
envs:
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_MLAPO: "1"
|
||||
HCCL_BUFFSIZE: "300"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--tensor-parallel-size"
|
||||
- "16"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "1"
|
||||
- "--max-model-len"
|
||||
- "262144"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.95"
|
||||
- "--seed"
|
||||
- "42"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch", "torch_profiler_dir": "./vllm_profile", "torch_profiler_with_stack": true}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in254k-bs16-prefix99-kimi
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 1
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 7.5611
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,61 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "MiniMax-M2.5-w8a8"
|
||||
model: "Eco-Tech/MiniMax-M2.5-w8a8-QuaRot"
|
||||
envs:
|
||||
HCCL_BUFFSIZE: "512"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM-ASCEND_ENABLE_FUSED_MC2: "2"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--data-parallel-size"
|
||||
- "4"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--max-model-len"
|
||||
- "153600"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.8"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k-lite
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 32768
|
||||
batch_size: 32
|
||||
baseline: 95
|
||||
threshold: 10
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 80
|
||||
max_out_len: 1500
|
||||
batch_size: 20
|
||||
request_rate: 0
|
||||
baseline: 730.0832
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,85 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
HCCL_BUFFSIZE: "1200"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_NUM_THREADS: "1"
|
||||
LD_PRELOAD: "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--data-parallel-size"
|
||||
- "2"
|
||||
- "--enable-expert-parallel"
|
||||
- "--async-scheduling"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--safetensors-load-strategy"
|
||||
- 'prefetch'
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--trust-remote-code"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--enable-auto-tool-choice"
|
||||
- "--tool-call-parser"
|
||||
- "minimax_m2"
|
||||
- "--speculative_config"
|
||||
- '{"method":"eagle3","model":"Eco-Tech/MiniMax-M2.7-eagle-model-short","num_speculative_tokens":3}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding": true, "enable_npugraph_ex": true, "enable_static_kernel": true,"enable_fused_mc2":true,"weight_nz_mode":true,"enable_flashcomm1":true}'
|
||||
|
||||
_benchmarks_3500: &benchmarks_3500
|
||||
perf_50:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_in3500_bs4000_minimax
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 760
|
||||
max_out_len: 1500
|
||||
batch_size: 190
|
||||
request_rate: 0
|
||||
baseline: 4573.02
|
||||
threshold: 0.97
|
||||
perf_20:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_in3500_bs4000_minimax
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 192
|
||||
max_out_len: 1500
|
||||
batch_size: 48
|
||||
request_rate: 0
|
||||
baseline: 2229.147
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "MiniMax-M2.7-3500"
|
||||
model: "vllm-ascend/MiniMax-M2.7-w8a8-QuaRot"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "70000"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.8"
|
||||
- "--no-enable-prefix-caching"
|
||||
benchmarks:
|
||||
<<: *benchmarks_3500
|
||||
@@ -0,0 +1,92 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen2.5-VL-7B-Instruct-epd"
|
||||
model: "Qwen/Qwen2.5-VL-7B-Instruct"
|
||||
service_mode: "epd"
|
||||
envs:
|
||||
ENCODE_PORT: "DEFAULT_PORT"
|
||||
PD_PORT: "DEFAULT_PORT"
|
||||
PROXY_PORT: "DEFAULT_PORT"
|
||||
epd_server_cmds:
|
||||
- - "--port"
|
||||
- "$ENCODE_PORT"
|
||||
- "--model"
|
||||
- "Qwen/Qwen2.5-VL-7B-Instruct"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.01"
|
||||
- "--tensor-parallel-size"
|
||||
- "1"
|
||||
- "--enforce-eager"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--max-model-len"
|
||||
- "10000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "10000"
|
||||
- "--max-num-seqs"
|
||||
- "1"
|
||||
- "--ec-transfer-config"
|
||||
- '{"ec_connector_extra_config":{"shared_storage_path":"/dev/shm/epd/storage"},"ec_connector":"ECExampleConnector","ec_role": "ec_producer"}'
|
||||
- - "--port"
|
||||
- "$PD_PORT"
|
||||
- "--model"
|
||||
- "Qwen/Qwen2.5-VL-7B-Instruct"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.95"
|
||||
- "--tensor-parallel-size"
|
||||
- "1"
|
||||
- "--enforce-eager"
|
||||
- "--max-model-len"
|
||||
- "10000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "10000"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--ec-transfer-config"
|
||||
- '{"ec_connector_extra_config":{"shared_storage_path":"/dev/shm/epd/storage"},"ec_connector":"ECExampleConnector","ec_role": "ec_consumer"}'
|
||||
epd_proxy_args:
|
||||
- "--host"
|
||||
- "127.0.0.1"
|
||||
- "--port"
|
||||
- "$PROXY_PORT"
|
||||
- "--encode-servers-urls"
|
||||
- "http://localhost:$ENCODE_PORT"
|
||||
- "--decode-servers-urls"
|
||||
- "http://localhost:$PD_PORT"
|
||||
- "--prefill-servers-urls"
|
||||
- "disable"
|
||||
test_content:
|
||||
benchmarks:
|
||||
warm_up:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/textvqa-perf-1080p
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: textvqa/textvqa_gen_base64
|
||||
num_prompts: 50
|
||||
max_out_len: 20
|
||||
batch_size: 32
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/textvqa-lite
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: textvqa/textvqa_gen_base64
|
||||
max_out_len: 2048
|
||||
batch_size: 128
|
||||
baseline: 82.05
|
||||
threshold: 5
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/textvqa-perf-1080p
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: textvqa/textvqa_gen_base64
|
||||
num_prompts: 512
|
||||
max_out_len: 256
|
||||
batch_size: 128
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,59 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "qwen3-14b"
|
||||
model: "vllm-ascend/Qwen3-14B-w8a8sc-310-vllm-tp1"
|
||||
envs:
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--tensor-parallel-size"
|
||||
- "1"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.8"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--dtype"
|
||||
- "float16"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--max-model-len"
|
||||
- "20480"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--load_format"
|
||||
- "sharded_state"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[1,4,8,16,24,32]}'
|
||||
- "--additional-config"
|
||||
- '{"ascend_compilation_config":{"enable_npugraph_ex":false, "fuse_norm_quant": false}}'
|
||||
- "--default-chat-template-kwargs"
|
||||
- '{"enable_thinking": false}'
|
||||
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in512-bs100-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 512
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 10.7
|
||||
threshold: 0.95
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/SuperGLUE
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: SuperGLUE/SuperGLUE_BoolQ_gen_0_shot_cot_str
|
||||
max_out_len: 10240
|
||||
batch_size: 32
|
||||
baseline: 89.3
|
||||
threshold: 1
|
||||
temperature: 0
|
||||
top_p: 0.95
|
||||
ignore_eos: false
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_USE_V1: "1"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--async-scheduling"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--data-parallel-size"
|
||||
- "4"
|
||||
- "--data-parallel-size-local"
|
||||
- "4"
|
||||
- "--data-parallel-start-rank"
|
||||
- "0"
|
||||
- "--data-parallel-rpc-port"
|
||||
- "2345"
|
||||
- "--max-num-seqs"
|
||||
- "120"
|
||||
- "--max-model-len"
|
||||
- "40960"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--enable-expert-parallel "
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--trust-remote-code"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--data-parallel-address"
|
||||
- "0.0.0.0"
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k_lite
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 38912
|
||||
num_prompts: 32
|
||||
batch_size: 32
|
||||
baseline: 100
|
||||
threshold: 10
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3_235B_Accuracy_PIECEWISE_gsm8k_lite"
|
||||
model: "Eco-Tech/Qwen3-235B-A22B-w8a8-QuaRot"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "PIECEWISE"}'
|
||||
- "--hf-overrides"
|
||||
- '{"rope_parameters": {"rope_type":"yarn","rope_theta": 1000000.0,"factor":4.3,"original_max_position_embeddings":32768}}'
|
||||
- "--additional-config"
|
||||
- '{"ascend_scheduler_config":{"enabled":false},"pa_shape_list": [4,8,16,32,48,64,96,128,160,192]}'
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
|
||||
- name: "Qwen3_235B_Accuracy_FULL_Graph_gsm8k_lite"
|
||||
model: "Eco-Tech/Qwen3-235B-A22B-w8a8-QuaRot"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
- "--hf-overrides"
|
||||
- '{"rope_parameters": {"rope_type":"yarn","rope_theta": 1000000.0,"factor":4.3,"original_max_position_embeddings":32768}}'
|
||||
- "--additional-config"
|
||||
- '{"ascend_scheduler_config":{"enabled":false},"pa_shape_list": [4,8,16,32,48,64,96,128,160,192]}'
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
- name: "Qwen3_235B_Accuracy_EPLB_gsm8k_lite"
|
||||
model: "Eco-Tech/Qwen3-235B-A22B-w8a8-QuaRot"
|
||||
envs:
|
||||
<<: *envs
|
||||
DYNAMIC_EPLB: "true"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "qwen3-32b"
|
||||
model: "vllm-ascend/Qwen3-32B-w8a8sc-310-vllm-tp4"
|
||||
envs:
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.8"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--dtype"
|
||||
- "float16"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--max-model-len"
|
||||
- "20480"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--load_format"
|
||||
- "sharded_state"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[8,32]}'
|
||||
- "--additional-config"
|
||||
- '{"ascend_compilation_config":{"enable_npugraph_ex":false, "fuse_norm_quant": false}}'
|
||||
- "--default-chat-template-kwargs"
|
||||
- '{"enable_thinking": false}'
|
||||
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in512-bs100-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 32
|
||||
max_out_len: 512
|
||||
batch_size: 8
|
||||
request_rate: 0
|
||||
baseline: 105.26
|
||||
threshold: 0.95
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 10240
|
||||
batch_size: 32
|
||||
baseline: 95.07
|
||||
threshold: 1
|
||||
temperature: 0
|
||||
top_p: 0.95
|
||||
55
tests/e2e/weekly/single_node/configs/Qwen3-32B.yaml
Normal file
55
tests/e2e/weekly/single_node/configs/Qwen3-32B.yaml
Normal file
@@ -0,0 +1,55 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
#workflow暂缺A2yaml
|
||||
# - name: qwen3-32b
|
||||
# os: linux-aarch64-a2b3-4
|
||||
# config_file_path: Qwen3-32B.yaml
|
||||
test_cases:
|
||||
- name: "Qwen3-32B-TP4"
|
||||
model: "Qwen/Qwen3-32B"
|
||||
envs:
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
OMP_PROC_BIND: "false"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PAGED_ATTENTION_MASK_LEN: "5500"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "36864"
|
||||
- "--max-num-batched-tokens"
|
||||
- "36864"
|
||||
- "--block-size"
|
||||
- "128"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--additional-config"
|
||||
- '{"enable_weight_nz_layout":true}'
|
||||
benchmarks:
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 32768
|
||||
num_prompts: 32
|
||||
batch_size: 32
|
||||
baseline: 95
|
||||
threshold: 5
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs400
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 80
|
||||
max_out_len: 1500
|
||||
batch_size: 20
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,58 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "qwen3-8b"
|
||||
model: "vllm-ascend/Qwen3-8B-w8a8sc-310-vllm-tp1"
|
||||
envs:
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--tensor-parallel-size"
|
||||
- "1"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.8"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--dtype"
|
||||
- "float16"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--max-model-len"
|
||||
- "20480"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--load_format"
|
||||
- "sharded_state"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[1,4,8,16,24,32]}'
|
||||
- "--additional-config"
|
||||
- '{"ascend_compilation_config":{"enable_npugraph_ex":false, "fuse_norm_quant": false}}'
|
||||
- "--default-chat-template-kwargs"
|
||||
- '{"enable_thinking": false}'
|
||||
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in512-bs100-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 512
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 18
|
||||
threshold: 0.95
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 10240
|
||||
batch_size: 32
|
||||
baseline: 93.3
|
||||
threshold: 1
|
||||
temperature: 0
|
||||
top_p: 0.95
|
||||
ignore_eos: false
|
||||
@@ -0,0 +1,74 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
#workflow暂缺A2yaml
|
||||
# - name: Qwen3.5-122B-A10B-W8A8-single-A2
|
||||
# os: linux-aarch64-a2b3-8
|
||||
# config_file_path: Qwen3.5-122B-A10B-W8A8-A2.yaml
|
||||
_envs: &envs
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
OMP_NUM_THREADS: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "0"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--tensor-parallel-size"
|
||||
- "8"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "262144"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--enable-expert-parallel"
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch", "torch_profiler_dir": "./profiling", "torch_profiler_with_stack": false}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,286,292,296,300,304,308,312,316,320,324,328,332,336,340], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 256
|
||||
max_out_len: 1500
|
||||
batch_size: 64
|
||||
baseline: 1
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-single-A2"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
@@ -0,0 +1,281 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding": true, "enable_shared_expert_dp": true}'
|
||||
- "--api-server-count"
|
||||
- "1"
|
||||
|
||||
_benchmark_acc: &_benchmark_acc
|
||||
acc_GPQA:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gpqa
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gpqa/gpqa_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 65536
|
||||
batch_size: 32
|
||||
baseline: 85
|
||||
threshold: 5
|
||||
thinking: true
|
||||
temperature: 1.0
|
||||
top_p: 0.95
|
||||
top_k: 20
|
||||
min_p: 0.0
|
||||
presence_penalty: 1.5
|
||||
repetition_penalty: 1.0
|
||||
ignore_eos: false
|
||||
acc_aime2025:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/aime2025
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
|
||||
max_out_len: 65536
|
||||
batch_size: 32
|
||||
baseline: 90
|
||||
threshold: 10
|
||||
thinking: true
|
||||
temperature: 1.0
|
||||
top_p: 0.95
|
||||
top_k: 20
|
||||
min_p: 0.0
|
||||
presence_penalty: 1.5
|
||||
repetition_penalty: 1.0
|
||||
ignore_eos: false
|
||||
|
||||
_benchmark_TPOT50_3_5k_1_5K: &_benchmark_TPOT50_3_5k_1_5K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 320
|
||||
max_out_len: 1500
|
||||
batch_size: 80
|
||||
request_rate: 0
|
||||
baseline: 1052.9
|
||||
threshold: 0.95
|
||||
|
||||
_benchmark_TPOT50_16K_1K: &_benchmark_TPOT50_16K_1K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 96
|
||||
max_out_len: 1024
|
||||
batch_size: 24
|
||||
request_rate: 0
|
||||
baseline: 400.183
|
||||
threshold: 0.95
|
||||
|
||||
_benchmark_TPOT20_16K_1K: &_benchmark_TPOT20_16K_1K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 32
|
||||
max_out_len: 1024
|
||||
batch_size: 8
|
||||
request_rate: 0
|
||||
baseline: 325.5
|
||||
threshold: 0.95
|
||||
|
||||
_benchmark_TPOT50_32K_0_5K: &_benchmark_TPOT50_32K_0_5K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 36
|
||||
max_out_len: 512
|
||||
batch_size: 9
|
||||
request_rate: 0
|
||||
baseline: 123.8
|
||||
threshold: 0.95
|
||||
|
||||
_benchmark_TPOT20_32K_0_5K: &_benchmark_TPOT20_32K_0_5K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 512
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 113.8
|
||||
threshold: 0.95
|
||||
|
||||
_benchmark_TPOT50_64K_1K: &_benchmark_TPOT50_64K_1K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 32
|
||||
max_out_len: 1024
|
||||
batch_size: 8
|
||||
request_rate: 0
|
||||
baseline: 114.1
|
||||
threshold: 0.95
|
||||
|
||||
_benchmark_TPOT20_64K_1K: &_benchmark_TPOT20_64K_1K
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 1024
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 99.28
|
||||
threshold: 0.95
|
||||
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-A3"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-model-len"
|
||||
- "8000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8000"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_3_5k_1_5K
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-TPOT50-16k-1k"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-model-len"
|
||||
- "18432"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_16K_1K
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-TPOT20-16k-1k"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "18432"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_16K_1K
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-TPOT50-32k-0.5k"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_32K_0_5K
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-TPOT20-32k-0.5k"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_32K_0_5K
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-TPOT50-64k-1k"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-expert-parallel"
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT50_64K_1K
|
||||
<<: *_benchmark_acc
|
||||
- name: "Qwen3.5-122B-A10B-W8A8-TPOT20-64k-1k"
|
||||
model: "Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "0"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5}'
|
||||
benchmarks:
|
||||
perf: *_benchmark_TPOT20_64K_1K
|
||||
549
tests/e2e/weekly/single_node/configs/Qwen3.5-27B-w8a8-A3.yaml
Normal file
549
tests/e2e/weekly/single_node/configs/Qwen3.5-27B-w8a8-A3.yaml
Normal file
@@ -0,0 +1,549 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
_envs: &envs
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_USE_MODELSCOPE: "true"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
test_cases:
|
||||
- name: "Qwen3.5-27B-w8a8-in3.5k-140-35"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--max-model-len"
|
||||
- "128000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 140
|
||||
max_out_len: 1500
|
||||
batch_size: 35
|
||||
request_rate: 0
|
||||
baseline: 610.22
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in16k-56-14"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "18432"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--no-enable-prefix-caching"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 56
|
||||
max_out_len: 1024
|
||||
batch_size: 14
|
||||
request_rate: 0
|
||||
baseline: 221.22
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in32k-56-14"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
- "--enable-prefix-caching"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in32768_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 56
|
||||
max_out_len: 512
|
||||
batch_size: 14
|
||||
request_rate: 0
|
||||
baseline: 197.00
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in64k-16-4"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--no-enable-prefix-caching"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 1024
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 56.17
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in64k-48-12"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--enable-prefix-caching"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 48
|
||||
max_out_len: 1024
|
||||
batch_size: 12
|
||||
request_rate: 0
|
||||
baseline: 195.87
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in128k-28-7"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--enable-prefix-caching"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 28
|
||||
max_out_len: 1024
|
||||
batch_size: 7
|
||||
request_rate: 0
|
||||
baseline: 101.12
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in16k-16-4"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "18432"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--no-enable-prefix-caching"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 1024
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 163.77
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in32k-8-2"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--no-enable-prefix-caching"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 512
|
||||
batch_size: 2
|
||||
request_rate: 0
|
||||
baseline: 53.82
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in32k-16-4"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
- "--enable-prefix-caching"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in32768_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 512
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 149.23
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in64k-8-2"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--no-enable-prefix-caching"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 1024
|
||||
batch_size: 2
|
||||
request_rate: 0
|
||||
baseline: 48.52
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in64k-8-2-90"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--enable-prefix-caching"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 1024
|
||||
batch_size: 2
|
||||
request_rate: 0
|
||||
baseline: 87.3
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.5-27B-w8a8-in128k-4-1"
|
||||
model: "Eco-Tech/Qwen3.5-27B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd:
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--enable-prefix-caching"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 36.44
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,61 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
VLLM_USE_MODELSCOPE: "true"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
|
||||
VLLM_RPC_TIMEOUT: "6000"
|
||||
server_cmd:
|
||||
- "--tensor-parallel-size"
|
||||
- "16"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--enable-expert-parallel"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--safetensors-load-strategy"
|
||||
- "lazy"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in131072-bs100-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 1024
|
||||
batch_size: 2
|
||||
request_rate: 0
|
||||
baseline: 56.357
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,402 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
|
||||
_envs: &envs
|
||||
VLLM_USE_MODELSCOPE: "true"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "0"
|
||||
VLLM_ENGINE_READY_TIMEOUT_S: "3000"
|
||||
VLLM_RPC_TIMEOUT: "600"
|
||||
_server_cmd: &server_cmd
|
||||
- "--tensor-parallel-size"
|
||||
- "16"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--enable-expert-parallel"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--trust-remote-code"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--safetensors-load-strategy"
|
||||
- "lazy"
|
||||
_benchmarks: &benchmarks
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 1024
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 165.23
|
||||
threshold: 0.97
|
||||
|
||||
|
||||
_benchmarks_bs144: &benchmarks_bs144
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 144
|
||||
max_out_len: 1024
|
||||
batch_size: 36
|
||||
request_rate: 0
|
||||
baseline: 630.88
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs36: &benchmarks_bs36
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 36
|
||||
max_out_len: 1024
|
||||
batch_size: 9
|
||||
request_rate: 0
|
||||
baseline: 336.80
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs48: &benchmarks_bs48
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 48
|
||||
max_out_len: 512
|
||||
batch_size: 12
|
||||
request_rate: 0
|
||||
baseline: 180.60
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs16: &benchmarks_bs16
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in32768_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 512
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 118.17
|
||||
threshold: 0.97
|
||||
|
||||
|
||||
_benchmarks_bs160: &benchmarks_bs160
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in32768_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 160
|
||||
max_out_len: 512
|
||||
batch_size: 40
|
||||
request_rate: 0
|
||||
baseline: 689.71
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs32: &benchmarks_bs32
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in32768_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 32
|
||||
max_out_len: 512
|
||||
batch_size: 8
|
||||
request_rate: 0
|
||||
baseline: 303.07
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs8: &benchmarks_bs8
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 8
|
||||
max_out_len: 1024
|
||||
batch_size: 2
|
||||
request_rate: 0
|
||||
baseline: 93.10
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs32_in65536: &benchmarks_bs32_in65536
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 32
|
||||
max_out_len: 1024
|
||||
batch_size: 8
|
||||
request_rate: 0
|
||||
baseline: 160.55
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs136: &benchmarks_bs136
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 136
|
||||
max_out_len: 1024
|
||||
batch_size: 34
|
||||
request_rate: 0
|
||||
baseline: 618.66
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs16_in65536: &benchmarks_bs16_in65536
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 16
|
||||
max_out_len: 1024
|
||||
batch_size: 4
|
||||
request_rate: 0
|
||||
baseline: 198.87
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_bs80: &benchmarks_bs80
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in131072-bs100-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 80
|
||||
max_out_len: 1024
|
||||
batch_size: 20
|
||||
request_rate: 0
|
||||
baseline: 335.95
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
test_cases:
|
||||
|
||||
#131072 1024 0.9 80
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-High—Throughput-bs80"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs80
|
||||
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-Minimal-Delay-bs2"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-High—Throughput-bs144"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "18432"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs144
|
||||
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-Minimal-Delay-bs36"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "18432"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs36
|
||||
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-High—Throughput-bs48"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs48
|
||||
|
||||
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-Minimal-Delay-bs16"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs16
|
||||
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-Minimal-Delay-bs160"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs160
|
||||
|
||||
#65536 1024 0 32
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-Minimal-Delay-bs32"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "34304"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs32
|
||||
|
||||
#65536 1024 0 8
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-High—Throughput-bs8"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs8
|
||||
|
||||
#65536 1024 0 32
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-High—Throughput-bs32_in65536"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs32_in65536
|
||||
|
||||
#67584 65536 1024 0.9 136
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-High—Throughput-bs136"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs136
|
||||
|
||||
#67584 65536 1024 0.9 16
|
||||
- name: "Qwen3-397B-A17B-w8a8-A3-Minimal-Delay-bs16_in65536"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--enable-prefix-caching"
|
||||
- "--speculative_config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 5, "enforce_eager": true}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,6,12,18,24,30,36,42,48,54,72,78,84,90,96,102,108,144,192], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
benchmarks:
|
||||
<<: *benchmarks_bs16_in65536
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
test_cases:
|
||||
- name: "Qwen3.5-397B-A17B-w8a8-3.5k-1.5k-0-50-single"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
VLLM_USE_MODELSCOPE: "true"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "0"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- --data-parallel-size
|
||||
- "1"
|
||||
- --tensor-parallel-size
|
||||
- "16"
|
||||
- --enable-expert-parallel
|
||||
- --max-model-len
|
||||
- "6144"
|
||||
- --max-num-batched-tokens
|
||||
- "16384"
|
||||
- --max-num-seqs
|
||||
- "128"
|
||||
- --gpu-memory-utilization
|
||||
- "0.9"
|
||||
- --compilation-config
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- --speculative_config
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- --trust-remote-code
|
||||
- --async-scheduling
|
||||
- --allowed-local-media-path
|
||||
- "/"
|
||||
- --quantization
|
||||
- "ascend"
|
||||
- --mm-processor-cache-gb
|
||||
- "0"
|
||||
- --additional-config
|
||||
- '{"enable_cpu_binding":true}'
|
||||
|
||||
benchmarks:
|
||||
perf_50:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 640
|
||||
max_out_len: 1500
|
||||
batch_size: 160
|
||||
request_rate: 5
|
||||
baseline: 2663.3728
|
||||
threshold: 0.97
|
||||
perf_20:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 128
|
||||
max_out_len: 1500
|
||||
batch_size: 32
|
||||
request_rate: 0
|
||||
baseline: 1421.89
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,91 @@
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
test_cases:
|
||||
- name: "Qwen3.5-397B-A17B-w8a8-64k-1k-90-50-single"
|
||||
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
||||
envs:
|
||||
VLLM_USE_MODELSCOPE: "true"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_BUFFSIZE: "2048"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "0"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- --data-parallel-size
|
||||
- "1"
|
||||
- --tensor-parallel-size
|
||||
- "16"
|
||||
- --quantization
|
||||
- "ascend"
|
||||
- --allowed-local-media-path
|
||||
- "/"
|
||||
- --seed
|
||||
- "1024"
|
||||
- --enable-prefix-caching
|
||||
- --enable-expert-parallel
|
||||
- --max-model-len
|
||||
- "140000"
|
||||
- --max-num-batched-tokens
|
||||
- "16384"
|
||||
- --max-num-seqs
|
||||
- "32"
|
||||
- --gpu-memory-utilization
|
||||
- "0.9"
|
||||
- --compilation-config
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,16,24,32,48,64,96,108,128,192],"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- --speculative_config
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 3}'
|
||||
- --additional-config
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- --trust-remote-code
|
||||
- --async-scheduling
|
||||
- --mm-processor-cache-gb
|
||||
- "0"
|
||||
- --mm-encoder-tp-mode
|
||||
- "data"
|
||||
|
||||
benchmarks:
|
||||
acc_aime2025:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/aime2025
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
|
||||
max_out_len: 65536
|
||||
batch_size: 32
|
||||
baseline: 90
|
||||
threshold: 10
|
||||
temperature: 0.6
|
||||
top_p: 0.95
|
||||
top_k: 20
|
||||
min_p: 0.0
|
||||
presence_penalty: 0.0
|
||||
repetition_penalty: 1.0
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 96
|
||||
max_out_len: 1024
|
||||
batch_size: 24
|
||||
request_rate: 5
|
||||
baseline: 374.5353
|
||||
threshold: 0.97
|
||||
perf_128k:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 80
|
||||
max_out_len: 1024
|
||||
batch_size: 20
|
||||
request_rate: 5
|
||||
baseline: 345.3004
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,183 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3.6-35B-A3B-in16k-out1k-0-128-32"
|
||||
model: "Eco-Tech/Qwen3.6-35B-A3B-w8a8"
|
||||
envs:
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--enable-expert-parallel"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-num-seqs"
|
||||
- "36"
|
||||
- "--max-model-len"
|
||||
- "20000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,20,24,28,40,48,64,76,80,88,96,100,104,108,112,116,120,136,140,144],"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch","torch_profiler_dir": "./profiling","torch_profiler_with_stack": false}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in16384_bs200_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 128
|
||||
max_out_len: 1024
|
||||
batch_size: 32
|
||||
request_rate: 0.6
|
||||
baseline: 548.2128
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.6-35B-A3B-in984k-out1k-0-4-2"
|
||||
model: "Eco-Tech/Qwen3.6-35B-A3B-w8a8"
|
||||
envs:
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
HCCL_BUFFSIZE: "200"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
VLLM_ASCEND_GDN_FAST_PATH: "1"
|
||||
VLLM_ASCEND_GDN_MAX_PADDING_RATIO: "2.0"
|
||||
VLLM_ASCEND_GDN_MAX_H_OVERALLOC_RATIO: "2.0"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--hf-overrides"
|
||||
- '{"text_config":{"rope_parameters":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":262144}}}'
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--max-model-len"
|
||||
- "1010000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--max-num-seqs"
|
||||
- "128"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512],"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
||||
- "--trust-remote-code"
|
||||
- "--async-scheduling"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_in1007616_bs16_prefix0_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 4
|
||||
max_out_len: 1024
|
||||
batch_size: 2
|
||||
request_rate: 0
|
||||
baseline: 2.2225
|
||||
threshold: 0.97
|
||||
- name: "Qwen3.6-35B-A3B-in128k-out1k-90-60-15"
|
||||
model: "Eco-Tech/Qwen3.6-35B-A3B-w8a8"
|
||||
envs:
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
VLLM_DISABLE_COMPILE_CACHE: "1"
|
||||
VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--enable-expert-parallel"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-num-seqs"
|
||||
- "36"
|
||||
- "--max-model-len"
|
||||
- "140000"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--enable-prefix-caching"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,4,8,12,16,20,24,28,40,48,64,76,80,88,96,100,104,108,112,116,120,136,140,144],"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch","torch_profiler_dir": "./profiling","torch_profiler_with_stack": false}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
benchmarks:
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 60
|
||||
max_out_len: 1024
|
||||
batch_size: 15
|
||||
request_rate: 0
|
||||
baseline: 223.2484
|
||||
threshold: 0.97
|
||||
@@ -0,0 +1,67 @@
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
test_cases:
|
||||
- name: "Qwen3.6-35B-A3B"
|
||||
model: "Eco-Tech/Qwen3.6-35B-A3B-w8a8"
|
||||
envs:
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
VLLM_ASCEND_BALANCE_SCHEDULING: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
server_cmd:
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--trust-remote-code"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--enable-expert-parallel"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--max-num-seqs"
|
||||
- "50"
|
||||
- "--max-model-len"
|
||||
- "65536"
|
||||
- "--max-num-batched-tokens"
|
||||
- "8192"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--seed"
|
||||
- "1024"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_capture_sizes":[1,8,16,24,28,40,52,64,76,88,96,100,112,120,132,144,160,176,184,192,196,200,204,208,212,216,220],"cudagraph_mode":"FULL_DECODE_ONLY"}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 3}'
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding":true}'
|
||||
- "--profiler-config"
|
||||
- '{"profiler": "torch","torch_profiler_dir": "./profiling","torch_profiler_with_stack": false}'
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
benchmarks:
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/aime2025
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: aime2025/aime2025_gen_0_shot_chat_prompt
|
||||
max_out_len: 32768
|
||||
batch_size: 32
|
||||
baseline: 90
|
||||
threshold: 10
|
||||
temperature: 1.0
|
||||
top_p: 0.95
|
||||
top_k: 20
|
||||
min_p: 0.0
|
||||
presence_penalty: 1.5
|
||||
repetition_penalty: 1.0
|
||||
78
tests/e2e/weekly/single_node/configs/Qwen3_235b_w8a8_A5.yaml
Normal file
78
tests/e2e/weekly/single_node/configs/Qwen3_235b_w8a8_A5.yaml
Normal file
@@ -0,0 +1,78 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_USE_V1: "1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
OMP_PROC_BIND: "false"
|
||||
OMP_NUM_THREADS: "1"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
||||
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--async-scheduling"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--data-parallel-size"
|
||||
- "4"
|
||||
- "--data-parallel-size-local"
|
||||
- "4"
|
||||
- "--data-parallel-start-rank"
|
||||
- "0"
|
||||
- "--data-parallel-rpc-port"
|
||||
- "2345"
|
||||
- "--max-num-seqs"
|
||||
- "40"
|
||||
- "--max-model-len"
|
||||
- "40960"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--enable-expert-parallel "
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--trust-remote-code"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--data-parallel-address"
|
||||
- "0.0.0.0"
|
||||
|
||||
_benchmarks: &benchmarks
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gpqa
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gpqa/gpqa_gen_0_shot_chat_prompt
|
||||
max_out_len: 65536
|
||||
batch_size: 32
|
||||
baseline: 85
|
||||
threshold: 5
|
||||
temperature: 0.6
|
||||
top_p: 0.95
|
||||
top_k: 20
|
||||
ignore_eos: false
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "vllm_Qwen3_235B_Accuracy_FULL_Graph_GPQA"
|
||||
model: "Eco-Tech/Qwen3-235B-A22B-w8a8-QuaRot"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
benchmarks:
|
||||
<<: *benchmarks
|
||||
|
||||
144
tests/e2e/weekly/single_node/configs/Qwen3_32b_W8A8_wl_A3.yaml
Normal file
144
tests/e2e/weekly/single_node/configs/Qwen3_32b_W8A8_wl_A3.yaml
Normal file
@@ -0,0 +1,144 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
|
||||
VLLM_USE_V1: "1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
LD_PRELOAD: "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--trust-remote-code"
|
||||
- "--reasoning-parser"
|
||||
- "qwen3"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--block-size"
|
||||
- "128"
|
||||
- "--no-enable-prefix-caching"
|
||||
|
||||
_benchmarks_gsm8k_lite: &benchmarks_gsm8k_lite
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k_lite
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 10240
|
||||
num_prompts: 32
|
||||
batch_size: 32
|
||||
baseline: 100
|
||||
threshold: 10
|
||||
|
||||
_benchmarks_aime: &benchmarks_aime
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/aime2024
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: aime2024/aime2024_gen_0_shot_chat_prompt
|
||||
max_out_len: 32768
|
||||
num_prompts: 32
|
||||
batch_size: 32
|
||||
baseline: 83.3
|
||||
threshold: 10
|
||||
|
||||
_benchmarks_perf1: &benchmarks_perf1
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 400
|
||||
max_out_len: 1500
|
||||
batch_size: 46
|
||||
request_rate: 0
|
||||
baseline: 942.23
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3_32B_w8a8_Accuracy_A3"
|
||||
model: "vllm-ascend/Qwen3-32B-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
OMP_PROC_BIND: "false"
|
||||
ASCEND_RT_VISIBLE_DEVICES: "0,1,2,3"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "35840"
|
||||
- "--max-num-batched-tokens"
|
||||
- "40960"
|
||||
- "--distributed_executor_backend"
|
||||
- "mp"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_gsm8k_lite
|
||||
- name: "Qwen3_32b_accuracy_aime2024_A3"
|
||||
model: "vllm-ascend/Qwen3-32B-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE: "1"
|
||||
VLLM_ASCEND_ENABLE_PREFETCH_MLP: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "36784"
|
||||
- "--max-num-batched-tokens"
|
||||
- "36784"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--additional-config"
|
||||
- '{"enable_weight_nz_layout":true}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_aime
|
||||
- name: "Qwen3_32B_w8a8_Performance_A3"
|
||||
model: "vllm-ascend/Qwen3-32B-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
OMP_PROC_BIND: "false"
|
||||
ASCEND_RT_VISIBLE_DEVICES: "0,1,2,3"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
server_cmd: *server_perf_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "36784"
|
||||
- "--max-num-batched-tokens"
|
||||
- "36784"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--additional-config"
|
||||
- '{"enable_weight_nz_layout":true}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_perf1
|
||||
- name: "single_Qwen3_32B_w8a8_server_A3"
|
||||
model: "vllm-ascend/Qwen3-32B-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
OMP_PROC_BIND: "false"
|
||||
ASCEND_RT_VISIBLE_DEVICES: "0,1,2,3"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
server_cmd: *server_perf_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "36784"
|
||||
- "--max-num-batched-tokens"
|
||||
- "36784"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--additional-config"
|
||||
- '{"enable_weight_nz_layout":true}'
|
||||
- "--enforce-eager"
|
||||
benchmarks:
|
||||
<<: *benchmarks_perf1
|
||||
@@ -0,0 +1,79 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_USE_V1: "1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--distributed_executor_backend"
|
||||
- "mp"
|
||||
- "--max-num-seqs"
|
||||
- "256"
|
||||
- "--max-model-len"
|
||||
- "5500"
|
||||
- "--trust-remote-code"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--reasoning-parser"
|
||||
- "qwen3"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
|
||||
_benchmarks_perf1: &benchmarks_perf1
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 304
|
||||
max_out_len: 1500
|
||||
batch_size: 76
|
||||
request_rate: 0
|
||||
baseline: 1326.599
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3_32b_int8_Feature_stack_A3"
|
||||
model: "vllm-ascend/Qwen3-32B-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-num-batched-tokens"
|
||||
- "5500"
|
||||
- "--enforce-eager"
|
||||
- "--additional-config"
|
||||
- '{"ascend_scheduler_config":{"enabled":false},"enable_weight_nz_layout":true}'
|
||||
- name: "Qwen3_32B_w8a8_Performance_3500_1500_A3"
|
||||
model: "vllm-ascend/Qwen3-32B-W8A8"
|
||||
envs:
|
||||
<<: *envs
|
||||
LD_PRELOAD: "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD"
|
||||
VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE: "1"
|
||||
VLLM_ASCEND_ENABLE_PREFETCH_MLP: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-num-batched-tokens"
|
||||
- "40960"
|
||||
- "--block-size"
|
||||
- "128"
|
||||
- "--async-scheduling"
|
||||
- "--additional-config"
|
||||
- '{"pa_shape_list":[32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256]}'
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[1,5,8,72,76,80,100,120,140,144,160,192,216,240,252,288,320,336,360,384,400,408,416,420,432,480,540,576,600]}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_perf1
|
||||
|
||||
113
tests/e2e/weekly/single_node/configs/Qwq_32b_A3.yaml
Normal file
113
tests/e2e/weekly/single_node/configs/Qwq_32b_A3.yaml
Normal file
@@ -0,0 +1,113 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
VLLM_USE_V1: "1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
VLLM_ASCEND_ENABLE_FLASHCOMM: "1"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--tensor-parallel-size"
|
||||
- "4"
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--distributed_executor_backend"
|
||||
- "mp"
|
||||
- "--trust-remote-code"
|
||||
- "--reasoning-parser"
|
||||
- "deepseek_r1"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
|
||||
_benchmarks_gsm8k_lite: &benchmarks_gsm8k_lite
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gsm8k_lite
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 10240
|
||||
num_prompts: 32
|
||||
batch_size: 32
|
||||
baseline: 100
|
||||
threshold: 10
|
||||
|
||||
_benchmarks_perf1: &benchmarks_perf1
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 340
|
||||
max_out_len: 1500
|
||||
batch_size: 60
|
||||
request_rate: 0
|
||||
baseline: 724.59
|
||||
threshold: 0.97
|
||||
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwq_32b_Feature_A3"
|
||||
model: "Qwen/QwQ-32B"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-num-seqs"
|
||||
- "256"
|
||||
- "--max-model-len"
|
||||
- "32768"
|
||||
- "--max-num-batched-tokens"
|
||||
- "32768"
|
||||
- "--enforce-eager"
|
||||
- "--additional-config"
|
||||
- '{"ascend_scheduler_config":{"enabled":false}}'
|
||||
- name: "Qwq_32B_gsm8k_Accuracy_A3"
|
||||
model: "Qwen/QwQ-32B"
|
||||
envs:
|
||||
<<: *envs
|
||||
LD_PRELOAD: "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD"
|
||||
VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE: "1"
|
||||
VLLM_ASCEND_ENABLE_PREFETCH_MLP: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "35840"
|
||||
- "--max-num-batched-tokens"
|
||||
- "40960"
|
||||
- "--block-size"
|
||||
- "128"
|
||||
- "--async-scheduling"
|
||||
- "--no-enable-prefix-caching"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_gsm8k_lite
|
||||
- name: "Qwq_Performance_A3_3500_1500_A3"
|
||||
model: "Qwen/QwQ-32B"
|
||||
envs:
|
||||
<<: *envs
|
||||
OMP_PROC_BIND: "false"
|
||||
ASCEND_RT_VISIBLE_DEVICES: "0,1,2,3"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "5500"
|
||||
- "--max-num-batched-tokens"
|
||||
- "40960"
|
||||
- "--block-size"
|
||||
- "128"
|
||||
- "--async-scheduling"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [ 1,5,8,72,76,80,100,120,140,144,160,192,216,240,252,288,320,336,360,384,400,408,416,420,432,480,540,576,600 ]}'
|
||||
- "--additional-config"
|
||||
- '{"pa_shape_list": [32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256 ]}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_perf1
|
||||
|
||||
219
tests/e2e/weekly/single_node/configs/qwen_3_6_27b_w8a8_A3.yaml
Normal file
219
tests/e2e/weekly/single_node/configs/qwen_3_6_27b_w8a8_A3.yaml
Normal file
@@ -0,0 +1,219 @@
|
||||
# ==========================================
|
||||
# Shared Configurations
|
||||
# ==========================================
|
||||
|
||||
_envs: &envs
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
OMP_NUM_THREADS: "1"
|
||||
TASK_QUEUE_ENABLE: "1"
|
||||
HCCL_IF_IP: "127.0.0.1"
|
||||
HCCL_OP_EXPANSION_MODE: "AIV"
|
||||
HCCL_BUFFSIZE: "1024"
|
||||
SERVER_PORT: "DEFAULT_PORT"
|
||||
|
||||
_server_cmd: &server_cmd
|
||||
- "--port"
|
||||
- "$SERVER_PORT"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--data-parallel-size"
|
||||
- "1"
|
||||
- "--tensor-parallel-size"
|
||||
- "2"
|
||||
- "--max-num-seqs"
|
||||
- "32"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.9"
|
||||
- "--trust-remote-code"
|
||||
- "--async-scheduling"
|
||||
- "--allowed-local-media-path"
|
||||
- "/"
|
||||
- "--quantization"
|
||||
- "ascend"
|
||||
- "--mm-processor-cache-gb"
|
||||
- "0"
|
||||
- "--mm-encoder-tp-mode"
|
||||
- "data"
|
||||
- "--mm-processor-cache-type"
|
||||
- "shm"
|
||||
- "--additional-config"
|
||||
- '{"enable_cpu_binding": true}'
|
||||
- "--api-server-count"
|
||||
- "4"
|
||||
|
||||
_benchmarks_acc: &benchmarks_acc
|
||||
acc:
|
||||
case_type: accuracy
|
||||
dataset_path: vllm-ascend/gpqa
|
||||
request_conf: vllm_api_general_chat
|
||||
dataset_conf: gpqa/gpqa_gen_0_shot_cot_chat_prompt
|
||||
max_out_len: 32768
|
||||
batch_size: 32
|
||||
baseline: 84.34
|
||||
threshold: 5
|
||||
temperature: 0.6
|
||||
top_p: 0.95 v54
|
||||
ignore_eos: false
|
||||
thinking: true
|
||||
|
||||
_benchmarks_3500_50: &benchmarks_3500_50
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 60
|
||||
max_out_len: 1500
|
||||
batch_size: 15
|
||||
request_rate: 0
|
||||
baseline: 701.55
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_3500_20: &benchmarks_3500_20
|
||||
perf:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in3500-bs8000-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 48
|
||||
max_out_len: 1500
|
||||
batch_size: 12
|
||||
request_rate: 0
|
||||
baseline: 115.03
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_prefix_64k_50: &benchmarks_prefix_64k_50
|
||||
perf_prefix_90:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in65536_bs1000_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 1
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.1
|
||||
perf_64k:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix0_in65536_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 72
|
||||
max_out_len: 1024
|
||||
batch_size: 18
|
||||
request_rate: 0
|
||||
baseline: 284.9
|
||||
threshold: 0.97
|
||||
|
||||
_benchmarks_prefix_128k_50: &benchmarks_prefix_128k_50
|
||||
perf_prefix_90:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K_prefix90_in131072_bs100_qwen
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 1
|
||||
max_out_len: 1024
|
||||
batch_size: 1
|
||||
request_rate: 0
|
||||
baseline: 1
|
||||
threshold: 0.1
|
||||
perf_128k:
|
||||
case_type: performance
|
||||
dataset_path: vllm-ascend/GSM8K-in131072-bs100-qwen3
|
||||
request_conf: vllm_api_stream_chat
|
||||
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
||||
num_prompts: 24
|
||||
max_out_len: 1024
|
||||
batch_size: 6
|
||||
request_rate: 0
|
||||
baseline: 95.24
|
||||
threshold: 0.97
|
||||
# ==========================================
|
||||
# ACTUAL TEST CASES
|
||||
# ==========================================
|
||||
|
||||
test_cases:
|
||||
- name: "Qwen3.6-27B-acc"
|
||||
model: "Eco-Tech/Qwen3.6-27B-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,4,8,12,16,20,24,28,32,36,40,44,48,52,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128]}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 3}'
|
||||
- "--enable-prefix-caching"
|
||||
benchmarks:
|
||||
<<: *benchmarks_acc
|
||||
- name: "Qwen3.6-27B-3500-50"
|
||||
model: "Eco-Tech/Qwen3.6-27B-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "18432"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,4,8,12,16,20,24,28,32,36,40,44,48,52,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128]}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 3}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_3500_50
|
||||
- name: "Qwen3.6-27B-3500-20"
|
||||
model: "Eco-Tech/Qwen3.6-27B-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--max-num-batched-tokens"
|
||||
- "65536"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192]}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 5}'
|
||||
benchmarks:
|
||||
<<: *benchmarks_3500_20
|
||||
- name: "Qwen3.6-27B-prefix_64k_50"
|
||||
model: "Eco-Tech/Qwen3.6-27B-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "67584"
|
||||
- "--max-num-batched-tokens"
|
||||
- "65536"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192]}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 5}'
|
||||
- "--enable-prefix-caching"
|
||||
benchmarks:
|
||||
<<: *benchmarks_prefix_64k_50
|
||||
- name: "Qwen3.6-27B-prefix_128k_50"
|
||||
model: "Eco-Tech/Qwen3.6-27B-w8a8"
|
||||
envs:
|
||||
<<: *envs
|
||||
server_cmd: *server_cmd
|
||||
server_cmd_extra:
|
||||
- "--max-model-len"
|
||||
- "133120"
|
||||
- "--max-num-batched-tokens"
|
||||
- "16384"
|
||||
- "--compilation-config"
|
||||
- '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,4,8,12,16,20,24,28,32,36,40,44,48,52,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128]}'
|
||||
- "--speculative-config"
|
||||
- '{"method": "qwen3_5_mtp","num_speculative_tokens": 3}'
|
||||
- "--enable-prefix-caching"
|
||||
benchmarks:
|
||||
<<: *benchmarks_prefix_128k_50
|
||||
Reference in New Issue
Block a user