70 lines
2.2 KiB
YAML
70 lines
2.2 KiB
YAML
# ==========================================
|
|
# ACTUAL TEST CASES
|
|
# ==========================================
|
|
test_cases:
|
|
- name: "Qwen3.5-397B-A17B-w8a8-3.5k-1.5k-0-50-single"
|
|
model: "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
|
|
envs:
|
|
VLLM_USE_MODELSCOPE: "true"
|
|
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
HCCL_BUFFSIZE: "1024"
|
|
OMP_NUM_THREADS: "1"
|
|
TASK_QUEUE_ENABLE: "1"
|
|
VLLM_ASCEND_ENABLE_FUSED_MC2: "0"
|
|
VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
|
|
HCCL_OP_EXPANSION_MODE: "AIV"
|
|
SERVER_PORT: "DEFAULT_PORT"
|
|
server_cmd:
|
|
- "--port"
|
|
- "$SERVER_PORT"
|
|
- --data-parallel-size
|
|
- "1"
|
|
- --tensor-parallel-size
|
|
- "16"
|
|
- --enable-expert-parallel
|
|
- --max-model-len
|
|
- "6144"
|
|
- --max-num-batched-tokens
|
|
- "16384"
|
|
- --max-num-seqs
|
|
- "128"
|
|
- --gpu-memory-utilization
|
|
- "0.9"
|
|
- --compilation-config
|
|
- '{"cudagraph_capture_sizes":[1,4,8,12,16,24,32,48,56,64,72,84,96,108,112,128,160,172,196,200,212,232,256,272,288,312,328,344,360,384,400,416,432,448,480,512], "cudagraph_mode":"FULL_DECODE_ONLY"}'
|
|
- --speculative_config
|
|
- '{"method": "qwen3_5_mtp", "num_speculative_tokens": 3, "enforce_eager": true}'
|
|
- --trust-remote-code
|
|
- --async-scheduling
|
|
- --allowed-local-media-path
|
|
- "/"
|
|
- --quantization
|
|
- "ascend"
|
|
- --mm-processor-cache-gb
|
|
- "0"
|
|
- --additional-config
|
|
- '{"enable_cpu_binding":true}'
|
|
|
|
benchmarks:
|
|
perf_50:
|
|
case_type: performance
|
|
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
|
|
request_conf: vllm_api_stream_chat
|
|
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
|
num_prompts: 640
|
|
max_out_len: 1500
|
|
batch_size: 160
|
|
request_rate: 5
|
|
baseline: 2663.3728
|
|
threshold: 0.97
|
|
perf_20:
|
|
case_type: performance
|
|
dataset_path: vllm-ascend/GSM8K-in3500-bs2800
|
|
request_conf: vllm_api_stream_chat
|
|
dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
|
|
num_prompts: 128
|
|
max_out_len: 1500
|
|
batch_size: 32
|
|
request_rate: 0
|
|
baseline: 1421.89
|
|
threshold: 0.97 |