enginex-ascend-910-vllm/tests/e2e/weekly/multi_node/external_dp/config/GLM-5.2-W4A8C8-3.5k-1.5k-0-50-PD.yaml

test_name: "Eco-Tech/GLM-5.2-w4a8c8-3.5k-1.5k-0-50"
model: "Eco-Tech/GLM-5.2-w4a8c8"
num_nodes: 4
npu_per_node: 16


routing:
  type: "disaggregated_prefill"
  groups:
    prefiller: [0, 1]
    decoder: [2, 3]

config:
  - node_index: 0
    port_start: 7100
    dp_rpc_port: 13321
    dp_size: 4
    dp_size_local: 2
    dp_rank_start: 0
    tp_size: 8
    dp_address: "${NODE_0_IP}"

  - node_index: 1
    port_start: 7100
    dp_rpc_port: 13321
    dp_size: 4
    dp_size_local: 2
    dp_rank_start: 2
    tp_size: 8
    dp_address: "${NODE_0_IP}"

  - node_index: 2
    port_start: 7100
    dp_rpc_port: 13321
    dp_size: 32
    dp_size_local: 16
    dp_rank_start: 0
    tp_size: 1
    dp_address: "${NODE_2_IP}"

  - node_index: 3
    port_start: 7100
    dp_rpc_port: 13321
    dp_size: 32
    dp_size_local: 16
    dp_rank_start: 16
    tp_size: 1
    dp_address: "${NODE_2_IP}"

env_common: &env_common
  VLLM_USE_MODELSCOPE: "true"
  SERVER_PORT: "${PORT}"
  ASCEND_RT_VISIBLE_DEVICES: "${VISIBLE_DEVICES}"
  VLLM_ASCEND_ENABLE_FUSED_MC2: "1"
  HCCL_OP_EXPANSION_MODE: "AIV"
  OMP_PROC_BIND: "false"
  OMP_NUM_THREADS: "1"
  PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
  ACL_OP_INIT_MODE: "1"
  ASCEND_A3_ENABLE: "1"

templates:
  - node_index: 0
    envs:
      <<: *env_common
      VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
      HCCL_BUFFSIZE: "400"
    server_cmd_template:
      - --host
      - "0.0.0.0"
      - --port
      - $SERVER_PORT
      - --data-parallel-size
      - ${DP_SIZE}
      - --data-parallel-rank
      - ${DP_RANK}
      - --data-parallel-address
      - ${DP_ADDRESS}
      - --data-parallel-rpc-port
      - ${DP_RPC_PORT}
      - --tensor-parallel-size
      - ${TP_SIZE}
      - --enable-expert-parallel
      - --hf-overrides
      - '{"use_index_cache": true, "index_topk_freq": 4}'
      - --speculative-config
      - '{"num_speculative_tokens": 3, "method":"deepseek_mtp","enforce_eager":true}'
      - --seed
      - "1024"
      - --max-model-len
      - "18432"
      - --additional-config
      - '{"fuse_muls_add":true,"recompute_scheduler_enable" : false,"multistream_overlap_shared_expert": true, "enable_dsa_cp":true, "enable_sparse_sfa_c8": false, "enable_sparse_li_c8": false, "c8_enable_reshape_optim":false}'
      - --max-num-batched-tokens
      - "8192"
      - --trust-remote-code
      - --no-enable-prefix-caching
      - --max-num-seqs
      - "64"
      - --quantization
      - "ascend"
      - --gpu-memory-utilization
      - "0.92"
      - --enforce-eager
      - --enable-auto-tool-choice
      - --tool-call-parser
      - "glm47"
      - --reasoning-parser
      - "glm45"
      - --kv-transfer-config
      - '{"kv_connector": "MooncakeConnectorV1",
        "kv_role": "kv_producer",
        "kv_port": "30001",
        "engine_id": "0",
        "kv_connector_extra_config": {
                    "use_ascend_direct": true,
                    "prefill": {
                            "dp_size": 4,
                            "tp_size": 8
                    },
                    "decode": {
                            "dp_size": 32,
                            "tp_size": 1
                    }
            }
        }'
  - node_index: 1
    envs:
      <<: *env_common
      VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
      HCCL_BUFFSIZE: "400"
    server_cmd_template:
      - --host
      - "0.0.0.0"
      - --port
      - $SERVER_PORT
      - --data-parallel-size
      - ${DP_SIZE}
      - --data-parallel-rank
      - ${DP_RANK}
      - --data-parallel-address
      - ${DP_ADDRESS}
      - --data-parallel-rpc-port
      - ${DP_RPC_PORT}
      - --tensor-parallel-size
      - ${TP_SIZE}
      - --enable-expert-parallel
      - --hf-overrides
      - '{"use_index_cache": true, "index_topk_freq": 4}'
      - --speculative-config
      - '{"num_speculative_tokens": 3, "method":"deepseek_mtp","enforce_eager":true}'
      - --seed
      - "1024"
      - --max-model-len
      - "18432"
      - --additional-config
      - '{"fuse_muls_add":true,"recompute_scheduler_enable" : false,"multistream_overlap_shared_expert": true, "enable_dsa_cp":true, "enable_sparse_sfa_c8": false, "enable_sparse_li_c8": false, "c8_enable_reshape_optim":false}'
      - --max-num-batched-tokens
      - "8192"
      - --trust-remote-code
      - --no-enable-prefix-caching
      - --max-num-seqs
      - "64"
      - --quantization
      - "ascend"
      - --gpu-memory-utilization
      - "0.92"
      - --enforce-eager
      - --enable-auto-tool-choice
      - --tool-call-parser
      - "glm47"
      - --reasoning-parser
      - "glm45"
      - --kv-transfer-config
      - '{"kv_connector": "MooncakeConnectorV1",
        "kv_role": "kv_producer",
        "kv_port": "30001",
        "engine_id": "1",
        "kv_connector_extra_config": {
                    "use_ascend_direct": true,
                    "prefill": {
                            "dp_size": 4,
                            "tp_size": 8
                    },
                    "decode": {
                            "dp_size": 32,
                            "tp_size": 1
                    }
            }
        }'
  - node_index: 2
    envs:
      <<: *env_common
      HCCL_BUFFSIZE: "256"
      TASK_QUEUE_ENABLE: "1"
      VLLM_ASCEND_ENABLE_MLAPO: "1"
    server_cmd_template:
      - --host
      - "0.0.0.0"
      - --port
      - $SERVER_PORT
      - --data-parallel-size
      - ${DP_SIZE}
      - --data-parallel-rank
      - ${DP_RANK}
      - --data-parallel-address
      - ${DP_ADDRESS}
      - --data-parallel-rpc-port
      - ${DP_RPC_PORT}
      - --tensor-parallel-size
      - ${TP_SIZE}
      - --enable-expert-parallel
      - --hf-overrides
      - '{"use_index_cache": true, "index_topk_freq": 4}'
      - --seed
      - "1024"
      - --max-model-len
      - "18432"
      - --max-num-batched-tokens
      - "164"
      - --compilation-config
      - '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
      - --speculative-config
      - '{"num_speculative_tokens": 3,  "method":"deepseek_mtp","enforce_eager":true}'
      - --additional-config
      - '{"fuse_muls_add":true,"recompute_scheduler_enable":true,"multistream_overlap_shared_expert":true, "enable_sparse_sfa_c8": false, "enable_sparse_li_c8": false, "finegrained_tp_config": {"lmhead_tensor_parallel_size":16}}'
      - --trust-remote-code
      - --max-num-seqs
      - "32"
      - --gpu-memory-utilization
      - "0.92"
      - --async-scheduling
      - --quantization
      - "ascend"
      - --enable-auto-tool-choice
      - --tool-call-parser
      - "glm47"
      - --reasoning-parser
      - "glm45"
      - --kv-transfer-config
      - '{"kv_connector": "MooncakeConnectorV1",
        "kv_role": "kv_consumer",
        "kv_port": "30203",
        "engine_id": "2",
        "kv_connector_extra_config": {
                    "use_ascend_direct": true,
                    "prefill": {
                            "dp_size": 4,
                            "tp_size": 8
                    },
                    "decode": {
                            "dp_size": 32,
                            "tp_size": 1
                    }
            }
        }'
  - node_index: 3
    envs:
      <<: *env_common
      HCCL_BUFFSIZE: "256"
      TASK_QUEUE_ENABLE: "1"
      VLLM_ASCEND_ENABLE_MLAPO: "1"
    server_cmd_template:
      - --host
      - "0.0.0.0"
      - --port
      - $SERVER_PORT
      - --data-parallel-size
      - ${DP_SIZE}
      - --data-parallel-rank
      - ${DP_RANK}
      - --data-parallel-address
      - ${DP_ADDRESS}
      - --data-parallel-rpc-port
      - ${DP_RPC_PORT}
      - --tensor-parallel-size
      - ${TP_SIZE}
      - --enable-expert-parallel
      - --hf-overrides
      - '{"use_index_cache": true, "index_topk_freq": 4}'
      - --seed
      - "1024"
      - --max-model-len
      - "18432"
      - --max-num-batched-tokens
      - "164"
      - --compilation-config
      - '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
      - --speculative-config
      - '{"num_speculative_tokens": 3,  "method":"deepseek_mtp","enforce_eager":true}'
      - --additional-config
      - '{"fuse_muls_add":true,"recompute_scheduler_enable":true,"multistream_overlap_shared_expert":true, "enable_sparse_sfa_c8": false, "enable_sparse_li_c8": false, "finegrained_tp_config": {"lmhead_tensor_parallel_size":16}}'
      - --trust-remote-code
      - --max-num-seqs
      - "32"
      - --gpu-memory-utilization
      - "0.92"
      - --async-scheduling
      - --quantization
      - "ascend"
      - --enable-auto-tool-choice
      - --tool-call-parser
      - "glm47"
      - --reasoning-parser
      - "glm45"
      - --kv-transfer-config
      - '{"kv_connector": "MooncakeConnectorV1",
        "kv_role": "kv_consumer",
        "kv_port": "30203",
        "engine_id": "3",
        "kv_connector_extra_config": {
                    "use_ascend_direct": true,
                    "prefill": {
                            "dp_size": 4,
                            "tp_size": 8
                    },
                    "decode": {
                            "dp_size": 32,
                            "tp_size": 1
                    }
            }
        }'


benchmarks:
      perf_TPOT50:
        case_type: performance
        dataset_path: vllm-ascend/GSM8K-in3500-bs2800
        request_conf: vllm_api_stream_chat
        dataset_conf: gsm8k/gsm8k_gen_0_shot_cot_str_perf
        num_prompts: 1920
        max_out_len: 1500
        batch_size: 480
        request_rate: 6.5
        baseline: 8240.61
        threshold: 0.97