From 384d84c7ef46776ce01033759aff96ba3d4ac25f Mon Sep 17 00:00:00 2001 From: drslark <96540755+drslark@users.noreply.github.com> Date: Sun, 25 Jan 2026 17:45:29 +0800 Subject: [PATCH] [Bugfix] Avoided a bug of drafter when `dp` and `sp` are enabled (#6226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What this PR does / why we need it? Avoided a bug of drafter when `dp` and `sp` are enabled. Specifically, disable `sp` when drafter is dense. ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? An aisbench test: ```shell python3 aisbench_test.py --input_len 3500 --output_len 1000 --data_num 100 --concurrency 320 --request_rate 8 ``` The result is okay. ```text [2026-01-24 22:38:20,256] [ais_bench.benchmark.openicl.icl_inferencer.icl_gen_inferencer] [INFO] Calculate global interval offsets time: 0.5922 s 01/24 22:38:20 - AISBench - INFO - Process 0 using precomputed sleep offsets with 100 requests Process-0 pid:220279: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [09:40<00:00, 5.81s/it] Pid: 220279 | Post: 100 | Received: 100 | Failed: 0 | Post Time:12.51s | Receive Time:580.92s: Encoding output text...: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:01<00:00, 93.75it/s] 01/24 22:48:02 - AISBench - INFO - Start converting origin data to detailed data ... 01/24 22:48:02 - AISBench - INFO - Finish converting origin data to detailed data█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:01<00:00, 95.08it/s] 01/24 22:48:02 - AISBench - INFO - Added 'Actual RPS: After Excluding Anomalies' to group 'Time - RPS: ' in legend explanation table 01/24 22:48:02 - AISBench - INFO - Successfully merged chart into position (1, 1) 01/24 22:48:02 - AISBench - INFO - RPS distribution charts saved to outputs/default/20260124_223809/performances/vllm-api-stream-chat/gsm8kdataset_rps_distribution_plot_with_actual_rps.html 01/24 22:48:02 - AISBench - INFO - Updated chart with actual RPS saved to outputs/default/20260124_223809/performances/vllm-api-stream-chat/gsm8kdataset_rps_distribution_plot_with_actual_rps.html [2026-01-24 22:48:02,557] [ais_bench.benchmark.openicl.icl_inferencer.icl_gen_perf_inferencer] [INFO] Start extracting pref datas ... [2026-01-24 22:48:02,558] [ais_bench.benchmark.openicl.icl_inferencer.icl_gen_perf_inferencer] [INFO] Finish extracting pref datas! [2026-01-24 22:48:02,558] [ais_bench.benchmark.openicl.icl_inferencer.icl_gen_perf_inferencer] [INFO] Dumping detail perf data ... Dumping data to h5: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 75.31it/s] [2026-01-24 22:48:02,588] [ais_bench.benchmark.openicl.icl_inferencer.icl_gen_perf_inferencer] [INFO] Dump detail perf data cost: 0.02995561994612217(s) [2026-01-24 22:48:02,588] [ais_bench.benchmark.openicl.icl_inferencer.icl_gen_perf_inferencer] [INFO] Performance task finished, results saved in outputs/default/20260124_223809/performances/vllm-api-stream-chat 01/24 22:48:02 - AISBench - INFO - time elapsed: 586.32s Running tasks: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [09:55<00:00, 595.91s/it] 01/24 22:48:05 - AISBench - INFO - Performance evaluation tasks completed. 01/24 22:48:05 - AISBench - INFO - Loading detail perf data of model='vllm-api-stream-chat' dataset='gsm8kdataset' ... 01/24 22:48:05 - AISBench - INFO - Starting request timeline processing... 01/24 22:48:05 - AISBench - INFO - Data preprocessing completed in 0.0004s 01/24 22:48:05 - AISBench - INFO - Generating timeline traces for 100 requests... 01/24 22:48:05 - AISBench - INFO - Generated timeline trace chunks in 0.0441s 01/24 22:48:05 - AISBench - INFO - Generating concurrency traces... 01/24 22:48:05 - AISBench - INFO - Generated concurrency trace chunks in 0.0011s 01/24 22:48:05 - AISBench - INFO - Creating figure layout... 01/24 22:48:05 - AISBench - INFO - Figure layout created in 0.0504s 01/24 22:48:05 - AISBench - INFO - Writing to outputs/default/20260124_223809/performances/vllm-api-stream-chat/gsm8kdataset_plot.html... 01/24 22:48:05 - AISBench - INFO - HTML written in 0.0181s 01/24 22:48:05 - AISBench - INFO - Completed! Total execution time: 0.1148s 01/24 22:48:05 - AISBench - INFO - The gsm8kdataset_plot has been saved in outputs/default/20260124_223809/performances/vllm-api-stream-chat/gsm8kdataset_plot.html 01/24 22:48:05 - AISBench - INFO - Converting perf results of stage ... 01/24 22:48:05 - AISBench - INFO - Finish Converting! 01/24 22:48:05 - AISBench - INFO - Start calculating metrics ... 01/24 22:48:05 - AISBench - INFO - Start calculating common metrics ... 01/24 22:48:05 - AISBench - INFO - Start calculating add units ... 01/24 22:48:05 - AISBench - INFO - Finish calculating perf data! 01/24 22:48:05 - AISBench - INFO - Summarizing performance results... 01/24 22:48:05 - AISBench - INFO - Performance Results of task: vllm-api-stream-chat/gsm8kdataset: ╒══════════════════════════╤═════════╤════════════════╤════════════════╤════════════════╤════════════════╤════════════════╤════════════════╤════════════════╤═════╕ │ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ ╞══════════════════════════╪═════════╪════════════════╪════════════════╪════════════════╪════════════════╪════════════════╪════════════════╪════════════════╪═════╡ │ E2EL │ total │ 300806.1781 ms │ 189326.0489 ms │ 568345.5121 ms │ 380629.6785 ms │ 384208.3527 ms │ 385363.7709 ms │ 566871.7684 ms │ 100 │ ├──────────────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────┤ │ TTFT │ total │ 107441.2231 ms │ 343.8054 ms │ 378132.3979 ms │ 188817.4877 ms │ 190985.8451 ms │ 192547.6847 ms │ 378008.356 ms │ 100 │ ├──────────────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────┤ │ TPOT │ total │ 193.5585 ms │ 185.1008 ms │ 197.262 ms │ 193.8146 ms │ 195.0803 ms │ 196.0323 ms │ 196.9688 ms │ 100 │ ├──────────────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────┤ │ ITL │ total │ 194.2067 ms │ 0.0108 ms │ 2782.7124 ms │ 184.9998 ms │ 194.2631 ms │ 221.2895 ms │ 304.363 ms │ 100 │ ├──────────────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────┤ │ InputTokens │ total │ 3506.86 │ 3431.0 │ 3508.0 │ 3508.0 │ 3508.0 │ 3508.0 │ 3508.0 │ 100 │ ├──────────────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────┤ │ OutputTokens │ total │ 1000.0 │ 1000.0 │ 1000.0 │ 1000.0 │ 1000.0 │ 1000.0 │ 1000.0 │ 100 │ ├──────────────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────┤ │ OutputTokenThroughput │ total │ 3.7745 token/s │ 1.7595 token/s │ 5.2819 token/s │ 2.6272 token/s │ 5.1028 token/s │ 5.1502 token/s │ 5.2754 token/s │ 100 │ ╘══════════════════════════╧═════════╧════════════════╧════════════════╧════════════════╧════════════════╧════════════════╧════════════════╧════════════════╧═════╛ ╒══════════════════════════╤═════════╤══════════════════╕ │ Common Metric │ Stage │ Value │ ╞══════════════════════════╪═════════╪══════════════════╡ │ Benchmark Duration │ total │ 580456.2704 ms │ ├──────────────────────────┼─────────┼──────────────────┤ │ Total Requests │ total │ 100 │ ├──────────────────────────┼─────────┼──────────────────┤ │ Failed Requests │ total │ 0 │ ├──────────────────────────┼─────────┼──────────────────┤ │ Success Requests │ total │ 100 │ ├──────────────────────────┼─────────┼──────────────────┤ │ Concurrency │ total │ 51.8224 │ ├──────────────────────────┼─────────┼──────────────────┤ │ Max Concurrency │ total │ 320 │ ├──────────────────────────┼─────────┼──────────────────┤ │ Request Throughput │ total │ 0.1723 req/s │ ├──────────────────────────┼─────────┼──────────────────┤ │ Total Input Tokens │ total │ 350686 │ ├──────────────────────────┼─────────┼──────────────────┤ │ Prefill Token Throughput │ total │ 32.6398 token/s │ ├──────────────────────────┼─────────┼──────────────────┤ │ Total generated tokens │ total │ 100000 │ ├──────────────────────────┼─────────┼──────────────────┤ │ Input Token Throughput │ total │ 604.1558 token/s │ ├──────────────────────────┼─────────┼──────────────────┤ │ Output Token Throughput │ total │ 172.2783 token/s │ ├──────────────────────────┼─────────┼──────────────────┤ │ Total Token Throughput │ total │ 776.434 token/s │ ╘══════════════════════════╧═════════╧══════════════════╛ 01/24 22:48:05 - AISBench - INFO - Performance Result files locate in outputs/default/20260124_223809/performances/vllm-api-stream-chat. ``` - vLLM version: v0.14.0 - vLLM main: https://github.com/vllm-project/vllm/commit/d68209402ddab3f54a09bc1f4de9a9495a283b60 Signed-off-by: drslark --- vllm_ascend/ascend_forward_context.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vllm_ascend/ascend_forward_context.py b/vllm_ascend/ascend_forward_context.py index ea137b04..d59f02be 100644 --- a/vllm_ascend/ascend_forward_context.py +++ b/vllm_ascend/ascend_forward_context.py @@ -87,6 +87,10 @@ def set_ascend_forward_context( if is_context_moe_model: sp_enabled = enable_sp(vllm_config) and num_tokens is not None mmrs_fusion = False + elif is_draft_model: + # TODO: for dense drafter, `sp` is redundant and is not compatible with `dp` and `graph`. + # Disable it to avoid more problems. + sp_enabled = False else: sp_enabled = enable_sp(vllm_config) and num_tokens is not None and num_tokens > 1000