From 99fa0ac882c79ae9282940125b042a44ea422757 Mon Sep 17 00:00:00 2001 From: yangqinghao-cmss Date: Fri, 1 Aug 2025 08:56:55 +0800 Subject: [PATCH] [BugFix] update the kv transfer config (#2121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What this PR does / why we need it? The functions KVTransferConfig.from_cli and AscendHcclConnector are missing in the latest vLLM version. To resolve this, I propose modifying the kv_connector to use LLMDataDistCMgrConnector, which depends on [PR #2079](https://github.com/vllm-project/vllm-ascend/pull/2079) ### Does this PR introduce _any_ user-facing change? ### How was this patch tested? vllm:main vllm-ascend:mian results: ```bash Adding requests: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 374.27it/s] Processed prompts: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 66.06it/s, est. speed input: 449.08 toks/s, output: 66.51 toks/s] Prefill node is finished. INFO 07-31 09:18:30 [model_runner_v1.py:2282] Graph capturing finished in 36 secs, took 0.21 GiB INFO 07-31 09:18:30 [core.py:201] init engine (profile, create kv cache, warmup model) took 52.49 seconds INFO 07-31 09:18:30 [factory.py:74] Creating v1 connector with name: LLMDataDistCMgrConnector and engine_id: 28c8ced8-575c-4f87-840a-48d04d0edf7e INFO 07-31 09:18:30 [platform.py:157] PIECEWISE compilation enabled on NPU. use_inductor not supported - using only ACL Graph mode INFO 07-31 09:18:30 [utils.py:333] Calculated maximum supported batch sizes for ACL graph: 76 INFO 07-31 09:18:30 [utils.py:359] No adjustment needed for ACL graph batch sizes: Qwen2ForCausalLM model (layers: 24) with 67 sizes INFO 07-31 09:18:30 [llm.py:293] Supported_tasks: ['generate'] Waiting for prefill node to finish... Adding requests: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 709.70it/s] Processed prompts: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 16.23it/s, est. speed input: 109.70 toks/s, output: 260.01 toks/s] Prompt: 'Hello, how are you today?', Generated text: " I'm a computer program, so I don't have feelings. But I can" Prompt: 'Hi, what is your name?', Generated text: ' I am a computer programmer. I have a question about the programming language I am' Prompt: 'Tell me a very long story.', Generated text: ' I want to read it. I want to read it. I want to read' Prompt: 'what is your favourite book?', Generated text: " I'm sorry, but as an AI language model, I don't have personal" Cleanup prefill resources All process done ``` - vLLM version: v0.10.0 - vLLM main: https://github.com/vllm-project/vllm/commit/9cb497bfa346721aaf5e09a7f483764a1a54f8b4 Signed-off-by: yangqinghao-cmss --- examples/offline_disaggregated_prefill_npu.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/offline_disaggregated_prefill_npu.py b/examples/offline_disaggregated_prefill_npu.py index 84fa3fe..f37b508 100644 --- a/examples/offline_disaggregated_prefill_npu.py +++ b/examples/offline_disaggregated_prefill_npu.py @@ -37,7 +37,10 @@ def clean_up(): def run_prefill(prefill_done, process_close): - os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1" + # ranktable.json needs be generated using gen_ranktable.sh + # from the examples/disaggregated_prefill_v1 in the main branch. + os.environ['DISAGGREGATED_PREFILL_RANK_TABLE_PATH'] = "./ranktable.json" + os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0" from vllm import LLM, SamplingParams from vllm.config import KVTransferConfig @@ -48,16 +51,15 @@ def run_prefill(prefill_done, process_close): ] sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=1) - ktc = KVTransferConfig.from_cli( - '{"kv_connector":"AscendHcclConnector","kv_buffer_device":"npu","kv_role":"kv_producer", "kv_parallel_size":2}' - ) - + ktc = KVTransferConfig(kv_connector="LLMDataDistCMgrConnector", kv_buffer_device="npu", kv_role="kv_producer", + kv_parallel_size=1, + kv_connector_module_path="vllm_ascend.distributed.llmdatadist_c_mgr_connector") # Set NPU memory utilization to 0.8 llm = LLM(model="deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", kv_transfer_config=ktc, max_model_len=2000, gpu_memory_utilization=0.8, - tensor_parallel_size=2) + tensor_parallel_size=1) llm.generate(prompts, sampling_params) print("Prefill node is finished.") @@ -77,7 +79,11 @@ def run_prefill(prefill_done, process_close): def run_decode(prefill_done): - os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "2,3" + os.environ['VLLM_LLMDD_RPC_PORT'] = '6634' + # ranktable.json needs be generated using gen_ranktable.sh + # from the examples/disaggregated_prefill_v1 module in the main branch. + os.environ['DISAGGREGATED_PREFILL_RANK_TABLE_PATH'] = "./ranktable.json" + os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "1" from vllm import LLM, SamplingParams from vllm.config import KVTransferConfig @@ -88,15 +94,14 @@ def run_decode(prefill_done): ] sampling_params = SamplingParams(temperature=0, top_p=0.95) - ktc = KVTransferConfig.from_cli( - '{"kv_connector":"AscendHcclConnector","kv_buffer_device":"npu","kv_role":"kv_consumer","kv_parallel_size":2}' - ) + ktc = KVTransferConfig(kv_connector="LLMDataDistCMgrConnector", kv_buffer_device="npu", kv_role="kv_consumer", + kv_parallel_size=1, kv_connector_module_path="vllm_ascend.distributed.llmdatadist_c_mgr_connector") llm = LLM(model="deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", kv_transfer_config=ktc, max_model_len=2000, gpu_memory_utilization=0.8, - tensor_parallel_size=2) + tensor_parallel_size=1) # Wait for the producer to start the consumer print("Waiting for prefill node to finish...")