bc0fd7ca72
### What this PR does / why we need it?
Adapt the graph mode (piecewise and full_decode_only) of PCP and DCP for
DeepSeek v3.2.
### How was this patch tested?
Test output:
{"object":"text_completion","model":"deepeek_v3","choices":[{"index":0,"text":"
the head of state and head of government of the United States,
indirectly elected to a four-year term by the American people through
the Electoral College. The officeholder leads the executive branch of
the federal government and is the commander-in-chief of the United
States","logprobs":null,"finish_reason":"length","stop_reason":null,"token_ids":null,"prompt_logprobs":null,"prompt_token_ids":null},{"index":1,"text":"
Paris. This is the largest city in France and its main political,
cultural and commercial center. The modern location of the city is the
north of the central part of the country, on the banks of the Seine
River Seine River Seine in
3\n\n","logprobs":null,"finish_reason":"length","stop_reason":null,"token_ids":null,"prompt_logprobs":null,"prompt_token_ids":null},{"index":2,"text":"
now\n\n# AI future is now\n\nThe world is changing at a rapid pace, and
artificial intelligence (AI) is at the forefront of this transformation.
From self-driving cars to virtual assistants, AI is already making a
significant impact on our daily
lives","logprobs":null,"finish_reason":"length","stop_reason":null,"token_ids":null,"prompt_logprobs":null,"prompt_token_ids":null},{"index":3,"text":"
a 3rd year student at the University of Lincoln studying Media
Production. This blog is about my work throughout my final year on the
course.\n\n## Tuesday 3 May 2016\n### Final Major Project -
Evaluation\n\nFor my final project
I","logprobs":null,"finish_reason":"length","stop_reason":null,"token_ids":null,"prompt_logprobs":null,"prompt_token_ids":null}],"service_tier":null,"system_fingerprint":null,"usage":{"prompt_tokens":27,"total_tokens":227,"completion_tokens":200,"prompt_tokens_details":null},"kv_transfer_params":null}
- vLLM version: v0.16.0
- vLLM main:
15d76f74e2
---------
Signed-off-by: xiaocongtou6 <2066962956@qq.com>
Signed-off-by: xiaocongtou6 <105542647+xiaocongtou6@users.noreply.github.com>
284 lines
11 KiB
Python
284 lines
11 KiB
Python
#
|
|
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
|
|
# Copyright 2023 The vLLM team.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# This file is a part of the vllm-ascend project.
|
|
# Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
|
|
#
|
|
import os
|
|
|
|
from vllm import SamplingParams
|
|
|
|
from tests.e2e.conftest import VllmRunner, wait_until_npu_memory_free
|
|
|
|
os.environ["HCCL_BUFFSIZE"] = "768"
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_models_pcp_dcp_basic():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
enforce_eager=True,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=2,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
model = "vllm-ascend/Qwen3-30B-A3B-W8A8"
|
|
with VllmRunner(model,
|
|
enforce_eager=True,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=1,
|
|
enable_expert_parallel=True,
|
|
block_size=128,
|
|
quantization="ascend",
|
|
) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
model = "vllm-ascend/DeepSeek-V3.2-W8A8-Pruning"
|
|
with VllmRunner(
|
|
model,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=2,
|
|
enable_expert_parallel=True,
|
|
gpu_memory_utilization=0.2,
|
|
block_size=128,
|
|
quantization="ascend",
|
|
) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
model = "Qwen/Qwen3-Next-80B-A3B-Instruct"
|
|
with VllmRunner(model,
|
|
enforce_eager=True,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=1,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
gpu_memory_utilization=0.8,
|
|
block_size=128) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_models_pcp_dcp_full_graph():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=2,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128,
|
|
compilation_config={
|
|
"cudagraph_mode": "FULL_DECODE_ONLY",
|
|
"cudagraph_capture_sizes": [4, 8, 24, 48, 60]
|
|
}) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
model = "vllm-ascend/Qwen3-30B-A3B-W8A8"
|
|
with VllmRunner(model,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=1,
|
|
enable_expert_parallel=True,
|
|
block_size=128,
|
|
quantization="ascend",
|
|
compilation_config={
|
|
"cudagraph_mode": "FULL_DECODE_ONLY",
|
|
"cudagraph_capture_sizes": [4, 8, 24, 48, 60]
|
|
}) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_models_pcp_dcp_piece_wise():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=2,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
cudagraph_capture_sizes=[1, 2, 4, 8],
|
|
block_size=128) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
model = "vllm-ascend/Qwen3-30B-A3B-W8A8"
|
|
with VllmRunner(model,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=1,
|
|
enable_expert_parallel=True,
|
|
cudagraph_capture_sizes=[1, 2, 4, 8],
|
|
block_size=128,
|
|
quantization="ascend") as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_pcp_basic():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
enforce_eager=True,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=1,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_pcp_full_graph():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
enforce_eager=False,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=1,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128,
|
|
compilation_config={
|
|
"cudagraph_mode": "FULL_DECODE_ONLY",
|
|
"cudagraph_capture_sizes": [4, 8, 24, 48, 60]
|
|
}) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_pcp_piece_wise():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
enforce_eager=False,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=2,
|
|
prefill_context_parallel_size=2,
|
|
decode_context_parallel_size=1,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_dcp_basic():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
enforce_eager=True,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=4,
|
|
prefill_context_parallel_size=1,
|
|
decode_context_parallel_size=2,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_dcp_full_graph():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
enforce_eager=False,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=4,
|
|
prefill_context_parallel_size=1,
|
|
decode_context_parallel_size=2,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128,
|
|
compilation_config={
|
|
"cudagraph_mode": "FULL_DECODE_ONLY",
|
|
"cudagraph_capture_sizes": [4, 8, 24, 48, 60]
|
|
}) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|
|
|
|
|
|
@wait_until_npu_memory_free()
|
|
def test_dcp_piece_wise():
|
|
prompts = [
|
|
"The capital of France is", "Hello, my name is Tom, I am",
|
|
"The president of United States is", "AI future is"
|
|
]
|
|
model = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
|
sampling_params = SamplingParams(max_tokens=32, temperature=0.0)
|
|
with VllmRunner(model,
|
|
enforce_eager=False,
|
|
max_model_len=1024,
|
|
tensor_parallel_size=4,
|
|
prefill_context_parallel_size=1,
|
|
decode_context_parallel_size=2,
|
|
max_num_batched_tokens=1024,
|
|
enable_expert_parallel=True,
|
|
block_size=128) as runner:
|
|
runner.model.generate(prompts, sampling_params)
|