EngineX/xc-llm-ascend
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[1/N][CI] Refactor accuracy test (#5400)

Browse Source 
### What this PR does / why we need it?
1. Accuracy testing no longer compares eager and graph modes; instead,
it directly extracts the golden result under the graph mode
configuration (the implicit purpose of this case is to verify whether
modifications affect existing results)
2. Next step: finer-grained supervision of logits/sampler results
### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

- vLLM version: release/v0.13.0
- vLLM main:
254f6b9867

Signed-off-by: wangli <wangli858794774@gmail.com>
This commit is contained in:
Li Wang
2026-01-07 20:58:15 +08:00
committed by GitHub
parent b94fc13d3f
commit 1165b2c863
4 changed files with 231 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/e2e/multicard/2-cards/test_shared_expert_dp.py
View File

@@ -9,7 +9,6 @@ from tests.e2e.model_utils import check_outputs_equal
MODELS = [ MODELS = [
"deepseek-ai/DeepSeek-V2-Lite", "deepseek-ai/DeepSeek-V2-Lite",
] ]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("model", MODELS)

346
tests/e2e/singlecard/test_aclgraph_accuracy.py
View File

@@ -14,298 +14,122 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
"""
Compare the outputs of vLLM with and without aclgraph.
Run `pytest tests/compile/test_aclgraph_accuracy.py`.
"""
import os
import pytest import pytest
from vllm import SamplingParams
from tests.e2e.conftest import VllmRunner from tests.e2e.singlecard.utils import (PROMPTS_LONG, PROMPTS_SHORT,
from tests.e2e.model_utils import check_outputs_equal LLMTestCase, gen_and_valid)
MODELS = [ CASE_QWEN_ACLGRAPH = LLMTestCase(
"Qwen/Qwen3-0.6B", model="Qwen/Qwen3-0.6B",
"vllm-ascend/DeepSeek-V2-Lite-W8A8", prompts=PROMPTS_SHORT,
] golden_answers=[
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("max_tokens", [32])
def test_models_output(
model: str,
max_tokens: int,
) -> None:
prompts = [
"Hello, my name is", "The president of the United States is",
"The capital of France is", "The future of AI is"
]
vllm_aclgraph_qwen_answers = [
" Lina. I'm a 22-year-old student from China. I'm interested in studying in the US. I want to know if there are any", " Lina. I'm a 22-year-old student from China. I'm interested in studying in the US. I want to know if there are any",
' the same as the president of the United Nations. This is because the president of the United States is the same as the president of the United Nations. The president', ' the same as the president of the United Nations. This is because the president of the United States is the same as the president of the United Nations. The president',
' Paris. The capital of France is also the capital of the Republic of France. The capital of France is also the capital of the European Union. The capital of', ' Paris. The capital of France is also the capital of the Republic of France. The capital of France is also the capital of the European Union. The capital of',
' not just a technological frontier but a profound transformation of how we live, work, and interact with the world. As we stand at the intersection of artificial intelligence and' ' not just a technological frontier but a profound transformation of how we live, work, and interact with the world. As we stand at the intersection of artificial intelligence and'
] ],
)
vllm_aclgraph_ds_answers = [ CASE_DS_ACLGRAPH = LLMTestCase(
model="vllm-ascend/DeepSeek-V2-Lite-W8A8",
quantization="ascend",
prompts=PROMPTS_SHORT,
golden_answers=[
'\nI am a 20 year old student from the UK. I am currently studying for a degree in English Literature and Creative Writing. I have a passion', '\nI am a 20 year old student from the UK. I am currently studying for a degree in English Literature and Creative Writing. I have a passion',
' a man who has been in the public eye for decades. He has been a senator, a governor, and a businessman. He has also been married to the', ' a man who has been in the public eye for decades. He has been a senator, a governor, and a businessman. He has also been married to the',
' Paris, which is also the largest city in the country. The city is located on the River Seine and is known for its beautiful architecture, museums, and art', ' Paris, which is also the largest city in the country. The city is located on the River Seine and is known for its beautiful architecture, museums, and art',
' here.\nThe future of AI is here.\nThe future of AI is here.\nThe future of AI is here.\nThe future of AI is' ' here.\nThe future of AI is here.\nThe future of AI is here.\nThe future of AI is here.\nThe future of AI is'
] ],
)
sampling_params = SamplingParams(max_tokens=max_tokens, temperature=0.0) CASE_QWEN_FULL_DECODE_ONLY = LLMTestCase(
if model == "vllm-ascend/DeepSeek-V2-Lite-W8A8": model="Qwen/Qwen3-0.6B",
with VllmRunner( prompts=PROMPTS_LONG,
model, golden_answers=[
max_model_len=1024, ' \n\nTo solve this problem, we need to use the Law of Sines and Law of Cosines. Let me start by drawing triangle $ABC$ with the',
cudagraph_capture_sizes=[1, 2, 4, 8], " \n\nTo solve this problem, we can use the following approach: Let $ABCD$ be a unit square with coordinates $A(0,0), B",
quantization="ascend", ' \n\nTo solve this problem, we can use the following approach: Let $ \\alpha $ be the common real root of the two equations. Then, we can'
) as runner:
vllm_aclgraph_outputs = runner.model.generate(
prompts, sampling_params)
else:
with VllmRunner(
model,
max_model_len=1024,
cudagraph_capture_sizes=[1, 2, 4, 8],
) as runner:
vllm_aclgraph_outputs = runner.model.generate(
prompts, sampling_params)
vllm_aclgraph_outputs_list = []
for output in vllm_aclgraph_outputs:
vllm_aclgraph_outputs_list.append(
([output.outputs[0].index], output.outputs[0].text))
vllm_eager_outputs_list = ([
([0], answer) for answer in vllm_aclgraph_ds_answers
] if model == "vllm-ascend/DeepSeek-V2-Lite-W8A8" else [
([0], answer) for answer in vllm_aclgraph_qwen_answers
]) ])
check_outputs_equal( CASE_DS_FULL_DECODE_ONLY = LLMTestCase(
outputs_0_lst=vllm_eager_outputs_list, model="vllm-ascend/DeepSeek-V2-Lite-W8A8",
outputs_1_lst=vllm_aclgraph_outputs_list, quantization="ascend",
name_0="vllm_eager_outputs", prompts=PROMPTS_LONG,
name_1="vllm_aclgraph_outputs", golden_answers=[
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("max_tokens", [32])
def test_models_output_between_eager_and_full_decode_only(
model: str,
max_tokens: int,
) -> None:
if 'HCCL_OP_EXPANSION_MODE' in os.environ:
del os.environ['HCCL_OP_EXPANSION_MODE']
# NOTE: Randomly fill the prompt with the requested amount for
# the specified capture shape to prevent accuracy issues caused by padding
prompts = [
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'In triangle $ABC$, $\\sin \\angle A = \\frac{4}{5}$ and $\\angle A < 90^\\circ$. Let $D$'
'be a point outside triangle $ABC$ such that $\\angle BAD = \\angle DAC$,'
'$\\angle BDC = 90^\\circ$. Suppose $AD = 1$ and $\\frac{BD}{CD} = \\frac{3}{2}$.'
'If $AB + AC$ can be expressed in the form $\\frac{a\\sqrt{b}}{c}$,'
'where $a, b, c$ are pairwise relatively prime integers, find $a + b + c$.'
),
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'Let $ABCD$ be a unit square in the plane. Points $X$ and $Y$ are chosen'
'independently and uniformly at random on the perimeter of $ABCD$.'
'If the expected value of the area of triangle $\\triangle AXY$'
'can be expressed as $\\frac{m}{n}$, for relatively prime positive'
'integers $m$ and $n$, compute $m+n$.'),
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'Let $a, b, c$ be distinct numbers such that the equations $x^2 + ax + 1 = 0$'
'and $x^2 + bx + c = 0$ have a common real root, and the equations $x^2 + x + a = 0$'
'and $x^2 + cx + b = 0$ also have a common real root.'
'Compute the sum $a + b + c$.')
]
vllm_aclgraph_qwen_answers = [
' \n\nTo solve this problem, we need to use the Law of Sines and Law of Cosines. Let me start by drawing triangle $ABC$ with the',
' \n\nTo solve this problem, we can use the following approach: Let $ABCD$ be a unit square with coordinates $A(0,0), B',
' \n\nTo solve this problem, we can use the following approach: Let $ \\alpha $ be the common real root of the two equations. Then, we can'
]
vllm_aclgraph_ds_answers = [
'\n\nSelect an assignment template', '\n\nSelect an assignment template',
'\n\nSelect an assignment template', '\n\nSelect an assignment template',
'\n\nSelect an assignment template' '\n\nSelect an assignment template'
]
sampling_params = SamplingParams(max_tokens=max_tokens,
n=1,
temperature=0.0,
top_p=1.0,
top_k=1)
if model == "vllm-ascend/DeepSeek-V2-Lite-W8A8":
with VllmRunner(
model,
max_model_len=1024,
compilation_config={
"cudagraph_capture_sizes": [4, 8, 32, 64],
"cudagraph_mode": "FULL_DECODE_ONLY"
},
quantization="ascend",
) as runner:
vllm_aclgraph_outputs = runner.model.generate(
prompts, sampling_params)
else:
with VllmRunner(
model,
max_model_len=1024,
compilation_config={
"cudagraph_capture_sizes": [4, 8, 32, 64],
"cudagraph_mode": "FULL_DECODE_ONLY"
},
) as runner:
vllm_aclgraph_outputs = runner.model.generate(
prompts, sampling_params)
vllm_aclgraph_outputs_list = []
for output in vllm_aclgraph_outputs:
vllm_aclgraph_outputs_list.append(
([output.outputs[0].index], output.outputs[0].text))
vllm_eager_outputs_list = []
vllm_eager_outputs_list = ([
([0], answer) for answer in vllm_aclgraph_ds_answers
] if model == "vllm-ascend/DeepSeek-V2-Lite-W8A8" else [
([0], answer) for answer in vllm_aclgraph_qwen_answers
]) ])
check_outputs_equal( CASE_QWEN_EX = LLMTestCase(
outputs_0_lst=vllm_eager_outputs_list, model="Qwen/Qwen3-0.6B",
outputs_1_lst=vllm_aclgraph_outputs_list, prompts=PROMPTS_LONG,
name_0="vllm_eager_outputs", golden_answers=[
name_1="vllm_aclgraph_outputs",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("max_tokens", [32])
def test_models_output_between_eager_and_fullgraph_npugraph_ex(
model: str,
max_tokens: int,
) -> None:
if 'HCCL_OP_EXPANSION_MODE' in os.environ:
del os.environ['HCCL_OP_EXPANSION_MODE']
# NOTE: Randomly fill the prompt with the requested amount for
# the specified capture shape to prevent accuracy issues caused by padding
prompts = [
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'In triangle $ABC$, $\\sin \\angle A = \\frac{4}{5}$ and $\\angle A < 90^\\circ$. Let $D$'
'be a point outside triangle $ABC$ such that $\\angle BAD = \\angle DAC$,'
'$\\angle BDC = 90^\\circ$. Suppose $AD = 1$ and $\\frac{BD}{CD} = \\frac{3}{2}$.'
'If $AB + AC$ can be expressed in the form $\\frac{a\\sqrt{b}}{c}$,'
'where $a, b, c$ are pairwise relatively prime integers, find $a + b + c$.'
),
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'Let $ABCD$ be a unit square in the plane. Points $X$ and $Y$ are chosen'
'independently and uniformly at random on the perimeter of $ABCD$.'
'If the expected value of the area of triangle $\\triangle AXY$'
'can be expressed as $\\frac{m}{n}$, for relatively prime positive'
'integers $m$ and $n$, compute $m+n$.'),
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'Let $a, b, c$ be distinct numbers such that the equations $x^2 + ax + 1 = 0$'
'and $x^2 + bx + c = 0$ have a common real root, and the equations $x^2 + x + a = 0$'
'and $x^2 + cx + b = 0$ also have a common real root.'
'Compute the sum $a + b + c$.')
]
vllm_aclgraph_qwen_answers = [
' \n\nTo solve this problem, we need to use the Law of Sines and Law of Cosines. Let me start by drawing triangle $ABC$ with the', ' \n\nTo solve this problem, we need to use the Law of Sines and Law of Cosines. Let me start by drawing triangle $ABC$ with the',
" \n\nTo solve this problem, we can use the fact that the expected value of the area of a triangle formed by two random points on a square's perimeter is", " \n\nTo solve this problem, we can use the fact that the expected value of the area of a triangle formed by two random points on a square's perimeter is",
' \n\nTo solve this problem, we can use the following approach: Let $ \\alpha $ be the common real root of the two equations. Then, we can' ' \n\nTo solve this problem, we can use the following approach: Let $ \\alpha $ be the common real root of the two equations. Then, we can'
] ])
vllm_aclgraph_ds_answers = [ CASE_DS_EX = LLMTestCase(model="vllm-ascend/DeepSeek-V2-Lite-W8A8",
quantization="ascend",
prompts=PROMPTS_LONG,
golden_answers=[
'\n\nSelect an assignment template', '\n\nSelect an assignment template',
'\n\nSelect an assignment template', '\n\nSelect an assignment template',
'\n\nSelect an assignment template' '\n\nSelect an assignment template'
]
sampling_params = SamplingParams(max_tokens=max_tokens,
n=1,
temperature=0.0,
top_p=1.0,
top_k=1)
if model == "vllm-ascend/DeepSeek-V2-Lite-W8A8":
with VllmRunner(
model,
max_model_len=1024,
compilation_config={
"cudagraph_capture_sizes": [4, 8, 32, 64],
"cudagraph_mode": "FULL_DECODE_ONLY"
},
additional_config={"enable_npugraph_ex": True},
quantization="ascend",
) as runner:
vllm_aclgraph_outputs = runner.model.generate(
prompts, sampling_params)
else:
with VllmRunner(
model,
max_model_len=1024,
compilation_config={
"cudagraph_capture_sizes": [4, 8, 32, 64],
"cudagraph_mode": "FULL_DECODE_ONLY"
},
additional_config={"enable_npugraph_ex": True},
) as runner:
vllm_aclgraph_outputs = runner.model.generate(
prompts, sampling_params)
vllm_aclgraph_outputs_list = []
for output in vllm_aclgraph_outputs:
vllm_aclgraph_outputs_list.append(
([output.outputs[0].index], output.outputs[0].text))
vllm_eager_outputs_list = []
vllm_eager_outputs_list = ([
([0], answer) for answer in vllm_aclgraph_ds_answers
] if model == "vllm-ascend/DeepSeek-V2-Lite-W8A8" else [
([0], answer) for answer in vllm_aclgraph_qwen_answers
]) ])
check_outputs_equal(
outputs_0_lst=vllm_eager_outputs_list, @pytest.mark.parametrize("cur_case", [CASE_QWEN_ACLGRAPH, CASE_DS_ACLGRAPH])
outputs_1_lst=vllm_aclgraph_outputs_list, def test_piecewise_res_consistency(cur_case: LLMTestCase):
name_0="vllm_eager_outputs", runner_kwargs = {
name_1="vllm_aclgraph_outputs", "model_name": cur_case.model,
) "max_model_len": 1024,
"cudagraph_capture_sizes": [1, 2, 4, 8],
"quantization": cur_case.quantization,
}
gen_and_valid(runner_kwargs=runner_kwargs,
prompts=cur_case.prompts,
sampling_params=cur_case.sampling_params,
golden_answers=cur_case.golden_answers)
def test_aclgraph_enable(): @pytest.mark.parametrize(
# Generally, this test is not belong to e2e, but it is a good way to check if "cur_case", [CASE_QWEN_FULL_DECODE_ONLY, CASE_DS_FULL_DECODE_ONLY])
# aclgraph is enabled in real environment def test_full_decode_only_res_consistency(cur_case: LLMTestCase, monkeypatch):
from vllm.config.compilation import CompilationMode, CUDAGraphMode monkeypatch.delenv("HCCL_OP_EXPANSION_MODE", raising=False)
from vllm.engine.arg_utils import EngineArgs runner_kwargs = {
"model_name": cur_case.model,
"max_model_len": 1024,
"compilation_config": {
"cudagraph_capture_sizes": [4, 8, 32, 64],
"cudagraph_mode": "FULL_DECODE_ONLY"
},
"quantization": cur_case.quantization,
}
gen_and_valid(runner_kwargs=runner_kwargs,
prompts=cur_case.prompts,
sampling_params=cur_case.sampling_params,
golden_answers=cur_case.golden_answers)
from vllm_ascend.platform import NPUPlatform
# vLLM default mode is piecewise cudagraph @pytest.mark.parametrize("cur_case", [CASE_QWEN_EX, CASE_DS_EX])
config = EngineArgs() def test_npugraph_ex_res_consistency(cur_case: LLMTestCase, monkeypatch):
VllmConfig = config.create_engine_config() monkeypatch.delenv("HCCL_OP_EXPANSION_MODE", raising=False)
assert VllmConfig.compilation_config.cudagraph_mode == CUDAGraphMode.PIECEWISE runner_kwargs = {
"model_name": cur_case.model,
# after check_and_update_config, mode should be VLLM_COMPILE and piecewise cudagraph "quantization": cur_case.quantization,
NPUPlatform.check_and_update_config(VllmConfig) "max_model_len": 1024,
assert VllmConfig.compilation_config.mode == CompilationMode.VLLM_COMPILE "compilation_config": {
assert VllmConfig.compilation_config.cudagraph_mode == CUDAGraphMode.PIECEWISE "cudagraph_capture_sizes": [4, 8, 32, 64],
"cudagraph_mode": "FULL_DECODE_ONLY"
},
"additional_config": {
"enable_npugraph_ex": True
},
}
gen_and_valid(runner_kwargs=runner_kwargs,
prompts=cur_case.prompts,
sampling_params=cur_case.sampling_params,
golden_answers=cur_case.golden_answers)

151
tests/e2e/singlecard/test_xlite.py
View File

@@ -23,106 +23,85 @@ Run `pytest tests/e2e/singlecard/test_xlite.py`.
import os import os
import pytest import pytest
from vllm import SamplingParams
from tests.e2e.conftest import VllmRunner from tests.e2e.singlecard.utils import (PROMPTS_SHORT, LLMTestCase,
from tests.e2e.model_utils import check_outputs_equal gen_and_valid)
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_ASCEND_ENABLE_NZ"] = "2" os.environ["VLLM_ASCEND_ENABLE_NZ"] = "2"
MODELS = [ CASE_DECODE_ONLY = LLMTestCase(
"Qwen/Qwen3-0.6B", model="Qwen/Qwen3-0.6B",
] prompts=PROMPTS_SHORT,
golden_answers=[
@pytest.mark.skip(
reason="TODO: Re-enable xlite_decode_only e2e test when stable.")
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("max_tokens", [15])
def test_models_with_xlite_decode_only(
model: str,
max_tokens: int,
) -> None:
prompts = [
"Hello, my name is", "The president of the United States is",
"The capital of France is", "The future of AI is"
]
with VllmRunner(
model,
block_size=128,
max_model_len=1024,
additional_config={"xlite_graph_config": {
"enabled": True
}},
) as runner:
vllm_xlite_outputs_list = runner.generate_greedy(prompts,
max_tokens=max_tokens)
for idx in range(len(vllm_xlite_outputs_list)):
vllm_xlite_outputs_list[idx] = ([0],
vllm_xlite_outputs_list[idx][1])
vllm_xlite_answers = [
"Hello, my name is Lina. I'm a 22-year-old student from China.", "Hello, my name is Lina. I'm a 22-year-old student from China.",
'The president of the United States is the same as the president of the United Nations. This is because the president', 'The president of the United States is the same as the president of the United Nations. This is because the president',
'The capital of France is Paris. The capital of France is also the capital of the French Republic.', 'The capital of France is Paris. The capital of France is also the capital of the French Republic.',
'The future of AI is not just a technological challenge but a profound transformation of how we live, work' 'The future of AI is not just a technological challenge but a profound transformation of how we live, work'
] ],
sampling_params=SamplingParams(
max_tokens=15,
temperature=0.0,
top_p=1.0,
top_k=0,
n=1,
))
vllm_eager_outputs_list = [] CASE_FULL_DECODE_ONLY = LLMTestCase(
vllm_eager_outputs_list = ([([0], answer) model="Qwen/Qwen3-0.6B",
for answer in vllm_xlite_answers]) prompts=[
check_outputs_equal(
outputs_0_lst=vllm_eager_outputs_list,
outputs_1_lst=vllm_xlite_outputs_list,
name_0="vllm_eager_outputs",
name_1="vllm_xlite_outputs",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("max_tokens", [32])
def test_models_with_xlite_full_mode(
model: str,
max_tokens: int,
) -> None:
prompts = [
"Hello, my name is", "The president of the United States is", "Hello, my name is", "The president of the United States is",
"The capital of France is", "The future of AI is" "The capital of France is", "The future of AI is"
] ],
golden_answers=[
" Lina. I'm a 22-year-old student from China. I'm interested in studying in the US. I'm looking for a job in the",
' the same as the president of the United Nations. This is because the president of the United States is the same as the president of the United Nations. The president',
' Paris. The capital of Italy is Rome. The capital of Spain is Madrid. The capital of China is Beijing. The capital of Japan is Tokyo. The capital',
" not just about the technology itself, but about how we use it to solve real-world problems. As AI continues to evolve, it's important to consider the ethical"
],
sampling_params=SamplingParams(
max_tokens=32,
temperature=0.0,
top_p=1.0,
top_k=0,
n=1,
))
with VllmRunner(
model, @pytest.mark.skip(
block_size=128, reason="TODO: Re-enable xlite_decode_only e2e test when stable.")
max_model_len=1024, @pytest.mark.parametrize("cur_case", [CASE_DECODE_ONLY])
additional_config={ def test_models_with_xlite_decode_only(cur_case: LLMTestCase):
runner_kwargs = {
"model_name": cur_case.model,
"max_model_len": 1024,
"block_size": 128,
"additional_config": {
"xlite_graph_config": {
"enabled": True
}
},
}
gen_and_valid(runner_kwargs=runner_kwargs,
prompts=cur_case.prompts,
sampling_params=cur_case.sampling_params,
golden_answers=cur_case.golden_answers)
@pytest.mark.parametrize("cur_case", [CASE_FULL_DECODE_ONLY])
def test_models_with_xlite_full_mode(cur_case: LLMTestCase):
runner_kwargs = {
"model_name": cur_case.model,
"max_model_len": 1024,
"block_size": 128,
"additional_config": {
"xlite_graph_config": { "xlite_graph_config": {
"enabled": True, "enabled": True,
"full_mode": True "full_mode": True
} }
}, },
) as runner: }
vllm_xlite_outputs_list = runner.generate_greedy(prompts, gen_and_valid(runner_kwargs=runner_kwargs,
max_tokens=max_tokens) prompts=cur_case.prompts,
for idx in range(len(vllm_xlite_outputs_list)): sampling_params=cur_case.sampling_params,
vllm_xlite_outputs_list[idx] = ([0], golden_answers=cur_case.golden_answers)
vllm_xlite_outputs_list[idx][1])
vllm_xlite_answers = [
"Hello, my name is Lina. I'm a 22-year-old student from China. I'm interested in studying in the US. I'm looking for a job in the",
'The president of the United States is the same as the president of the United Nations. This is because the president of the United States is the same as the president of the United Nations. The president',
'The capital of France is Paris. The capital of Italy is Rome. The capital of Spain is Madrid. The capital of China is Beijing. The capital of Japan is Tokyo. The capital',
"The future of AI is not just about the technology itself, but about how we use it to solve real-world problems. As AI continues to evolve, it's important to consider the ethical"
]
vllm_eager_outputs_list = []
vllm_eager_outputs_list = ([([0], answer)
for answer in vllm_xlite_answers])
check_outputs_equal(
outputs_0_lst=vllm_eager_outputs_list,
outputs_1_lst=vllm_xlite_outputs_list,
name_0="vllm_eager_outputs",
name_1="vllm_xlite_outputs",
)

76
tests/e2e/singlecard/utils.py Normal file
View File

@@ -0,0 +1,76 @@
from dataclasses import dataclass, field
from typing import Optional
from vllm import SamplingParams
from tests.e2e.conftest import VllmRunner
from tests.e2e.model_utils import check_outputs_equal
PROMPTS_SHORT = [
"Hello, my name is", "The president of the United States is",
"The capital of France is", "The future of AI is"
]
# NOTE: Randomly fill the prompt with the requested amount for
# the specified capture shape to prevent accuracy issues caused by padding
PROMPTS_LONG = [
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'In triangle $ABC$, $\\sin \\angle A = \\frac{4}{5}$ and $\\angle A < 90^\\circ$. Let $D$'
'be a point outside triangle $ABC$ such that $\\angle BAD = \\angle DAC$,'
'$\\angle BDC = 90^\\circ$. Suppose $AD = 1$ and $\\frac{BD}{CD} = \\frac{3}{2}$.'
'If $AB + AC$ can be expressed in the form $\\frac{a\\sqrt{b}}{c}$,'
'where $a, b, c$ are pairwise relatively prime integers, find $a + b + c$.'
),
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'Let $ABCD$ be a unit square in the plane. Points $X$ and $Y$ are chosen'
'independently and uniformly at random on the perimeter of $ABCD$.'
'If the expected value of the area of triangle $\\triangle AXY$'
'can be expressed as $\\frac{m}{n}$, for relatively prime positive'
'integers $m$ and $n$, compute $m+n$.'),
('Solve the following math problem step by step.'
'The last line of your response should be of the form Answer: '
'$Answer (without quotes) where $Answer is the answer to the problem.\n\n'
'Let $a, b, c$ be distinct numbers such that the equations $x^2 + ax + 1 = 0$'
'and $x^2 + bx + c = 0$ have a common real root, and the equations $x^2 + x + a = 0$'
'and $x^2 + cx + b = 0$ also have a common real root.'
'Compute the sum $a + b + c$.')
]
@dataclass(frozen=True)
class LLMTestCase:
model: str
prompts: list[str]
golden_answers: list[str]
quantization: Optional[str] = None
sampling_params: SamplingParams = field(
default_factory=lambda: SamplingParams(
max_tokens=32,
temperature=0.0,
top_p=1.0,
top_k=0,
n=1,
))
def gen_and_valid(runner_kwargs: dict, prompts: list[str],
sampling_params: SamplingParams, golden_answers: list[str]):
with VllmRunner(**runner_kwargs) as runner:
vllm_aclgraph_outputs = runner.model.generate(
prompts=prompts, sampling_params=sampling_params)
outputs_gen = []
for output in vllm_aclgraph_outputs:
outputs_gen.append(([output.outputs[0].index], output.outputs[0].text))
output_origin = [([0], answer) for answer in golden_answers]
check_outputs_equal(
outputs_0_lst=output_origin,
outputs_1_lst=outputs_gen,
name_0="output_origin",
name_1="outputs_gen",
)