EngineX/xc-llm-ascend
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[main][Bugfix] Fix ngram precision issue and open e2e ngram test (#4090)

Browse Source 
### What this PR does / why we need it?
Fix ngram precision issue and open e2e ngram test

- vLLM version: v0.11.0
- vLLM main:
83f478bb19

---------

Signed-off-by: Icey <1790571317@qq.com>
Signed-off-by: zhaomingyu <zhaomingyu13@h-partners.com>
Co-authored-by: Icey <1790571317@qq.com>
This commit is contained in:
zhaomingyu13
2025-11-11 09:06:24 +08:00
committed by GitHub
parent 64220c68c5
commit 7ffbe73d54
3 changed files with 21 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/_e2e_test.yaml vendored
View File

@@ -108,8 +108,8 @@ jobs:
# ------------------------------------ v1 spec decode test ------------------------------------ #
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_mtp_correctness.py
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_mtp_torchair_correctness.py
# Fix me: OOM error
# pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_spec_decode.py
# Fix me: test_eagle_correctness OOM error
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_spec_decode.py
# TODO: Move ops test to nightly test
#pytest -sv tests/e2e/singlecard/ops/

7
tests/e2e/singlecard/spec_decode_v1/test_v1_spec_decode.py
View File

@@ -13,7 +13,7 @@ from tests.e2e.conftest import VllmRunner
@pytest.fixture
def test_prompts():
prompt_types = ["repeat", "sentence"]
num_prompts = 10
num_prompts = 100
prompts = []
random.seed(0)
@@ -70,7 +70,6 @@ def test_ngram_correctness(
Compare the outputs of a original LLM and a speculative LLM
should be the same when using ngram speculative decoding.
'''
pytest.skip("Not current support for the test.")
ref_llm = LLM(model=model_name, max_model_len=1024, enforce_eager=False)
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
del ref_llm
@@ -96,7 +95,7 @@ def test_ngram_correctness(
# Heuristic: expect at least 70% of the prompts to match exactly
# Upon failure, inspect the outputs to check for inaccuracy.
assert matches > int(0.7 * len(ref_outputs))
assert matches > int(0.66 * len(ref_outputs))
@pytest.mark.parametrize("use_eagle3", [False, True], ids=["eagle", "eagle3"])
@@ -110,7 +109,7 @@ def test_eagle_correctness(
Compare the outputs of a original LLM and a speculative LLM
should be the same when using eagle speculative decoding.
'''
pytest.skip("exist OOM error")
ref_llm = LLM(model=model_name, max_model_len=2048, enforce_eager=False)
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
del ref_llm

29
vllm_ascend/spec_decode/ngram_proposer.py
View File

@@ -39,30 +39,33 @@ class NgramProposer(VllmNgramProposer, Proposer):
hidden_states=None,
attn_metadata=None,
aux_hidden_states=None) -> list[list[int]]:
# TODO(woosuk): Optimize.
draft_token_ids: list[list[int]] = []
valid_ngram_requests = []
for i, sampled_ids in enumerate(valid_sampled_token_ids):
num_sampled_ids = len(sampled_ids)
if not num_sampled_ids:
# Skip speculative decoding.
draft_token_ids.append([])
continue
# Skip requests that require top-p, top-k, etc.
req_id = self.runner.input_batch.req_ids[i]
if req_id in self.runner.input_batch.spec_decode_unsupported_reqs:
draft_token_ids.append([])
continue
# Add sampled_token_ids to token_ids_cpu.
num_tokens = self.runner.input_batch.num_tokens_no_spec[i]
if num_tokens >= self.runner.input_batch.max_model_len:
# Skip requests that have already reached the max model length.
continue
start_idx = self.runner.input_batch.num_tokens_no_spec[i]
end_idx = start_idx + num_sampled_ids
self.runner.input_batch.token_ids_cpu[
i, start_idx:end_idx] = sampled_ids
drafter_output = self.propose(
self.runner.input_batch.token_ids_cpu[i, :end_idx])
if drafter_output is None or len(drafter_output) == 0:
draft_token_ids.append([])
else:
draft_token_ids.append(drafter_output.tolist())
valid_ngram_requests.append(i)
draft_token_ids = self.batch_propose(
len(valid_sampled_token_ids),
valid_ngram_requests,
self.runner.input_batch.num_tokens_no_spec,
self.runner.input_batch.token_ids_cpu,
)
return draft_token_ids