init v0.23.0

Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>

tests/e2e/pull_request/one_card/spec_decode/conftest.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import random
+from typing import Any
+
+import pytest
+from vllm import SamplingParams
+
+
+@pytest.fixture
+def test_prompts() -> list[list[dict[str, Any]]]:
+    prompt_types = ["repeat", "sentence"]
+    num_prompts = 100
+    prompts = []
+
+    random.seed(0)
+    random_prompt_type_choices = random.choices(prompt_types, k=num_prompts)
+
+    for kind in random_prompt_type_choices:
+        word_choices = ["test", "temp", "hello", "where"]
+        word = random.choice(word_choices)
+        if kind == "repeat":
+            prompt = f"""
+            please repeat the word '{word}' 10 times.
+            give no other output than the word at least ten times in a row,
+            in lowercase with spaces between each word and without quotes.
+            """
+        elif kind == "sentence":
+            prompt = f"""
+            please give a ten-word sentence that
+            uses the word {word} at least once.
+            give no other output than that simple sentence without quotes.
+            """
+        else:
+            raise ValueError(f"Unknown prompt type: {kind}")
+        prompts.append([{"role": "user", "content": prompt}])
+
+    return prompts
+
+
+@pytest.fixture
+def sampling_config() -> SamplingParams:
+    return SamplingParams(temperature=0, max_tokens=10, ignore_eos=False)
+
+
+@pytest.fixture
+def model_name() -> str:
+    return "LLM-Research/Meta-Llama-3.1-8B-Instruct"
+
+
+@pytest.fixture
+def vl_model_name() -> str:
+    return "Qwen/Qwen3-VL-8B-Instruct"

tests/e2e/pull_request/one_card/spec_decode/test_dflash.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import pytest
+from transformers import AutoTokenizer
+from vllm import SamplingParams
+from vllm.config import CompilationConfig
+from vllm.v1.metrics.reader import Counter, Vector
+
+from tests.e2e.conftest import VllmRunner
+from tests.e2e.pull_request.one_card.spec_decode.utils import BASELINES, DFLASH, calculate_acceptance_per_pos
+
+
+@pytest.mark.parametrize("method", DFLASH.keys())
+@pytest.mark.parametrize("num_speculative_tokens", [8])
+def test_dflash_acceptance(
+    method: str,
+    num_speculative_tokens: int,
+):
+    main_model_name = DFLASH[method]["main"]
+    spec_model_name = DFLASH[method]["spec"]
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        main_model_name,
+        trust_remote_code=True,
+    )
+    sampling_params = SamplingParams(
+        temperature=0,
+        ignore_eos=False,
+        max_tokens=256,
+    )
+
+    prompts = [{"role": "user", "content": "Hello, your name is"}]
+    prompts = [
+        tokenizer.apply_chat_template(
+            [prompt],
+            tokenize=False,
+            add_generation_prompt=True,
+            enable_thinking=False,
+        )
+        for prompt in prompts
+    ]
+
+    speculative_config = {
+        "method": "dflash",
+        "model": spec_model_name,
+        "num_speculative_tokens": num_speculative_tokens,
+    }
+
+    compilation_config = CompilationConfig(cudagraph_mode="FULL_DECODE_ONLY", cudagraph_capture_sizes=[9, 18])
+
+    with VllmRunner(
+        main_model_name,
+        max_model_len=4096,
+        disable_log_stats=False,
+        tensor_parallel_size=1,
+        max_num_seqs=256,
+        distributed_executor_backend="mp",
+        gpu_memory_utilization=0.8,
+        speculative_config=speculative_config,
+        compilation_config=compilation_config,
+        enable_prefix_caching=False,
+    ) as llm:
+        outputs = llm.model.generate(prompts, sampling_params)
+        metrics = llm.model.get_metrics()
+
+    for output in outputs:
+        prompt = output.prompt
+        generated_text = output.outputs[0].text
+        output_tokens = output.outputs[0].token_ids
+        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+        print(f"Output tokens: {output_tokens}")
+
+    acceptance_per_pos = calculate_acceptance_per_pos(metrics, num_speculative_tokens, Counter, Vector)
+    golden = BASELINES[method]
+
+    match = all(abs(a - b) < 0.1 for a, b in zip(acceptance_per_pos, golden))
+    if not match:
+        print(f"acceptance_per_pos: {acceptance_per_pos}")
+        print(f"golden: {golden}")
+
+    assert match

tests/e2e/pull_request/one_card/spec_decode/test_draft_parallel.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import pytest
+from transformers import AutoTokenizer
+from vllm import SamplingParams
+from vllm.config import CompilationConfig
+from vllm.tokenizers.registry import resolve_tokenizer_args
+from vllm.v1.metrics.reader import Counter, Vector
+
+from tests.e2e.conftest import VllmRunner
+from tests.e2e.pull_request.one_card.spec_decode.utils import (
+    BASELINES,
+    DRAFT_PARALLEL_MODELS,
+    calculate_acceptance_per_pos,
+)
+
+
+@pytest.mark.parametrize("method", DRAFT_PARALLEL_MODELS.keys())
+@pytest.mark.parametrize("num_speculative_tokens", [8])
+@pytest.mark.parametrize("draft_tensor_parallel_size", [1])
+def test_parallel_drafting_acceptance(
+    method: str,
+    num_speculative_tokens: int,
+    draft_tensor_parallel_size: None | int,
+):
+    main_model_name = DRAFT_PARALLEL_MODELS[method]["main"]
+    spec_model_name = DRAFT_PARALLEL_MODELS[method]["spec"]
+
+    tokenizer_path = resolve_tokenizer_args(main_model_name)[1]
+    tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer_path,
+        trust_remote_code=True,
+    )
+    sampling_params = SamplingParams(
+        temperature=0,
+        ignore_eos=False,
+        max_tokens=256,
+    )
+
+    prompts = [{"role": "user", "content": "Hello, your name is"}]
+    prompts = [
+        tokenizer.apply_chat_template(
+            [prompt],
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        for prompt in prompts
+    ]
+
+    speculative_config = {
+        "method": "draft_model",
+        "model": spec_model_name,
+        "num_speculative_tokens": num_speculative_tokens,
+        "draft_tensor_parallel_size": draft_tensor_parallel_size,
+        "parallel_drafting": True,
+    }
+
+    compilation_config = CompilationConfig(
+        cudagraph_mode="PIECEWISE",
+        cudagraph_capture_sizes=[12],
+    )
+
+    with VllmRunner(
+        main_model_name,
+        max_model_len=4096,
+        disable_log_stats=False,
+        tensor_parallel_size=1,
+        max_num_seqs=256,
+        distributed_executor_backend="mp",
+        gpu_memory_utilization=0.8,
+        speculative_config=speculative_config,
+        compilation_config=compilation_config,
+        enable_prefix_caching=False,
+    ) as llm:
+        outputs = llm.model.generate(prompts, sampling_params)
+        metrics = llm.model.get_metrics()
+
+    for output in outputs:
+        prompt = output.prompt
+        generated_text = output.outputs[0].text
+        output_tokens = output.outputs[0].token_ids
+        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+        print(f"Output tokens: {output_tokens}")
+
+    acceptance_per_pos = calculate_acceptance_per_pos(metrics, num_speculative_tokens, Counter, Vector)
+    golden = BASELINES[method]
+
+    match = all(abs(a - b) < 0.1 for a, b in zip(acceptance_per_pos, golden))
+    if not match:
+        print(f"acceptance_per_pos: {acceptance_per_pos}")
+        print(f"golden: {golden}")
+
+    assert match

tests/e2e/pull_request/one_card/spec_decode/test_eagle.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from transformers import AutoTokenizer
+from vllm import SamplingParams
+from vllm.config import CompilationConfig
+from vllm.tokenizers.registry import resolve_tokenizer_args
+from vllm.v1.metrics.reader import Counter, Vector
+
+from tests.e2e.conftest import VllmRunner
+from tests.e2e.pull_request.one_card.spec_decode.utils import MODELS, calculate_acceptance_per_pos
+
+
+def test_qwen3_vl_eagle(
+    test_prompts: list[list[dict[str, Any]]],
+    sampling_config: SamplingParams,
+    vl_model_name: str,
+):
+    with VllmRunner(
+        vl_model_name,
+        max_model_len=1024,
+        cudagraph_capture_sizes=[1, 2, 4, 8],
+    ) as ref_llm:
+        ref_llm.model.chat(test_prompts, sampling_config)
+
+
+@pytest.mark.parametrize("method", MODELS.keys())
+@pytest.mark.parametrize("num_speculative_tokens", [3])
+@pytest.mark.parametrize("draft_tensor_parallel_size", [1])
+@pytest.mark.parametrize("disable_padded_drafter_batch", [False])
+@pytest.mark.parametrize("async_scheduling", [True])
+def test_qwen_eagle3_acceptance(
+    method: str,
+    num_speculative_tokens: int,
+    draft_tensor_parallel_size: None | int,
+    disable_padded_drafter_batch: bool,
+    async_scheduling: bool,
+):
+    main_model_name = MODELS[method]["main"]
+    spec_model_name = MODELS[method]["spec"]
+
+    tokenizer_path = resolve_tokenizer_args(main_model_name)[1]
+    tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer_path,
+        trust_remote_code=True,
+    )
+    sampling_params = SamplingParams(
+        temperature=0,
+        ignore_eos=False,
+        max_tokens=256,
+    )
+
+    prompts = [
+        {"role": "user", "content": "Hello, my name is"},
+        {"role": "user", "content": "The president of the United States is"},
+        {"role": "user", "content": "The capital of France is"},
+        {"role": "user", "content": "The future of AI is"},
+    ]
+    prompts = [
+        tokenizer.apply_chat_template(
+            [prompt],
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        for prompt in prompts
+    ]
+
+    speculative_config = {
+        "method": method,
+        "num_speculative_tokens": num_speculative_tokens,
+        "draft_tensor_parallel_size": draft_tensor_parallel_size,
+        "disable_padded_drafter_batch": disable_padded_drafter_batch,
+        "model": spec_model_name,
+    }
+
+    compilation_config = CompilationConfig(cudagraph_mode="FULL_DECODE_ONLY", cudagraph_capture_sizes=[12])
+
+    with VllmRunner(
+        main_model_name,
+        max_model_len=2048,
+        disable_log_stats=False,
+        tensor_parallel_size=1,
+        max_num_seqs=256,
+        distributed_executor_backend="mp",
+        gpu_memory_utilization=0.7,
+        speculative_config=speculative_config,
+        compilation_config=compilation_config,
+        async_scheduling=async_scheduling,
+    ) as llm:
+        outputs = llm.model.generate(prompts, sampling_params)
+        metrics = llm.model.get_metrics()
+
+    for output in outputs:
+        prompt = output.prompt
+        generated_text = output.outputs[0].text
+        output_tokens = output.outputs[0].token_ids
+        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+        print(f"Output tokens: {output_tokens}")
+
+    acceptance_per_pos = calculate_acceptance_per_pos(metrics, num_speculative_tokens, Counter, Vector)
+    golden = [0.68, 0.40, 0.18]
+
+    match = all(abs(a - b) < 0.08 for a, b in zip(acceptance_per_pos, golden))
+    if not match:
+        print(f"acceptance_per_pos: {acceptance_per_pos}")
+        print(f"golden: {golden}")
+
+    assert match

tests/e2e/pull_request/one_card/spec_decode/test_extract_hidden_states.py

@@ -0,0 +1,213 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# Copyright 2023 The vLLM team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""E2E tests for the extract_hidden_states speculative decoding method.
+
+Follows the pattern from vllm's test_extraction.py, validating that hidden
+states are correctly extracted and saved on the Ascend NPU. Parametrized over:
+
+* a dense model (Qwen3-8B) in both eager and ACL graph modes, using real
+  weights so outputs can be checked to be non-zero, and
+* a hybrid attention model (Qwen3.5-0.8B, GatedDeltaNet + full_attention)
+  loaded with dummy weights as a shape/round-trip smoke test. The hybrid case
+  mirrors upstream vLLM PR #39949.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from dataclasses import dataclass
+
+import pytest
+import torch
+from vllm import LLM, SamplingParams
+
+from vllm_ascend.utils import vllm_version_is
+
+if vllm_version_is("0.23.0"):
+    from safetensors import safe_open
+else:
+    from vllm.distributed.kv_transfer.kv_connector.v1 import example_hidden_states_connector
+
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+DENSE_MODEL = "Qwen/Qwen3-8B"
+# Qwen3-8B has 36 layers; pick a spread of layer indices to extract.
+DENSE_AUX_HIDDEN_STATE_LAYER_IDS = [2, 18, 34]
+
+HYBRID_MODEL = "Qwen/Qwen3.5-0.8B"
+HYBRID_AUX_HIDDEN_STATE_LAYER_IDS = [5, 11, 17]
+
+
+@dataclass
+class ExtractHiddenStatesCase:
+    model_name: str
+    aux_hidden_state_layer_ids: list[int]
+    prompts: list[str]
+    enforce_eager: bool
+    # ``None`` means "do not pass the argument", preserving each model's
+    # original defaults.
+    gpu_memory_utilization: float | None = None
+    max_num_seqs: int | None = None
+    max_model_len: int | None = None
+    load_format: str | None = None
+    # Dummy-weight runs can't assert non-zero outputs; real-weight runs can.
+    verify_nonzero: bool = True
+    # Hybrid smoke test additionally checks the token_ids round-trip.
+    verify_token_ids: bool = False
+
+
+CASES = [
+    pytest.param(
+        ExtractHiddenStatesCase(
+            model_name=DENSE_MODEL,
+            aux_hidden_state_layer_ids=DENSE_AUX_HIDDEN_STATE_LAYER_IDS,
+            prompts=[
+                "Hello, how are you?",
+                "What is machine learning?",
+                "Explain quantum computing briefly.",
+            ],
+            enforce_eager=True,
+            gpu_memory_utilization=0.8,
+            max_num_seqs=16,
+        ),
+        id="dense_eager",
+    ),
+    pytest.param(
+        ExtractHiddenStatesCase(
+            model_name=DENSE_MODEL,
+            aux_hidden_state_layer_ids=DENSE_AUX_HIDDEN_STATE_LAYER_IDS,
+            prompts=[
+                "Hello, how are you?",
+                "What is machine learning?",
+            ],
+            enforce_eager=False,
+            max_num_seqs=16,
+        ),
+        id="dense_aclgraph",
+    ),
+    pytest.param(
+        ExtractHiddenStatesCase(
+            model_name=HYBRID_MODEL,
+            aux_hidden_state_layer_ids=HYBRID_AUX_HIDDEN_STATE_LAYER_IDS,
+            prompts=[
+                "Hello world",
+                "Test prompt with several tokens",
+            ],
+            enforce_eager=True,
+            gpu_memory_utilization=0.4,
+            max_model_len=256,
+            load_format="dummy",
+            verify_nonzero=False,
+            verify_token_ids=True,
+        ),
+        id="hybrid_dummy_eager",
+    ),
+]
+
+
+@pytest.fixture
+def sampling_config():
+    return SamplingParams(temperature=0, max_tokens=1)
+
+
+def _verify_output(output, expected_shape, *, verify_nonzero, verify_token_ids):
+    """Verify a single hidden-states dump (matches vllm's check pattern)."""
+    assert output.kv_transfer_params is not None
+    hidden_states_path = output.kv_transfer_params.get("hidden_states_path")
+    assert hidden_states_path is not None
+
+    if vllm_version_is("0.23.0"):
+        assert os.path.exists(hidden_states_path)
+        with safe_open(hidden_states_path, "pt") as f:
+            tensor_names = f.keys()
+            assert "hidden_states" in tensor_names
+            hidden_states = f.get_tensor("hidden_states")
+            assert hidden_states.shape == expected_shape
+
+            if verify_token_ids:
+                token_ids = f.get_tensor("token_ids")
+                assert torch.equal(token_ids, torch.tensor(output.prompt_token_ids))
+
+            if verify_nonzero:
+                assert not torch.allclose(hidden_states, torch.zeros_like(hidden_states))
+    else:
+        obj = example_hidden_states_connector.load_hidden_states(hidden_states_path)
+        example_hidden_states_connector.cleanup_hidden_states(hidden_states_path)
+        hidden_states = obj["hidden_states"]
+        assert hidden_states.shape == expected_shape
+
+        if verify_token_ids:
+            token_ids = obj["token_ids"]
+            assert torch.equal(token_ids, torch.tensor(output.prompt_token_ids))
+
+        if verify_nonzero:
+            assert not torch.allclose(hidden_states, torch.zeros_like(hidden_states))
+
+
+@pytest.mark.parametrize("case", CASES)
+def test_extract_hidden_states(case: ExtractHiddenStatesCase, sampling_config):
+    """Extract hidden states from the target model and validate the dump."""
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        llm_kwargs = dict(
+            model=case.model_name,
+            tensor_parallel_size=1,
+            enforce_eager=case.enforce_eager,
+            enable_chunked_prefill=False,
+            speculative_config={
+                "method": "extract_hidden_states",
+                "num_speculative_tokens": 1,
+                "draft_model_config": {
+                    "hf_config": {
+                        "eagle_aux_hidden_state_layer_ids": case.aux_hidden_state_layer_ids,
+                    }
+                },
+            },
+            kv_transfer_config={
+                "kv_connector": "ExampleHiddenStatesConnector",
+                "kv_role": "kv_producer",
+                "kv_connector_extra_config": {
+                    "shared_storage_path": tmpdirname,
+                },
+            },
+        )
+        if case.gpu_memory_utilization is not None:
+            llm_kwargs["gpu_memory_utilization"] = case.gpu_memory_utilization
+        if case.max_num_seqs is not None:
+            llm_kwargs["max_num_seqs"] = case.max_num_seqs
+        if case.max_model_len is not None:
+            llm_kwargs["max_model_len"] = case.max_model_len
+        if case.load_format is not None:
+            llm_kwargs["load_format"] = case.load_format
+
+        llm = LLM(**llm_kwargs)
+
+        outputs = llm.generate(case.prompts, sampling_config)
+        hidden_size = llm.llm_engine.model_config.get_hidden_size()
+        num_layers = len(case.aux_hidden_state_layer_ids)
+
+        assert len(outputs) == len(case.prompts)
+
+        for output in outputs:
+            num_tokens = len(output.prompt_token_ids)
+            expected_shape = (num_tokens, num_layers, hidden_size)
+            _verify_output(
+                output,
+                expected_shape,
+                verify_nonzero=case.verify_nonzero,
+                verify_token_ids=case.verify_token_ids,
+            )

tests/e2e/pull_request/one_card/spec_decode/test_mtp_eagle_correctness.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# Copyright 2025 The vLLM team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This file is a part of the vllm-ascend project.
+# Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
+#
+"""Compare the short outputs of HF and vLLM when using greedy sampling."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+from vllm import SamplingParams
+from vllm.config import CompilationConfig
+
+from tests.e2e.conftest import VllmRunner, cleanup_dist_env_and_memory
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+MODELS = ["wemaster/deepseek_mtp_main_random_bf16"]
+
+
+@pytest.mark.parametrize("model_name", MODELS)
+@pytest.mark.parametrize("num_speculative_tokens", [3])
+@pytest.mark.parametrize("cudagraph_mode", ["FULL_DECODE_ONLY"])
+@pytest.mark.parametrize("disable_padded_drafter_batch", [False])
+def test_deepseek_mtp(
+    model_name: str, num_speculative_tokens: int, cudagraph_mode: str, disable_padded_drafter_batch: bool
+):
+    example_prompts = [
+        "Hello, my name is",
+        "The president of the United States is",
+        "The capital of France is",
+        "The future of AI is",
+    ]
+    """
+    Compare the outputs of a original LLM and a speculative LLM
+    should be the same when using mtp speculative decoding.
+    """
+    with VllmRunner(
+        model_name,
+        tensor_parallel_size=1,
+        max_num_seqs=256,
+        gpu_memory_utilization=0.7,
+        distributed_executor_backend="mp",
+        enable_expert_parallel=True,
+        speculative_config={
+            "method": "mtp",
+            "num_speculative_tokens": num_speculative_tokens,
+            "disable_padded_drafter_batch": disable_padded_drafter_batch,
+        },
+        max_model_len=2000,
+        compilation_config=CompilationConfig(
+            cudagraph_mode=cudagraph_mode,
+            cudagraph_capture_sizes=[20],
+        ),
+    ) as spec_llm:
+        sampling_config = SamplingParams(temperature=0, max_tokens=256, ignore_eos=False)
+        spec_llm.generate(example_prompts, sampling_config)
+
+    cleanup_dist_env_and_memory()
+    del spec_llm

tests/e2e/pull_request/one_card/spec_decode/test_ngram.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing import Any
+
+from vllm import SamplingParams
+
+from tests.e2e.conftest import VllmRunner
+
+
+def test_ngram(
+    test_prompts: list[list[dict[str, Any]]],
+    sampling_config: SamplingParams,
+    model_name: str,
+):
+    with VllmRunner(
+        model_name,
+        speculative_config={
+            "method": "ngram",
+            "prompt_lookup_max": 5,
+            "prompt_lookup_min": 3,
+            "num_speculative_tokens": 3,
+        },
+        max_model_len=1024,
+        cudagraph_capture_sizes=[1, 2, 4, 8],
+    ) as runner:
+        runner.model.chat(test_prompts, sampling_config)

tests/e2e/pull_request/one_card/spec_decode/test_ngram_npu.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from vllm import SamplingParams
+from vllm.config import CompilationConfig
+from vllm.v1.metrics.reader import Counter, Vector
+
+from tests.e2e.conftest import VllmRunner
+from tests.e2e.pull_request.one_card.spec_decode.utils import calculate_acceptance_per_pos
+
+
+@pytest.mark.parametrize("num_speculative_tokens", [3])
+def test_ngram_npu_async_acceptance(
+    test_prompts: list[list[dict[str, Any]]],
+    num_speculative_tokens: int,
+    model_name: str,
+):
+    sampling_params = SamplingParams(
+        temperature=0,
+        ignore_eos=False,
+        max_tokens=256,
+    )
+
+    speculative_config = {
+        "method": "ngram_gpu",
+        "prompt_lookup_max": 2,
+        "prompt_lookup_min": 2,
+        "num_speculative_tokens": num_speculative_tokens,
+    }
+
+    compilation_config = CompilationConfig(
+        cudagraph_mode="PIECEWISE",
+        cudagraph_capture_sizes=[12],
+    )
+
+    with VllmRunner(
+        model_name,
+        max_model_len=2048,
+        disable_log_stats=False,
+        tensor_parallel_size=1,
+        max_num_seqs=256,
+        distributed_executor_backend="mp",
+        gpu_memory_utilization=0.7,
+        speculative_config=speculative_config,
+        compilation_config=compilation_config,
+        async_scheduling=True,
+    ) as llm:
+        outputs = llm.model.chat(test_prompts, sampling_params)
+        metrics = llm.model.get_metrics()
+
+    for output in outputs:
+        prompt = output.prompt
+        generated_text = output.outputs[0].text
+        output_tokens = output.outputs[0].token_ids
+        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+        print(f"Output tokens: {output_tokens}")
+
+    acceptance_per_pos = calculate_acceptance_per_pos(metrics, num_speculative_tokens, Counter, Vector)
+    golden = [0.50, 0.30, 0.20]
+
+    match = all(abs(a - b) < 1.0 for a, b in zip(acceptance_per_pos, golden))
+    if not match:
+        print(f"acceptance_per_pos: {acceptance_per_pos}")
+        print(f"golden: {golden}")
+
+    assert match

tests/e2e/pull_request/one_card/spec_decode/test_suffix.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from typing import Any
+
+from vllm import SamplingParams
+
+from tests.e2e.conftest import VllmRunner
+
+
+def test_suffix_acceptance(
+    test_prompts: list[list[dict[str, Any]]],
+    sampling_config: SamplingParams,
+    model_name: str,
+):
+    num_draft = []
+    num_accept = []
+    with VllmRunner(
+        model_name,
+        speculative_config={
+            "method": "suffix",
+            "suffix_decoding_max_spec_factor": 2.0,
+            "suffix_decoding_max_cached_requests": 1000,
+            "num_speculative_tokens": 10,
+        },
+        max_model_len=1024,
+        max_cudagraph_capture_size=16,
+        disable_log_stats=False,
+    ) as runner:
+        for i in range(10):
+            runner.model.chat(test_prompts[i], sampling_config)
+            metrics = runner.model.get_metrics()
+            for metric in metrics:
+                print(metric)
+                if metric.name == "vllm:spec_decode_num_draft_tokens":
+                    num_draft.append(metric.value)
+                if metric.name == "vllm:spec_decode_num_accepted_tokens":
+                    num_accept.append(metric.value)
+
+    first_accept_tokens = num_accept[0]
+    first_draft_tokens = num_draft[0]
+    first_accept_rate = first_accept_tokens / first_draft_tokens
+
+    last_accept_tokens = num_accept[-1] - num_accept[-2]
+    last_draft_tokens = num_draft[-1] - num_draft[-2]
+    last_accept_rate = last_accept_tokens / last_draft_tokens
+
+    assert first_accept_tokens < last_accept_tokens
+    assert first_accept_rate < last_accept_rate
+    assert last_accept_rate > 0.60

tests/e2e/pull_request/one_card/spec_decode/utils.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import os
+from typing import Any
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+MODELS = {
+    "eagle3": {
+        "main": "Qwen/Qwen3-8B",
+        "spec": "RedHatAI/Qwen3-8B-speculator.eagle3",
+    },
+}
+
+DRAFT_PARALLEL_MODELS = {
+    "draft_parallel": {
+        "main": "LLM-Research/Meta-Llama-3.1-8B-Instruct",
+        "spec": "amd/PARD-Llama-3.2-1B",
+    },
+}
+
+DFLASH = {
+    "dflash": {
+        "main": "Qwen/Qwen3-8B",
+        "spec": "z-lab/Qwen3-8B-DFlash-b16",
+    }
+}
+
+BASELINES = {
+    "eagle": [0.74, 0.44, 0.29],
+    "eagle3": [0.68, 0.40, 0.18],
+    "draft_parallel": [0.83, 0.50, 0.33, 0.17, 0.17, 0.17, 0.17, 0.00],
+    "dflash": [0.60, 0.50, 0.30, 0.20, 0.20, 0.10, 0.00, 0.00],
+}
+
+
+def eagle_model_name():
+    return "vllm-ascend/EAGLE-LLaMA3.1-Instruct-8B"
+
+
+def eagle3_model_name():
+    return "vllm-ascend/EAGLE3-LLaMA3.1-Instruct-8B"
+
+
+def vl_eagle3_model_name():
+    return "MNN/Qwen3-VL-8B-Instruct-Eagle3"
+
+
+def calculate_acceptance_per_pos(metrics: list[Any], num_speculative_tokens: int, counter_type: Any, vector_type: Any):
+    num_drafts = 0
+    num_accepted_tokens_per_pos = [0] * num_speculative_tokens
+    for metric in metrics:
+        if metric.name == "vllm:spec_decode_num_drafts":
+            assert isinstance(metric, counter_type)
+            num_drafts += metric.value
+        elif metric.name == "vllm:spec_decode_num_accepted_tokens_per_pos":
+            assert isinstance(metric, vector_type)
+            for pos in range(len(metric.values)):
+                num_accepted_tokens_per_pos[pos] += metric.values[pos]
+
+    return [num_accepted_tokens / num_drafts for num_accepted_tokens in num_accepted_tokens_per_pos]