[Refactor][EAGLE] 1/N delete __init__ in mtp_proposer (#5176)
### What this PR does / why we need it?
This PR aims to refactor eagle-related modules in vllm-ascend.
This is the starting PR of eagle refactoring. Provided with vllm-eagle,
ascend-eagle and ascend-mtp, we first let ascend-mtp inherit from
ascend-eagle and let ascend-eagle inherit from vllm-eagle. As a
initialization, we just delete `__init__` in mtp_proposer and simplify
the corresponding logic in eagle_proposer.
Based on "vllm-eagle <----- ascend-eagle <----- ascend-mtp", our target
is to gradually delete ascend-mtp and enable ascend-eagle to converge to
vllm-eagle. So the main workspace is eagle_proposer. In this way, we
hope that contributors can concurrently refactor eagle.
Incoming changes:
1. delete common methods in vllm-eagle & ascend-eagle & ascend-mtp
2. delete `load_model` in mtp_proposer
3. delete `dummy_run` and `propose` in mtp_proposer
4. ......
RFC: #5467
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
by ci
- vLLM version: v0.12.0
- vLLM main:
ad32e3e19c
---------
Signed-off-by: Zetong Li <slippersss@126.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import torch
|
||||
from vllm.config import CacheConfig, CompilationMode, CUDAGraphMode, VllmConfig
|
||||
|
||||
from tests.ut.base import TestBase
|
||||
from vllm_ascend.ascend_config import init_ascend_config
|
||||
from vllm_ascend.spec_decode.eagle_proposer import EagleProposer
|
||||
from vllm_ascend.spec_decode.interface import SpecDcodeType
|
||||
|
||||
@@ -25,13 +26,24 @@ class TestEagleProposerInitialization(TestBase):
|
||||
self.vllm_config.scheduler_config.max_num_seqs = 32
|
||||
self.vllm_config.model_config.dtype = torch.float16
|
||||
self.vllm_config.model_config.max_model_len = 2048
|
||||
self.vllm_config.model_config.uses_mrope = False
|
||||
self.vllm_config.speculative_config.num_speculative_tokens = 2
|
||||
self.vllm_config.speculative_config.speculative_token_tree = str([
|
||||
(i + 1) * (0, ) for i in range(2)
|
||||
])
|
||||
self.vllm_config.additional_config = None
|
||||
|
||||
self.mock_cpugpubuffer = patch(
|
||||
"vllm_ascend.spec_decode.eagle_proposer.CpuGpuBuffer")
|
||||
"vllm.v1.spec_decode.eagle.CpuGpuBuffer")
|
||||
self.mock_cpugpubuffer.start()
|
||||
self.mock_supports_multimodal_inputs = patch(
|
||||
"vllm.multimodal.registry.MultiModalRegistry.supports_multimodal_inputs"
|
||||
)
|
||||
self.mock_supports_multimodal_inputs.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.mock_cpugpubuffer.stop()
|
||||
self.mock_supports_multimodal_inputs.stop()
|
||||
|
||||
def test_initialization_eagle_graph(self):
|
||||
self.vllm_config.speculative_config.method = "eagle"
|
||||
@@ -40,12 +52,12 @@ class TestEagleProposerInitialization(TestBase):
|
||||
self.vllm_config.model_config.enforce_eager = False
|
||||
self.vllm_config.speculative_config.enforce_eager = False
|
||||
self.vllm_config.scheduler_config.async_scheduling = False
|
||||
init_ascend_config(self.vllm_config)
|
||||
|
||||
proposer = EagleProposer(vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
runner=self.runner)
|
||||
|
||||
self.assertEqual(proposer.name, SpecDcodeType.EAGLE)
|
||||
self.assertEqual(proposer.block_size, 16)
|
||||
self.assertEqual(proposer.hidden_size, 4096)
|
||||
self.assertTrue(proposer.use_cuda_graph)
|
||||
@@ -60,12 +72,12 @@ class TestEagleProposerInitialization(TestBase):
|
||||
self.vllm_config.speculative_config.draft_model_config.get_hidden_size.return_value = 2048
|
||||
self.vllm_config.compilation_config.mode = CompilationMode.NONE
|
||||
self.vllm_config.model_config.enforce_eager = True
|
||||
init_ascend_config(self.vllm_config)
|
||||
|
||||
proposer = EagleProposer(vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
runner=self.runner)
|
||||
|
||||
self.assertEqual(proposer.name, SpecDcodeType.EAGLE3)
|
||||
self.assertEqual(proposer.hidden_size, 2048)
|
||||
self.assertFalse(proposer.use_cuda_graph)
|
||||
self.assertEqual(proposer.hidden_states.shape, (1024, 2048))
|
||||
@@ -77,12 +89,12 @@ class TestEagleProposerInitialization(TestBase):
|
||||
self.vllm_config.model_config.enforce_eager = False
|
||||
self.vllm_config.speculative_config.enforce_eager = False
|
||||
self.vllm_config.scheduler_config.async_scheduling = True
|
||||
init_ascend_config(self.vllm_config)
|
||||
|
||||
proposer = EagleProposer(vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
runner=self.runner)
|
||||
|
||||
self.assertEqual(proposer.name, SpecDcodeType.EAGLE3)
|
||||
self.assertEqual(proposer.hidden_size, 2048)
|
||||
self.assertFalse(proposer.use_cuda_graph)
|
||||
self.assertEqual(proposer.hidden_states.shape, (1024, 2048))
|
||||
@@ -102,16 +114,28 @@ class TestEagleProposerLoadModel(TestBase):
|
||||
self.vllm_config.scheduler_config.max_num_seqs = 32
|
||||
self.vllm_config.model_config.dtype = torch.float16
|
||||
self.vllm_config.model_config.max_model_len = 2048
|
||||
self.vllm_config.model_config.uses_mrope = False
|
||||
self.vllm_config.speculative_config.num_speculative_tokens = 2
|
||||
self.vllm_config.speculative_config.speculative_token_tree = str([
|
||||
(i + 1) * (0, ) for i in range(2)
|
||||
])
|
||||
self.vllm_config.additional_config = None
|
||||
init_ascend_config(self.vllm_config)
|
||||
|
||||
self.mock_cpugpubuffer = patch(
|
||||
"vllm_ascend.spec_decode.eagle_proposer.CpuGpuBuffer")
|
||||
"vllm.v1.spec_decode.eagle.CpuGpuBuffer")
|
||||
self.mock_cpugpubuffer.start()
|
||||
self.mock_supports_multimodal_inputs = patch(
|
||||
"vllm.multimodal.registry.MultiModalRegistry.supports_multimodal_inputs"
|
||||
)
|
||||
self.mock_supports_multimodal_inputs.start()
|
||||
self.proposer = EagleProposer(vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
runner=self.runner)
|
||||
|
||||
def tearDown(self):
|
||||
self.mock_cpugpubuffer.stop()
|
||||
self.mock_supports_multimodal_inputs.stop()
|
||||
|
||||
@patch(
|
||||
"vllm_ascend.spec_decode.eagle_proposer.get_layers_from_vllm_config")
|
||||
@@ -204,10 +228,20 @@ class TestEagleProposerDummyRun(TestBase):
|
||||
self.vllm_config.scheduler_config.max_num_seqs = 32
|
||||
self.vllm_config.model_config.dtype = torch.float16
|
||||
self.vllm_config.model_config.max_model_len = 2048
|
||||
self.vllm_config.model_config.uses_mrope = False
|
||||
self.vllm_config.speculative_config.speculative_token_tree = str([
|
||||
(i + 1) * (0, ) for i in range(4)
|
||||
])
|
||||
self.vllm_config.additional_config = None
|
||||
init_ascend_config(self.vllm_config)
|
||||
|
||||
self.mock_cpugpubuffer = patch(
|
||||
"vllm_ascend.spec_decode.eagle_proposer.CpuGpuBuffer")
|
||||
"vllm.v1.spec_decode.eagle.CpuGpuBuffer")
|
||||
self.mock_cpugpubuffer.start()
|
||||
self.mock_supports_multimodal_inputs = patch(
|
||||
"vllm.multimodal.registry.MultiModalRegistry.supports_multimodal_inputs"
|
||||
)
|
||||
self.mock_supports_multimodal_inputs.start()
|
||||
self.proposer = EagleProposer(vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
runner=self.runner)
|
||||
@@ -216,6 +250,7 @@ class TestEagleProposerDummyRun(TestBase):
|
||||
|
||||
def tearDown(self):
|
||||
self.mock_cpugpubuffer.stop()
|
||||
self.mock_supports_multimodal_inputs.stop()
|
||||
|
||||
@patch("vllm_ascend.spec_decode.eagle_proposer.get_forward_context")
|
||||
@patch("vllm_ascend.spec_decode.eagle_proposer.set_ascend_forward_context")
|
||||
@@ -287,16 +322,28 @@ class TestEagleProposerGenerateTokenIds(TestBase):
|
||||
1: MagicMock(get_token_id=lambda x: 101),
|
||||
2: MagicMock(get_token_id=lambda x: 102),
|
||||
}
|
||||
self.runner.pcp_size = 1
|
||||
|
||||
self.vllm_config.cache_config.block_size = 16
|
||||
self.vllm_config.scheduler_config.max_num_batched_tokens = 1024
|
||||
self.vllm_config.scheduler_config.max_num_seqs = 32
|
||||
self.vllm_config.model_config.dtype = torch.float16
|
||||
self.vllm_config.model_config.max_model_len = 2048
|
||||
self.vllm_config.model_config.uses_mrope = False
|
||||
self.vllm_config.speculative_config.num_speculative_tokens = 2
|
||||
self.vllm_config.speculative_config.speculative_token_tree = str([
|
||||
(i + 1) * (0, ) for i in range(2)
|
||||
])
|
||||
self.vllm_config.additional_config = None
|
||||
init_ascend_config(self.vllm_config)
|
||||
|
||||
self.mock_cpugpubuffer = patch(
|
||||
"vllm_ascend.spec_decode.eagle_proposer.CpuGpuBuffer")
|
||||
"vllm.v1.spec_decode.eagle.CpuGpuBuffer")
|
||||
self.mock_cpugpubuffer.start()
|
||||
self.mock_supports_multimodal_inputs = patch(
|
||||
"vllm.multimodal.registry.MultiModalRegistry.supports_multimodal_inputs"
|
||||
)
|
||||
self.mock_supports_multimodal_inputs.start()
|
||||
self.proposer = EagleProposer(vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
runner=self.runner)
|
||||
@@ -306,6 +353,7 @@ class TestEagleProposerGenerateTokenIds(TestBase):
|
||||
|
||||
def tearDown(self):
|
||||
self.mock_cpugpubuffer.stop()
|
||||
self.mock_supports_multimodal_inputs.stop()
|
||||
|
||||
# TODO: This is equivalent to disable_padded_drafter_batch=True.
|
||||
# We need to add some cases about disable_padded_drafter_batch=False in future.
|
||||
@@ -355,16 +403,28 @@ class TestEagleProposerHelperMethods(TestBase):
|
||||
self.vllm_config.scheduler_config.max_num_seqs = 32
|
||||
self.vllm_config.model_config.dtype = torch.float16
|
||||
self.vllm_config.model_config.max_model_len = 2048
|
||||
self.vllm_config.model_config.uses_mrope = False
|
||||
self.vllm_config.speculative_config.num_speculative_tokens = 2
|
||||
self.vllm_config.speculative_config.speculative_token_tree = str([
|
||||
(i + 1) * (0, ) for i in range(2)
|
||||
])
|
||||
self.vllm_config.additional_config = None
|
||||
init_ascend_config(self.vllm_config)
|
||||
|
||||
self.mock_cpugpubuffer = patch(
|
||||
"vllm_ascend.spec_decode.eagle_proposer.CpuGpuBuffer")
|
||||
"vllm.v1.spec_decode.eagle.CpuGpuBuffer")
|
||||
self.mock_cpugpubuffer.start()
|
||||
self.mock_supports_multimodal_inputs = patch(
|
||||
"vllm.multimodal.registry.MultiModalRegistry.supports_multimodal_inputs"
|
||||
)
|
||||
self.mock_supports_multimodal_inputs.start()
|
||||
self.proposer = EagleProposer(vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
runner=self.runner)
|
||||
|
||||
def tearDown(self):
|
||||
self.mock_cpugpubuffer.stop()
|
||||
self.mock_supports_multimodal_inputs.stop()
|
||||
|
||||
# TODO: This is equivalent to disable_padded_drafter_batch=True.
|
||||
# We need to add a test_prepare_inputs_padded in future.
|
||||
|
||||
Reference in New Issue
Block a user