init v0.23.0

Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>

vllm_ascend/spec_decode/ngram_proposer.py

@@ -1,65 +1,102 @@
 import torch
-from vllm.v1.spec_decode.ngram_proposer import \
-    NgramProposer as VllmNgramProposer
+from vllm.v1.spec_decode.ngram_proposer import NgramProposer
 
-from vllm_ascend.spec_decode.interface import Proposer, SpecDcodeType
+from vllm_ascend.utils import vllm_version_is
 
 
-class NgramProposer(VllmNgramProposer, Proposer):
-
-    def __init__(self, vllm_config, device, runner):
-        super().__init__(vllm_config)
-        self.name = SpecDcodeType.NGRAM
-        self.device = device
+class AscendNgramProposer(NgramProposer):
+    def __init__(self, vllm_config, runner):
         self.runner = runner
+        super().__init__(vllm_config)
 
     def load_model(self, *args, **kwargs):
         # No model to load.
         pass
 
     @torch.inference_mode()
-    def dummy_run(self,
-                  num_tokens,
-                  with_prefill=None,
-                  skip_attn=None,
-                  num_reqs=None,
-                  num_tokens_across_dp=None):
+    def dummy_run(
+        self,
+        num_tokens,
+        with_prefill=None,
+        in_graph_capturing=None,
+        num_reqs=None,
+        num_tokens_across_dp=None,
+        aclgraph_runtime_mode=None,
+        batch_descriptor=None,
+        dummy_compute_logits=lambda hidden_states: None,
+        is_profile=False,
+    ):
         pass
 
-    def generate_token_ids(self,
-                           valid_sampled_token_ids,
-                           sampling_metadata=None,
-                           scheduler_output=None,
-                           spec_decode_metadata=None,
-                           positions=None,
-                           num_scheduled_tokens=None,
-                           hidden_states=None,
-                           attn_metadata=None,
-                           aux_hidden_states=None) -> list[list[int]]:
-        # TODO(woosuk): Optimize.
-        draft_token_ids: list[list[int]] = []
-        for i, sampled_ids in enumerate(valid_sampled_token_ids):
-            num_sampled_ids = len(sampled_ids)
-            if not num_sampled_ids:
-                # Skip speculative decoding.
-                draft_token_ids.append([])
-                continue
+    if vllm_version_is("0.23.0"):
 
-            # Skip requests that require top-p, top-k, etc.
-            req_id = self.runner.input_batch.req_ids[i]
-            if req_id in self.runner.input_batch.spec_decode_unsupported_reqs:
-                draft_token_ids.append([])
-                continue
+        def propose(
+            self,
+            sampled_token_ids: list[list[int]],
+            num_tokens_no_spec=None,
+            token_ids_cpu=None,
+            slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None,
+        ) -> list[list[int]]:
+            input_batch = self.runner.input_batch
+            valid_ngram_requests = []
+            for i, sampled_ids in enumerate(sampled_token_ids):
+                num_sampled_ids = len(sampled_ids)
+                if not num_sampled_ids:
+                    continue
 
-            # Add sampled_token_ids to token_ids_cpu.
-            start_idx = self.runner.input_batch.num_tokens_no_spec[i]
-            end_idx = start_idx + num_sampled_ids
-            self.runner.input_batch.token_ids_cpu[
-                i, start_idx:end_idx] = sampled_ids
-            drafter_output = self.propose(
-                self.runner.input_batch.token_ids_cpu[i, :end_idx])
-            if drafter_output is None or len(drafter_output) == 0:
-                draft_token_ids.append([])
-            else:
-                draft_token_ids.append(drafter_output.tolist())
-        return draft_token_ids
+                req_id = input_batch.req_ids[i]
+                if req_id in input_batch.spec_decode_unsupported_reqs:
+                    continue
+
+                num_tokens = input_batch.num_tokens_no_spec[i]
+                if num_tokens >= input_batch.max_model_len:
+                    # Skip requests that have already reached the max model length.
+                    continue
+
+                start_idx = input_batch.num_tokens_no_spec[i]
+                end_idx = start_idx + num_sampled_ids
+                input_batch.token_ids_cpu[i, start_idx:end_idx] = sampled_ids
+
+                valid_ngram_requests.append(i)
+
+            return self.batch_propose(
+                len(sampled_token_ids),
+                valid_ngram_requests,
+                input_batch.num_tokens_no_spec,
+                input_batch.token_ids_cpu,
+            )
+
+    else:
+
+        def propose(  # type: ignore[misc]
+            self,
+            num_speculative_tokens: int,
+            sampled_token_ids: list[list[int]],
+            num_tokens_no_spec=None,
+            token_ids_cpu=None,
+            slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None,
+        ) -> list[list[int]]:
+            assert num_speculative_tokens <= self.k
+            assert num_tokens_no_spec is not None
+            assert token_ids_cpu is not None
+
+            valid_ngram_requests = []
+            for i, sampled_ids in enumerate(sampled_token_ids):
+                num_sampled_ids = len(sampled_ids)
+                if not num_sampled_ids:
+                    continue
+
+                num_tokens = num_tokens_no_spec[i]
+                if num_tokens >= self.max_model_len:
+                    # Skip requests that have already reached the max model length.
+                    continue
+
+                valid_ngram_requests.append(i)
+
+            return self.batch_propose(
+                len(sampled_token_ids),
+                valid_ngram_requests,
+                num_tokens_no_spec,
+                token_ids_cpu,
+                num_speculative_tokens,
+            )