EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[EAGLE] Fix some boundary situation when retract reqs and req's max token = 1 (#2939)

Browse Source 
Co-authored-by: josephyou <josephyou@tencent.com>
This commit is contained in:
996_icu
2025-01-21 09:46:43 +08:00
committed by GitHub
parent 60b2a44a80
commit b730aa6b9e
2 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/srt/managers/schedule_batch.py
View File

@@ -1112,6 +1112,8 @@ class ScheduleBatch:
self.has_grammar = any(req.grammar for req in self.reqs) self.has_grammar = any(req.grammar for req in self.reqs)
self.sampling_info.filter_batch(keep_indices, new_indices) self.sampling_info.filter_batch(keep_indices, new_indices)
if self.spec_info:
self.spec_info.filter_batch(new_indices)
def merge_batch(self, other: "ScheduleBatch"): def merge_batch(self, other: "ScheduleBatch"):
# Penalizer orchestrator must be merged before Batch.reqs is merged. This is because # Penalizer orchestrator must be merged before Batch.reqs is merged. This is because

8
python/sglang/srt/speculative/eagle_utils.py
View File

@@ -228,6 +228,14 @@ class EAGLEDraftInput(SpecInfo):
assert len(batch.extend_lens) == 1 assert len(batch.extend_lens) == 1
batch.input_ids = torch.concat((batch.input_ids[1:], self.verified_id)) batch.input_ids = torch.concat((batch.input_ids[1:], self.verified_id))
def filter_batch(
self,
new_indices: torch.Tensor,
):
self.sample_output = self.sample_output[: len(new_indices)]
self.hidden_states = self.hidden_states[: len(new_indices)]
self.verified_id = self.verified_id[: len(new_indices)]
def prepare_for_decode(self, batch: ScheduleBatch): def prepare_for_decode(self, batch: ScheduleBatch):
prob = self.sample_output # shape: (b * top_k, vocab) or (b, vocab) prob = self.sample_output # shape: (b * top_k, vocab) or (b, vocab)
top = torch.topk(prob, self.topk, dim=-1) top = torch.topk(prob, self.topk, dim=-1)