EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Fix] Avoid calling fill_vocab_mask for terminated requests (#2175)

Browse Source
This commit is contained in:
Yixin Dong
2024-11-25 04:31:25 -05:00
committed by GitHub
parent 55842eb81a
commit 538fa0ae13
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/sglang/srt/sampling/sampling_batch_info.py
View File

@@ -170,7 +170,10 @@ class SamplingBatchInfo:
for i, grammar in enumerate(self.grammars): for i, grammar in enumerate(self.grammars):
if grammar is not None: if grammar is not None:
try:
grammar.fill_vocab_mask(self.vocab_mask, i) grammar.fill_vocab_mask(self.vocab_mask, i)
except RuntimeError:
continue
def filter_batch(self, unfinished_indices: List[int], new_indices: torch.Tensor): def filter_batch(self, unfinished_indices: List[int], new_indices: torch.Tensor):
self.penalizer_orchestrator.filter(unfinished_indices, new_indices) self.penalizer_orchestrator.filter(unfinished_indices, new_indices)