[Fix] Avoid calling fill_vocab_mask for terminated requests (#2175)

2024-11-25 04:31:25 -05:00
parent 55842eb81a
commit 538fa0ae13
1 changed files with 4 additions and 1 deletions
--- a/python/sglang/srt/sampling/sampling_batch_info.py
+++ b/python/sglang/srt/sampling/sampling_batch_info.py
@@ -170,7 +170,10 @@ class SamplingBatchInfo:

        for i, grammar in enumerate(self.grammars):
            if grammar is not None:
-                grammar.fill_vocab_mask(self.vocab_mask, i)
+                try:
+                    grammar.fill_vocab_mask(self.vocab_mask, i)
+                except RuntimeError:
+                    continue

    def filter_batch(self, unfinished_indices: List[int], new_indices: torch.Tensor):
        self.penalizer_orchestrator.filter(unfinished_indices, new_indices)