Add a new event loop (#1677)

2024-10-16 01:33:20 -07:00
parent a5114b6f91
commit 9116b2896f
9 changed files with 161 additions and 25 deletions
--- a/python/sglang/srt/managers/schedule_batch.py
+++ b/python/sglang/srt/managers/schedule_batch.py
@@ -736,6 +736,10 @@ class ScheduleBatch:
        self.input_ids = self.output_ids
        self.seq_lens.add_(1)
        self.output_ids = None
+        if self.sampling_info.penalizer_orchestrator:
+            self.sampling_info.penalizer_orchestrator.cumulate_output_tokens(
+                self.input_ids
+            )

        # Alloc mem
        bs = len(self.reqs)