Enable overlap by default (#2067)

2024-11-19 22:07:58 -08:00
parent 699384cb01
commit 7d671e4ad2
17 changed files with 92 additions and 75 deletions
--- a/python/sglang/srt/managers/schedule_batch.py
+++ b/python/sglang/srt/managers/schedule_batch.py
@@ -899,10 +899,7 @@ class ScheduleBatch:

        self.input_ids = self.output_ids
        self.output_ids = None
-        if self.sampling_info.penalizer_orchestrator:
-            self.sampling_info.penalizer_orchestrator.cumulate_output_tokens(
-                self.input_ids
-            )
+        self.sampling_info.penalizer_orchestrator.cumulate_output_tokens(self.input_ids)

        # Alloc mem
        bs = len(self.reqs)