support more optioin about usage in stream mode (#985)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
2024-08-08 17:41:57 +08:00
parent 1ac304eeb4
commit 3a79613c28
4 changed files with 96 additions and 28 deletions
--- a/python/sglang/srt/managers/schedule_batch.py
+++ b/python/sglang/srt/managers/schedule_batch.py
@@ -217,7 +217,9 @@ class Req:
            return

        if len(self.output_ids) >= self.sampling_params.max_new_tokens:
-            self.finished_reason = FINISH_LENGTH(len(self.output_ids))
+            self.finished_reason = FINISH_LENGTH(
+                length=self.sampling_params.max_new_tokens
+            )
            return

        if (