Fix prefill oom (#1743)

2024-10-21 03:54:35 -07:00
parent 09603c6dc9
commit efb099cdee
1 changed files with 1 additions and 1 deletions
--- a/python/sglang/srt/managers/scheduler.py
+++ b/python/sglang/srt/managers/scheduler.py
@@ -427,7 +427,7 @@ class Scheduler:
                if req.sampling_params.max_new_tokens is not None
                else 1 << 30
            ),
-            self.max_req_input_len - 1 - len(req.origin_input_ids),
+            self.max_req_input_len - len(req.origin_input_ids),
        )

        self.waiting_queue.append(req)