Fix chunked prefill condition (#1594)

2024-10-07 14:34:14 +08:00
parent 6a5b352aaf
commit 68f8b60d22
2 changed files with 15 additions and 1 deletions
--- a/python/sglang/srt/managers/schedule_policy.py
+++ b/python/sglang/srt/managers/schedule_policy.py
@@ -249,7 +249,10 @@ class PrefillAdder:
                return AddReqResult.NO_TOKEN
            tokens_freed += tokens_occupied

-        if req.extend_input_len <= self.rem_chunk_tokens:
+        if (
+            self.rem_chunk_tokens is None
+            or req.extend_input_len <= self.rem_chunk_tokens
+        ):
            self.can_run_list.append(req)
            self._prefill_one_req(
                0,