Enable chunked prefill by default (#1040)

2024-08-14 21:56:20 -07:00
parent 8d2d876fc8
commit e86b1ccbf0
4 changed files with 10 additions and 10 deletions
--- a/python/sglang/srt/server_args.py
+++ b/python/sglang/srt/server_args.py
@@ -49,7 +49,7 @@ class ServerArgs:
    max_running_requests: Optional[int] = None
    max_num_reqs: Optional[int] = None
    max_total_tokens: Optional[int] = None
-    chunked_prefill_size: int = -1
+    chunked_prefill_size: int = 8192
    max_prefill_tokens: int = 16384
    schedule_policy: str = "lpm"
    schedule_conservativeness: float = 1.0