Simplify mem state (#623)

2024-07-15 02:01:09 -07:00
parent bae9541e4c
commit 5ac8b80677
7 changed files with 61 additions and 66 deletions
--- a/python/sglang/global_config.py
+++ b/python/sglang/global_config.py
@@ -25,7 +25,8 @@ class GlobalConfig:
        # This can improve the speed for large batch sizes during prefill.
        self.layer_sync_threshold = 8192

-        # Runtime constants: Flashinfer
+        # Runtime constants: others
+        self.num_continue_decode_steps = 10
        self.flashinfer_workspace_size = 192 * 1024 * 1024

        # Output tokenization configs