fix benchmark (#743)

Co-authored-by: hnyls2002 <hnyls2002@gmail.com> Co-authored-by: Ying Sheng <sqy1415@gmail.com>
2024-07-26 21:26:13 +10:00
parent 01fbb11bb7
commit 768e05d08f
6 changed files with 103 additions and 180 deletions
--- a/python/sglang/global_config.py
+++ b/python/sglang/global_config.py
@@ -17,7 +17,7 @@ class GlobalConfig:

        # Runtime constants: New generation token ratio estimation
        self.init_new_token_ratio = 0.7
-        self.base_min_new_token_ratio = 0.2
+        self.base_min_new_token_ratio = 0.1
        self.new_token_ratio_decay = 0.001
        self.new_token_ratio_recovery = 0.05