Cache optimizations (#418)

2024-05-13 12:47:13 +08:00
parent 562b8857d8
commit 39191c8515
8 changed files with 117 additions and 96 deletions
--- a/python/sglang/global_config.py
+++ b/python/sglang/global_config.py
@@ -25,5 +25,8 @@ class GlobalConfig:
        # adjust_cache: Adjust the position embedding of KV cache.
        self.concate_and_append_mode = "no_adjust"

+        # Request dependency time due to network delay
+        self.request_dependency_time = 0.03
+

 global_config = GlobalConfig()