EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Use min new token ratio at start (#701)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-07-23 11:52:50 -07:00
committed by GitHub
parent 824a77d04d
commit 268684439b
2 changed files with 2 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/managers/controller/tp_worker.py
View File

@@ -161,15 +161,12 @@ class ModelTpServer:
assert (
server_args.schedule_conservativeness >= 0
), "Invalid schedule_conservativeness"
self.new_token_ratio = min(
global_config.base_new_token_ratio * server_args.schedule_conservativeness,
1.0,
)
self.min_new_token_ratio = min(
global_config.base_min_new_token_ratio
* server_args.schedule_conservativeness,
1.0,
)
self.new_token_ratio = self.min_new_token_ratio
self.new_token_ratio_decay = global_config.new_token_ratio_decay
self.new_token_ratio_recovery = global_config.new_token_ratio_recovery

2
python/sglang/srt/server_args.py
View File

@@ -29,7 +29,7 @@ class ServerArgs:
max_prefill_tokens: Optional[int] = None
max_running_requests: Optional[int] = None
schedule_heuristic: str = "lpm"
schedule_conservativeness: float = 0.8
schedule_conservativeness: float = 1.0
# Other runtime options
tp_size: int = 1