EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Teak mem fraction (#20)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-01-17 04:43:17 -08:00
committed by GitHub
parent bf51ddc6e5
commit f9d723816a
2 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/server_args.py
View File

@@ -26,10 +26,14 @@ class ServerArgs:
if self.tokenizer_path is None:
self.tokenizer_path = self.model_path
if self.mem_fraction_static is None:
if self.tp_size > 1:
self.mem_fraction_static = 0.8
if self.tp_size >= 8:
self.mem_fraction_static = 0.80
elif self.tp_size >= 4:
self.mem_fraction_static = 0.82
elif self.tp_size >= 2:
self.mem_fraction_static = 0.85
else:
self.mem_fraction_static = 0.9
self.mem_fraction_static = 0.90
@staticmethod
def add_cli_args(parser: argparse.ArgumentParser):