From c3f1aac811c85abe97129ad5c917e8878890e2f9 Mon Sep 17 00:00:00 2001 From: Ying Sheng Date: Mon, 22 Jul 2024 03:19:24 -0700 Subject: [PATCH] Tune params (#696) --- python/sglang/srt/server_args.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index d77ea8782..8ec153887 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -70,15 +70,15 @@ class ServerArgs: self.tokenizer_path = self.model_path if self.mem_fraction_static is None: if self.tp_size >= 16: - self.mem_fraction_static = 0.74 + self.mem_fraction_static = 0.80 elif self.tp_size >= 8: - self.mem_fraction_static = 0.78 + self.mem_fraction_static = 0.84 elif self.tp_size >= 4: - self.mem_fraction_static = 0.82 + self.mem_fraction_static = 0.86 elif self.tp_size >= 2: - self.mem_fraction_static = 0.85 - else: self.mem_fraction_static = 0.88 + else: + self.mem_fraction_static = 0.89 if isinstance(self.additional_ports, int): self.additional_ports = [self.additional_ports] elif self.additional_ports is None: