EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix gpu mem check on CPU (#6317)

Browse Source 
Signed-off-by: yiliu30 <yi4.liu@intel.com>
This commit is contained in:
Yi Liu
2025-05-16 00:37:45 +08:00
committed by GitHub
parent fb4959b2c5
commit cfc9f9ab8d
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/server_args.py
View File

@@ -229,7 +229,7 @@ class ServerArgs:
# Set mem fraction static, which depends on the tensor parallelism size # Set mem fraction static, which depends on the tensor parallelism size
if self.mem_fraction_static is None: if self.mem_fraction_static is None:
parallel_size = self.tp_size * self.pp_size parallel_size = self.tp_size * self.pp_size
if gpu_mem <= 81920: if gpu_mem is not None and gpu_mem <= 81920:
if parallel_size >= 16: if parallel_size >= 16:
self.mem_fraction_static = 0.79 self.mem_fraction_static = 0.79
elif parallel_size >= 8: elif parallel_size >= 8:
@@ -242,7 +242,7 @@ class ServerArgs:
self.mem_fraction_static = 0.88 self.mem_fraction_static = 0.88
else: else:
self.mem_fraction_static = 0.88 self.mem_fraction_static = 0.88
if gpu_mem > 96 * 1024: if gpu_mem is not None and gpu_mem > 96 * 1024:
mem_fraction = self.mem_fraction_static mem_fraction = self.mem_fraction_static
self.mem_fraction_static = min( self.mem_fraction_static = min(
mem_fraction + 48 * 1024 * (1 - mem_fraction) / gpu_mem, mem_fraction + 48 * 1024 * (1 - mem_fraction) / gpu_mem,