EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

fix: update HostKVCache init to report correct msg when available memory is not enough (#8102)

Browse Source
This commit is contained in:
Ziqi Fan
2025-07-17 06:24:34 -07:00
committed by GitHub
parent 519ff5c8e6
commit 01857fab61
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/mem_cache/memory_pool_host.py
View File

@@ -71,11 +71,12 @@ class HostKVCache(abc.ABC):
requested_bytes = self.size * self.size_per_token requested_bytes = self.size * self.size_per_token
# preserve at least 10GB for other usage # preserve at least 10GB for other usage
ten_gb = 10 * (1024**3) ten_gb = 10 * (1024**3)
if requested_bytes > host_mem.available - ten_gb: available_bytes = host_mem.available - ten_gb
if requested_bytes > available_bytes:
raise ValueError( raise ValueError(
f"Not enough host memory available. Requesting " f"Not enough host memory available. Requesting "
f"{requested_bytes / 1e9:.2f} GB but only have " f"{requested_bytes / 1e9:.2f} GB but only have "
f"{host_mem.available / 1e9:.2f} GB free. Please reduce the " f"{available_bytes / 1e9:.2f} GB free. Please reduce the "
f"size of the hierarchical cache." f"size of the hierarchical cache."
) )
else: else: