Add memory_saver check (#4986)

Signed-off-by: Kebe <mail@kebe7jun.com>
This commit is contained in:
Kebe
2025-04-27 11:20:05 +08:00
committed by GitHub
parent a086a11305
commit 7e944246c3

View File

@@ -381,6 +381,18 @@ class ServerArgs:
self.disable_radix_cache = True
logger.warning("KV cache is forced as chunk cache for decode server")
if self.enable_memory_saver:
try:
import torch_memory_saver
except ImportError:
logger.warning(
"enable_memory_saver is enabled, but "
"torch-memory-saver is not installed. Please install it "
"via `pip3 uninstall torch-memory-saver`. "
"For normal operation, it will be disabled."
)
raise
os.environ["SGLANG_ENABLE_TORCH_COMPILE"] = (
"1" if self.enable_torch_compile else "0"
)