@@ -381,6 +381,18 @@ class ServerArgs:
|
|||||||
self.disable_radix_cache = True
|
self.disable_radix_cache = True
|
||||||
logger.warning("KV cache is forced as chunk cache for decode server")
|
logger.warning("KV cache is forced as chunk cache for decode server")
|
||||||
|
|
||||||
|
if self.enable_memory_saver:
|
||||||
|
try:
|
||||||
|
import torch_memory_saver
|
||||||
|
except ImportError:
|
||||||
|
logger.warning(
|
||||||
|
"enable_memory_saver is enabled, but "
|
||||||
|
"torch-memory-saver is not installed. Please install it "
|
||||||
|
"via `pip3 uninstall torch-memory-saver`. "
|
||||||
|
"For normal operation, it will be disabled."
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
os.environ["SGLANG_ENABLE_TORCH_COMPILE"] = (
|
os.environ["SGLANG_ENABLE_TORCH_COMPILE"] = (
|
||||||
"1" if self.enable_torch_compile else "0"
|
"1" if self.enable_torch_compile else "0"
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user