Provide an offline engine API (#1567)

This commit is contained in:
Byron Hsu
2024-10-06 20:27:03 -07:00
committed by GitHub
parent 91877a9f9c
commit 551a3a9d38
6 changed files with 148 additions and 5 deletions

View File

@@ -33,13 +33,23 @@ def function(
def Runtime(*args, **kwargs):
# Avoid importing unnecessary dependency
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
# Avoid importing unnecessary dependency
from sglang.srt.server import Runtime
return Runtime(*args, **kwargs)
def Engine(*args, **kwargs):
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
# Avoid importing unnecessary dependency
from sglang.srt.server import Engine
return Engine(*args, **kwargs)
def set_default_backend(backend: BaseBackend):
global_config.default_backend = backend