From 8ac3ccc060fe59c0ff76011ac40cb1a2f22edfdc Mon Sep 17 00:00:00 2001 From: Byron Hsu Date: Sat, 5 Oct 2024 11:47:35 -0700 Subject: [PATCH] Backend method not found when SRT Runtime is used (#1576) --- python/sglang/api.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/sglang/api.py b/python/sglang/api.py index 9405606b7..5adda6022 100644 --- a/python/sglang/api.py +++ b/python/sglang/api.py @@ -48,6 +48,10 @@ def flush_cache(backend: Optional[BaseBackend] = None): backend = backend or global_config.default_backend if backend is None: return False + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint return backend.flush_cache() @@ -55,6 +59,10 @@ def get_server_args(backend: Optional[BaseBackend] = None): backend = backend or global_config.default_backend if backend is None: return None + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint return backend.get_server_args()