Backend method not found when SRT Runtime is used (#1576)

2024-10-05 11:47:35 -07:00
parent 9b0926ceeb
commit 8ac3ccc060
1 changed files with 8 additions and 0 deletions
--- a/python/sglang/api.py
+++ b/python/sglang/api.py
@@ -48,6 +48,10 @@ def flush_cache(backend: Optional[BaseBackend] = None):
    backend = backend or global_config.default_backend
    if backend is None:
        return False
+
+    # If backend is Runtime
+    if hasattr(backend, "endpoint"):
+        backend = backend.endpoint
    return backend.flush_cache()


@@ -55,6 +59,10 @@ def get_server_args(backend: Optional[BaseBackend] = None):
    backend = backend or global_config.default_backend
    if backend is None:
        return None
+
+    # If backend is Runtime
+    if hasattr(backend, "endpoint"):
+        backend = backend.endpoint
    return backend.get_server_args()