[Core] in batch prefix caching by delay scheduling (#2442)

2024-12-11 12:51:50 -08:00
parent 864bf2ba00
commit 9208618b3e
8 changed files with 87 additions and 16 deletions
--- a/python/sglang/lang/backend/runtime_endpoint.py
+++ b/python/sglang/lang/backend/runtime_endpoint.py
@@ -55,6 +55,7 @@ class RuntimeEndpoint(BaseBackend):
            self.base_url + "/flush_cache",
            api_key=self.api_key,
            verify=self.verify,
+            method="POST",
        )
        self._assert_success(res)