[rust] cache-aware DP - approx tree (#1934)

2024-11-10 21:57:32 -08:00
parent 087ab83223
commit f9633fa9b9
13 changed files with 1472 additions and 177 deletions
--- a/rust/py_src/main.py
+++ b/rust/py_src/main.py
@@ -0,0 +1,12 @@
+from sglang_router import PolicyType, Router
+
+router = Router(
+    worker_urls=[
+        "http://localhost:30000",
+        "http://localhost:30001",
+    ],
+    policy=PolicyType.ApproxTree,
+    tokenizer_path="/shared/public/elr-models/meta-llama/Meta-Llama-3.1-8B-Instruct/07eb05b21d191a58c577b4a45982fe0c049d0693/tokenizer.json",
+)
+
+router.start()