From 4b1d7a2583cacb9bf3abf200a5efbcf4556a91c5 Mon Sep 17 00:00:00 2001
From: Austin Liu <austin362667@gmail.com>
Date: Thu, 7 Nov 2024 10:08:30 +0800
Subject: [PATCH] Add Rust Router Python Binding (#1891)

Signed-off-by: Austin Liu <austin362667@gmail.com>
Co-authored-by: ByronHsu <byronhsu1230@gmail.com>
---
 rust/pyproject.toml   | 15 +++++++++
 rust/readme.md        | 72 ++++++++++++++++++++++++++-----------------
 rust/test_bindings.py |  3 +-
 3 files changed, 61 insertions(+), 29 deletions(-)
 create mode 100644 rust/pyproject.toml

diff --git a/rust/pyproject.toml b/rust/pyproject.toml
new file mode 100644
index 000000000..6eb82967f
--- /dev/null
+++ b/rust/pyproject.toml
@@ -0,0 +1,15 @@
+[build-system]
+requires = ["maturin>=1.5.1,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "sglang_router"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+]
+dynamic = ["version"]
+
+[tool.maturin]
+bindings = 'pyo3'
diff --git a/rust/readme.md b/rust/readme.md
index a6d1556a8..716e50a97 100644
--- a/rust/readme.md
+++ b/rust/readme.md
@@ -1,10 +1,21 @@
-# SGLang Router
+# SGLang Router (Experimental)
 
 SGLang router is a standalone module implemented in Rust to achieve data parallelism across SGLang instances.
 
-### Installation
+## Installation
+
+WIP. Ideally just
+
+```bash
+pip install sglang-router
+```
+
+## Development
+
+### Rust
 
 1. Install Rust
+
 ```bash
 # Install rustup (Rust installer and version manager)
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
@@ -18,6 +29,7 @@ cargo --version
 ```
 
 2. Build the router
+
 ```bash
 # Navigate to the rust directory
 cd ./rust
@@ -42,7 +54,36 @@ Options:
   -V, --version                    Print version
 ```
 
-### Setting Up Workers
+### Python Binding
+
+1. Create a virtual environment
+
+```bash
+$ python -m venv .venv
+$ source .venv/bin/activate
+```
+
+2. Install python dependencies
+
+```bash
+$ pip install maturin
+$ pip install patchelf
+```
+
+3. Install rust python binding
+
+```bash
+$ maturin develop
+🔗 Found pyo3 bindings
+🐍 Found CPython 3.10 at /home/jobuser/resources/sglang/rust/.venv/bin/python
+📡 Using build options bindings from pyproject.toml
+    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.11s
+📦 Built wheel for CPython 3.10 to /tmp/.tmpJb65sc/sglang_router-0.0.0-cp310-cp310-linux_x86_64.whl
+✏️  Setting installed package as editable
+🛠 Installed sglang_router-0.0.0
+```
+
+## Usage
 
 1. Launch worker instances
 ```bash
@@ -65,28 +106,3 @@ python -m sglang.launch_server \
 ```bash
 ./target/debug/router --worker-urls http://127.0.0.1:30000,http://127.0.0.1:30002
 ```
-
-**Note**: This module is still experimental. Please expect active changes and updates.
-
-### Python bindings
-
-```bash
-$ cargo build --release
-$ maturin build -i /usr/bin/python
-$ pip install <path to wheel>
-```
-
-
-### Development
-
-1. Run test
-
-```
-$ cargo test
-```
-
-2. Run lint
-
-```
-$ cargo fmt
-```
diff --git a/rust/test_bindings.py b/rust/test_bindings.py
index f85cfabe9..d81e1451f 100644
--- a/rust/test_bindings.py
+++ b/rust/test_bindings.py
@@ -1,4 +1,4 @@
-import router
+import sglang_router as router
 
 # Create a Router instance with:
 # - host: the address to bind to (e.g., "127.0.0.1")
@@ -11,6 +11,7 @@ router = router.Router(
         "http://localhost:30000",
         "http://localhost:30002",
     ],
+    policy="random"
 )
 
 # Start the router - this will block and run the server