EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router] Expose worker startup interval (#3019)

Browse Source
This commit is contained in:
Byron Hsu
2025-01-20 14:36:54 -08:00
committed by GitHub
parent 0311ce8e1c
commit 3a8428ecaa
5 changed files with 72 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
sgl-router/py_src/sglang_router/launch_router.py
View File

@@ -34,6 +34,7 @@ class RouterArgs:
# Routing policy
policy: str = "cache_aware"
worker_startup_timeout_secs: int = 300
worker_startup_check_interval: int = 10
cache_threshold: float = 0.5
balance_abs_threshold: int = 32
balance_rel_threshold: float = 1.0001
@@ -94,6 +95,12 @@ class RouterArgs:
default=RouterArgs.worker_startup_timeout_secs,
help="Timeout in seconds for worker startup",
)
parser.add_argument(
f"--{prefix}worker-startup-check-interval",
type=int,
default=RouterArgs.worker_startup_check_interval,
help="Interval in seconds between checks for worker startup",
)
parser.add_argument(
f"--{prefix}cache-threshold",
type=float,
@@ -157,6 +164,9 @@ class RouterArgs:
worker_startup_timeout_secs=getattr(
args, f"{prefix}worker_startup_timeout_secs"
),
worker_startup_check_interval=getattr(
args, f"{prefix}worker_startup_check_interval"
),
cache_threshold=getattr(args, f"{prefix}cache_threshold"),
balance_abs_threshold=getattr(args, f"{prefix}balance_abs_threshold"),
balance_rel_threshold=getattr(args, f"{prefix}balance_rel_threshold"),
@@ -202,6 +212,7 @@ def launch_router(args: argparse.Namespace) -> Optional[Router]:
port=router_args.port,
policy=policy_from_str(router_args.policy),
worker_startup_timeout_secs=router_args.worker_startup_timeout_secs,
worker_startup_check_interval=router_args.worker_startup_check_interval,
cache_threshold=router_args.cache_threshold,
balance_abs_threshold=router_args.balance_abs_threshold,
balance_rel_threshold=router_args.balance_rel_threshold,

3
sgl-router/py_src/sglang_router/router.py
View File

@@ -18,6 +18,7 @@ class Router:
host: Host address to bind the router server. Default: '127.0.0.1'
port: Port number to bind the router server. Default: 3001
worker_startup_timeout_secs: Timeout in seconds for worker startup. Default: 300
worker_startup_check_interval: Interval in seconds between checks for worker initialization. Default: 10
cache_threshold: Cache threshold (0.0-1.0) for cache-aware routing. Routes to cached worker
if the match rate exceeds threshold, otherwise routes to the worker with the smallest
tree. Default: 0.5
@@ -39,6 +40,7 @@ class Router:
host: str = "127.0.0.1",
port: int = 3001,
worker_startup_timeout_secs: int = 300,
worker_startup_check_interval: int = 10,
cache_threshold: float = 0.50,
balance_abs_threshold: int = 32,
balance_rel_threshold: float = 1.0001,
@@ -53,6 +55,7 @@ class Router:
host=host,
port=port,
worker_startup_timeout_secs=worker_startup_timeout_secs,
worker_startup_check_interval=worker_startup_check_interval,
cache_threshold=cache_threshold,
balance_abs_threshold=balance_abs_threshold,
balance_rel_threshold=balance_rel_threshold,