[router] add token bucket rate limiter (#9656)

sgl-router/py_src/sglang_router/launch_router.py

@@ -72,6 +72,12 @@ class RouterArgs:
     request_timeout_secs: int = 1800
     # Max concurrent requests for rate limiting
     max_concurrent_requests: int = 256
+    # Queue size for pending requests when max concurrent limit reached
+    queue_size: int = 100
+    # Maximum time (in seconds) a request can wait in queue before timing out
+    queue_timeout_secs: int = 60
+    # Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
+    rate_limit_tokens_per_second: Optional[int] = None
     # CORS allowed origins
     cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
     # Retry configuration
@@ -402,6 +408,24 @@ class RouterArgs:
             default=RouterArgs.max_concurrent_requests,
             help="Maximum number of concurrent requests allowed (for rate limiting)",
         )
+        parser.add_argument(
+            f"--{prefix}queue-size",
+            type=int,
+            default=RouterArgs.queue_size,
+            help="Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately)",
+        )
+        parser.add_argument(
+            f"--{prefix}queue-timeout-secs",
+            type=int,
+            default=RouterArgs.queue_timeout_secs,
+            help="Maximum time (in seconds) a request can wait in queue before timing out",
+        )
+        parser.add_argument(
+            f"--{prefix}rate-limit-tokens-per-second",
+            type=int,
+            default=RouterArgs.rate_limit_tokens_per_second,
+            help="Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests",
+        )
         parser.add_argument(
             f"--{prefix}cors-allowed-origins",
             type=str,
@@ -478,6 +502,21 @@ class RouterArgs:
                 f"{prefix}max_concurrent_requests",
                 RouterArgs.max_concurrent_requests,
             ),
+            queue_size=getattr(
+                args,
+                f"{prefix}queue_size",
+                RouterArgs.queue_size,
+            ),
+            queue_timeout_secs=getattr(
+                args,
+                f"{prefix}queue_timeout_secs",
+                RouterArgs.queue_timeout_secs,
+            ),
+            rate_limit_tokens_per_second=getattr(
+                args,
+                f"{prefix}rate_limit_tokens_per_second",
+                RouterArgs.rate_limit_tokens_per_second,
+            ),
             cors_allowed_origins=getattr(args, f"{prefix}cors_allowed_origins", []),
             retry_max_retries=getattr(args, f"{prefix}retry_max_retries"),
             retry_initial_backoff_ms=getattr(args, f"{prefix}retry_initial_backoff_ms"),
@@ -700,6 +739,9 @@ def launch_router(args: argparse.Namespace) -> Optional[Router]:
             ),
             request_id_headers=router_args.request_id_headers,
             max_concurrent_requests=router_args.max_concurrent_requests,
+            queue_size=router_args.queue_size,
+            queue_timeout_secs=router_args.queue_timeout_secs,
+            rate_limit_tokens_per_second=router_args.rate_limit_tokens_per_second,
             cors_allowed_origins=router_args.cors_allowed_origins,
             retry_max_retries=router_args.retry_max_retries,
             retry_initial_backoff_ms=router_args.retry_initial_backoff_ms,

sgl-router/py_src/sglang_router/router.py

@@ -64,7 +64,10 @@ class Router:
         bootstrap_port_annotation: Kubernetes annotation name for bootstrap port (PD mode).
             Default: 'sglang.ai/bootstrap-port'
         request_timeout_secs: Request timeout in seconds. Default: 600
-        max_concurrent_requests: Maximum number of concurrent requests allowed for rate limiting. Default: 64
+        max_concurrent_requests: Maximum number of concurrent requests allowed for rate limiting. Default: 256
+        queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
+        queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
+        rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
         cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
         health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
         health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
@@ -108,6 +111,9 @@ class Router:
         prefill_policy: Optional[PolicyType] = None,
         decode_policy: Optional[PolicyType] = None,
         max_concurrent_requests: int = 256,
+        queue_size: int = 100,
+        queue_timeout_secs: int = 60,
+        rate_limit_tokens_per_second: Optional[int] = None,
         cors_allowed_origins: List[str] = None,
         retry_max_retries: int = 5,
         retry_initial_backoff_ms: int = 50,
@@ -169,6 +175,9 @@ class Router:
             prefill_policy=prefill_policy,
             decode_policy=decode_policy,
             max_concurrent_requests=max_concurrent_requests,
+            queue_size=queue_size,
+            queue_timeout_secs=queue_timeout_secs,
+            rate_limit_tokens_per_second=rate_limit_tokens_per_second,
             cors_allowed_origins=cors_allowed_origins,
             retry_max_retries=retry_max_retries,
             retry_initial_backoff_ms=retry_initial_backoff_ms,