[router] add token bucket rate limiter (#9656)

2025-08-26 10:36:26 -07:00
parent 3578eb1e9b
commit 90313fb09a
15 changed files with 533 additions and 10 deletions
--- a/sgl-router/tests/streaming_tests.rs
+++ b/sgl-router/tests/streaming_tests.rs
@@ -37,6 +37,9 @@ impl TestContext {
            log_level: None,
            request_id_headers: None,
            max_concurrent_requests: 64,
+            queue_size: 0,
+            queue_timeout_secs: 60,
+            rate_limit_tokens_per_second: None,
            cors_allowed_origins: vec![],
            retry: RetryConfig::default(),
            circuit_breaker: CircuitBreakerConfig::default(),