From 0975ba99bcf38c34c28e738c14e9df0abb9cb10a Mon Sep 17 00:00:00 2001 From: Keyang Ru Date: Thu, 16 Oct 2025 09:00:08 -0700 Subject: [PATCH] [router] fix get_models endpoint for openai router (#11687) --- sgl-router/src/routers/router_manager.rs | 17 +++++- sgl-router/tests/test_openai_routing.rs | 66 ++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/sgl-router/src/routers/router_manager.rs b/sgl-router/src/routers/router_manager.rs index 740354f4f..13ece2c97 100644 --- a/sgl-router/src/routers/router_manager.rs +++ b/sgl-router/src/routers/router_manager.rs @@ -313,7 +313,22 @@ impl RouterTrait for RouterManager { .into_response() } - async fn get_models(&self, _req: Request) -> Response { + async fn get_models(&self, req: Request) -> Response { + // In single-router mode, delegate to the router (especially for OpenAI mode) + if !self.enable_igw { + let router = self + .default_router + .read() + .expect("Default router lock is poisoned") + .as_ref() + .and_then(|id| self.routers.get(id).map(|r| r.value().clone())); + + if let Some(router) = router { + return router.get_models(req).await; + } + } + + // In multi-router mode, aggregate models from worker registry let models = self.worker_registry.get_models(); if models.is_empty() { diff --git a/sgl-router/tests/test_openai_routing.rs b/sgl-router/tests/test_openai_routing.rs index b38b90f7e..5dca1a327 100644 --- a/sgl-router/tests/test_openai_routing.rs +++ b/sgl-router/tests/test_openai_routing.rs @@ -956,3 +956,69 @@ fn oracle_config_validation_accepts_wallet_alias() { ConfigValidator::validate(&config).expect("wallet-based config should validate"); } + +/// Test that RouterManager delegates /v1/models to OpenAI router in single-router mode +#[tokio::test] +async fn test_router_manager_delegates_models_to_openai_router() { + use sglang_router_rs::routers::router_manager::RouterManager; + use sglang_router_rs::server::ServerConfig; + + // Start a mock OpenAI server + let mock_server = MockOpenAIServer::new().await; + let base_url = mock_server.base_url(); + + // Create router config in OpenAI mode + let routing_mode = RoutingMode::OpenAI { + worker_urls: vec![base_url.clone()], + }; + + let router_config = + RouterConfig::new(routing_mode, sglang_router_rs::config::PolicyConfig::Random); + + let server_config = ServerConfig { + host: "127.0.0.1".to_string(), + port: 0, + router_config: router_config.clone(), + max_payload_size: 1024 * 1024, + log_dir: None, + log_level: None, + service_discovery_config: None, + prometheus_config: None, + request_timeout_secs: 300, + request_id_headers: None, + }; + + // Create app context + let app_context = common::create_test_context(router_config); + + // Create RouterManager (single-router mode, enable_igw=false) + let router_manager = RouterManager::from_config(&server_config, &app_context) + .await + .expect("RouterManager creation should succeed"); + + // Make a request to /v1/models + let req = Request::builder() + .method(Method::GET) + .uri("/v1/models") + .body(Body::empty()) + .unwrap(); + + // RouterManager should delegate to OpenAI router, which proxies to mock server + let response = router_manager.get_models(req).await; + + // Should succeed (not 503 "No models available") + assert_eq!( + response.status(), + StatusCode::OK, + "RouterManager should delegate to OpenAI router and return models" + ); + + // Verify response contains model list + let (_, body) = response.into_parts(); + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap(); + let body_str = String::from_utf8(body_bytes.to_vec()).unwrap(); + let models: serde_json::Value = serde_json::from_str(&body_str).unwrap(); + + assert_eq!(models["object"], "list"); + assert!(models["data"].is_array()); +}