EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router] support Openai router conversation API CRUD (#11297)

Browse Source
This commit is contained in:
Keyang Ru
2025-10-07 15:31:35 -07:00
committed by GitHub
parent cd4b39a900
commit 4ed67c27e3
15 changed files with 1258 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
sgl-router/tests/responses_api_test.rs
View File

@@ -239,6 +239,100 @@ async fn test_non_streaming_mcp_minimal_e2e_with_persistence() {
mcp.stop().await;
}
#[tokio::test]
async fn test_conversations_crud_basic() {
// Router in OpenAI mode (no actual upstream calls in these tests)
let router_cfg = RouterConfig {
mode: RoutingMode::OpenAI {
worker_urls: vec!["http://localhost".to_string()],
},
connection_mode: ConnectionMode::Http,
policy: PolicyConfig::Random,
host: "127.0.0.1".to_string(),
port: 0,
max_payload_size: 8 * 1024 * 1024,
request_timeout_secs: 60,
worker_startup_timeout_secs: 1,
worker_startup_check_interval_secs: 1,
dp_aware: false,
api_key: None,
discovery: None,
metrics: None,
log_dir: None,
log_level: Some("warn".to_string()),
request_id_headers: None,
max_concurrent_requests: 8,
queue_size: 0,
queue_timeout_secs: 5,
rate_limit_tokens_per_second: None,
cors_allowed_origins: vec![],
retry: RetryConfig::default(),
circuit_breaker: CircuitBreakerConfig::default(),
disable_retries: false,
disable_circuit_breaker: false,
health_check: HealthCheckConfig::default(),
enable_igw: false,
model_path: None,
tokenizer_path: None,
history_backend: sglang_router_rs::config::HistoryBackend::Memory,
oracle: None,
reasoning_parser: None,
tool_call_parser: None,
};
let ctx = AppContext::new(router_cfg, reqwest::Client::new(), 8, None).expect("ctx");
let router = RouterFactory::create_router(&Arc::new(ctx))
.await
.expect("router");
// Create
let create_body = serde_json::json!({ "metadata": { "project": "alpha" } });
let create_resp = router.create_conversation(None, &create_body).await;
assert_eq!(create_resp.status(), axum::http::StatusCode::OK);
let create_bytes = axum::body::to_bytes(create_resp.into_body(), usize::MAX)
.await
.unwrap();
let create_json: serde_json::Value = serde_json::from_slice(&create_bytes).unwrap();
let conv_id = create_json["id"].as_str().expect("id missing");
assert!(conv_id.starts_with("conv_"));
assert_eq!(create_json["object"], "conversation");
// Get
let get_resp = router.get_conversation(None, conv_id).await;
assert_eq!(get_resp.status(), axum::http::StatusCode::OK);
let get_bytes = axum::body::to_bytes(get_resp.into_body(), usize::MAX)
.await
.unwrap();
let get_json: serde_json::Value = serde_json::from_slice(&get_bytes).unwrap();
assert_eq!(get_json["metadata"]["project"], serde_json::json!("alpha"));
// Update (merge)
let update_body = serde_json::json!({ "metadata": { "owner": "alice" } });
let upd_resp = router
.update_conversation(None, conv_id, &update_body)
.await;
assert_eq!(upd_resp.status(), axum::http::StatusCode::OK);
let upd_bytes = axum::body::to_bytes(upd_resp.into_body(), usize::MAX)
.await
.unwrap();
let upd_json: serde_json::Value = serde_json::from_slice(&upd_bytes).unwrap();
assert_eq!(upd_json["metadata"]["project"], serde_json::json!("alpha"));
assert_eq!(upd_json["metadata"]["owner"], serde_json::json!("alice"));
// Delete
let del_resp = router.delete_conversation(None, conv_id).await;
assert_eq!(del_resp.status(), axum::http::StatusCode::OK);
let del_bytes = axum::body::to_bytes(del_resp.into_body(), usize::MAX)
.await
.unwrap();
let del_json: serde_json::Value = serde_json::from_slice(&del_bytes).unwrap();
assert_eq!(del_json["deleted"], serde_json::json!(true));
// Get again -> 404
let not_found = router.get_conversation(None, conv_id).await;
assert_eq!(not_found.status(), axum::http::StatusCode::NOT_FOUND);
}
#[test]
fn test_responses_request_creation() {
let request = ResponsesRequest {

74
sgl-router/tests/test_openai_routing.rs
View File

@@ -13,7 +13,10 @@ use sglang_router_rs::{
config::{
ConfigError, ConfigValidator, HistoryBackend, OracleConfig, RouterConfig, RoutingMode,
},
data_connector::{MemoryResponseStorage, ResponseId, ResponseStorage, StoredResponse},
data_connector::{
MemoryConversationStorage, MemoryResponseStorage, ResponseId, ResponseStorage,
StoredResponse,
},
protocols::spec::{
ChatCompletionRequest, ChatMessage, CompletionRequest, GenerateRequest, ResponseInput,
ResponsesGetParams, ResponsesRequest, UserMessageContent,
@@ -91,6 +94,7 @@ async fn test_openai_router_creation() {
"https://api.openai.com".to_string(),
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await;
@@ -108,6 +112,7 @@ async fn test_openai_router_server_info() {
"https://api.openai.com".to_string(),
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
@@ -137,6 +142,7 @@ async fn test_openai_router_models() {
mock_server.base_url(),
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
@@ -211,9 +217,14 @@ async fn test_openai_router_responses_with_mock() {
let base_url = format!("http://{}", addr);
let storage = Arc::new(MemoryResponseStorage::new());
let router = OpenAIRouter::new(base_url, None, storage.clone())
.await
.unwrap();
let router = OpenAIRouter::new(
base_url,
None,
storage.clone(),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
let request1 = ResponsesRequest {
model: Some("gpt-4o-mini".to_string()),
@@ -252,7 +263,7 @@ async fn test_openai_router_responses_with_mock() {
);
let stored1 = storage
.get_response(&ResponseId::from_string(resp1_id.clone()))
.get_response(&ResponseId::from(resp1_id.clone()))
.await
.unwrap()
.expect("first response missing");
@@ -261,7 +272,7 @@ async fn test_openai_router_responses_with_mock() {
assert!(stored1.previous_response_id.is_none());
let stored2 = storage
.get_response(&ResponseId::from_string(resp2_id.to_string()))
.get_response(&ResponseId::from(resp2_id))
.await
.unwrap()
.expect("second response missing");
@@ -463,12 +474,17 @@ async fn test_openai_router_responses_streaming_with_mock() {
"Earlier answer".to_string(),
None,
);
previous.id = ResponseId::from_string("resp_prev_chain".to_string());
previous.id = ResponseId::from("resp_prev_chain");
storage.store_response(previous).await.unwrap();
let router = OpenAIRouter::new(base_url, None, storage.clone())
.await
.unwrap();
let router = OpenAIRouter::new(
base_url,
None,
storage.clone(),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
let mut metadata = HashMap::new();
metadata.insert("topic".to_string(), json!("unicorns"));
@@ -504,7 +520,7 @@ async fn test_openai_router_responses_streaming_with_mock() {
assert!(body_text.contains("Once upon a streamed unicorn adventure."));
// Wait for the storage task to persist the streaming response.
let target_id = ResponseId::from_string("resp_stream_123".to_string());
let target_id = ResponseId::from("resp_stream_123");
let stored = loop {
if let Some(resp) = storage.get_response(&target_id).await.unwrap() {
break resp;
@@ -569,6 +585,7 @@ async fn test_unsupported_endpoints() {
"https://api.openai.com".to_string(),
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
@@ -605,9 +622,14 @@ async fn test_openai_router_chat_completion_with_mock() {
let base_url = mock_server.base_url();
// Create router pointing to mock server
let router = OpenAIRouter::new(base_url, None, Arc::new(MemoryResponseStorage::new()))
.await
.unwrap();
let router = OpenAIRouter::new(
base_url,
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
// Create a minimal chat completion request
let mut chat_request = create_minimal_chat_request();
@@ -642,9 +664,14 @@ async fn test_openai_e2e_with_server() {
let base_url = mock_server.base_url();
// Create router
let router = OpenAIRouter::new(base_url, None, Arc::new(MemoryResponseStorage::new()))
.await
.unwrap();
let router = OpenAIRouter::new(
base_url,
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
// Create Axum app with chat completions endpoint
let app = Router::new().route(
@@ -707,9 +734,14 @@ async fn test_openai_e2e_with_server() {
async fn test_openai_router_chat_streaming_with_mock() {
let mock_server = MockOpenAIServer::new().await;
let base_url = mock_server.base_url();
let router = OpenAIRouter::new(base_url, None, Arc::new(MemoryResponseStorage::new()))
.await
.unwrap();
let router = OpenAIRouter::new(
base_url,
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
// Build a streaming chat request
let val = json!({
@@ -759,6 +791,7 @@ async fn test_openai_router_circuit_breaker() {
"http://invalid-url-that-will-fail".to_string(),
Some(cb_config),
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();
@@ -786,6 +819,7 @@ async fn test_openai_router_models_auth_forwarding() {
mock_server.base_url(),
None,
Arc::new(MemoryResponseStorage::new()),
Arc::new(MemoryConversationStorage::new()),
)
.await
.unwrap();