EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router] support Openai router conversation API CRUD (#11297)

Browse Source
This commit is contained in:
Keyang Ru
2025-10-07 15:31:35 -07:00
committed by GitHub
parent cd4b39a900
commit 4ed67c27e3
15 changed files with 1258 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sgl-router/src/routers/factory.rs
View File

@@ -128,6 +128,7 @@ impl RouterFactory {
base_url,
Some(ctx.router_config.circuit_breaker.clone()),
ctx.response_storage.clone(),
ctx.conversation_storage.clone(),
)
.await?;

329
sgl-router/src/routers/http/openai_router.rs
View File

@@ -2,7 +2,10 @@
use crate::config::CircuitBreakerConfig;
use crate::core::{CircuitBreaker, CircuitBreakerConfig as CoreCircuitBreakerConfig};
use crate::data_connector::{ResponseId, SharedResponseStorage, StoredResponse};
use crate::data_connector::{
Conversation, ConversationId, ConversationMetadata, ResponseId, SharedConversationStorage,
SharedResponseStorage, StoredResponse,
};
use crate::protocols::spec::{
ChatCompletionRequest, CompletionRequest, EmbeddingRequest, GenerateRequest, RerankRequest,
ResponseContentPart, ResponseInput, ResponseInputOutputItem, ResponseOutputItem,
@@ -16,6 +19,7 @@ use axum::{
extract::Request,
http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode},
response::{IntoResponse, Response},
Json,
};
use bytes::Bytes;
use futures_util::StreamExt;
@@ -75,6 +79,8 @@ pub struct OpenAIRouter {
healthy: AtomicBool,
/// Response storage for managing conversation history
response_storage: SharedResponseStorage,
/// Conversation storage backend
conversation_storage: SharedConversationStorage,
/// Optional MCP manager (enabled via config presence)
mcp_manager: Option<Arc<crate::mcp::McpClientManager>>,
}
@@ -705,6 +711,7 @@ impl OpenAIRouter {
base_url: String,
circuit_breaker_config: Option<CircuitBreakerConfig>,
response_storage: SharedResponseStorage,
conversation_storage: SharedConversationStorage,
) -> Result<Self, String> {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(300))
@@ -751,6 +758,7 @@ impl OpenAIRouter {
circuit_breaker,
healthy: AtomicBool::new(true),
response_storage,
conversation_storage,
mcp_manager,
})
}
@@ -2337,16 +2345,16 @@ impl OpenAIRouter {
stored_response.previous_response_id = response_json
.get("previous_response_id")
.and_then(|v| v.as_str())
.map(|s| ResponseId::from_string(s.to_string()))
.map(ResponseId::from)
.or_else(|| {
original_body
.previous_response_id
.as_ref()
.map(|id| ResponseId::from_string(id.clone()))
.map(|id| ResponseId::from(id.as_str()))
});
if let Some(id_str) = response_json.get("id").and_then(|v| v.as_str()) {
stored_response.id = ResponseId::from_string(id_str.to_string());
stored_response.id = ResponseId::from(id_str);
}
stored_response.raw_response = response_json.clone();
@@ -3393,7 +3401,7 @@ impl super::super::RouterTrait for OpenAIRouter {
// Handle previous_response_id by loading prior context
let mut conversation_items: Option<Vec<ResponseInputOutputItem>> = None;
if let Some(prev_id_str) = request_body.previous_response_id.clone() {
let prev_id = ResponseId::from_string(prev_id_str.clone());
let prev_id = ResponseId::from(prev_id_str.as_str());
match self
.response_storage
.get_response_chain(&prev_id, None)
@@ -3516,7 +3524,7 @@ impl super::super::RouterTrait for OpenAIRouter {
response_id: &str,
params: &ResponsesGetParams,
) -> Response {
let stored_id = ResponseId::from_string(response_id.to_string());
let stored_id = ResponseId::from(response_id);
if let Ok(Some(stored_response)) = self.response_storage.get_response(&stored_id).await {
let stream_requested = params.stream.unwrap_or(false);
let raw_value = stored_response.raw_response.clone();
@@ -3646,10 +3654,6 @@ impl super::super::RouterTrait for OpenAIRouter {
}
}
fn router_type(&self) -> &'static str {
"openai"
}
async fn route_embeddings(
&self,
_headers: Option<&HeaderMap>,
@@ -3675,4 +3679,309 @@ impl super::super::RouterTrait for OpenAIRouter {
)
.into_response()
}
async fn create_conversation(&self, _headers: Option<&HeaderMap>, body: &Value) -> Response {
// TODO: move this spec validation to the right place
let metadata = match body.get("metadata") {
Some(Value::Object(map)) => {
if map.len() > MAX_METADATA_PROPERTIES {
return (
StatusCode::BAD_REQUEST,
Json(json!({
"error": {
"message": format!(
"Invalid 'metadata': too many properties. Max {}, got {}",
MAX_METADATA_PROPERTIES, map.len()
),
"type": "invalid_request_error",
"param": "metadata",
"code": "metadata_max_properties_exceeded"
}
})),
)
.into_response();
}
Some(map.clone())
}
Some(Value::Null) | None => None,
Some(other) => {
return (
StatusCode::BAD_REQUEST,
Json(json!({
"error": {
"message": format!(
"Invalid 'metadata': expected object or null but got {}",
other
),
"type": "invalid_request_error",
"param": "metadata",
"code": "metadata_invalid_type"
}
})),
)
.into_response();
}
};
match self
.conversation_storage
.create_conversation(crate::data_connector::NewConversation { metadata })
.await
{
Ok(conversation) => {
(StatusCode::OK, Json(conversation_to_json(&conversation))).into_response()
}
Err(err) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({
"error": {
"message": err.to_string(),
"type": "internal_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response(),
}
}
async fn get_conversation(
&self,
_headers: Option<&HeaderMap>,
conversation_id: &str,
) -> Response {
let id: ConversationId = conversation_id.to_string().into();
match self.conversation_storage.get_conversation(&id).await {
Ok(Some(conv)) => (StatusCode::OK, Json(conversation_to_json(&conv))).into_response(),
Ok(None) => (
StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": format!("Conversation with id '{}' not found.", conversation_id),
"type": "invalid_request_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response(),
Err(err) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({
"error": {
"message": err.to_string(),
"type": "internal_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response(),
}
}
async fn update_conversation(
&self,
_headers: Option<&HeaderMap>,
conversation_id: &str,
body: &Value,
) -> Response {
let id: ConversationId = conversation_id.to_string().into();
let existing = match self.conversation_storage.get_conversation(&id).await {
Ok(Some(c)) => c,
Ok(None) => {
return (
StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": format!("Conversation with id '{}' not found.", conversation_id),
"type": "invalid_request_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response();
}
Err(err) => {
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({
"error": {
"message": err.to_string(),
"type": "internal_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response();
}
};
// Parse metadata patch
enum Patch {
NoChange,
ClearAll,
Merge(ConversationMetadata),
}
let patch = match body.get("metadata") {
None => Patch::NoChange,
Some(Value::Null) => Patch::ClearAll,
Some(Value::Object(map)) => Patch::Merge(map.clone()),
Some(other) => {
return (
StatusCode::BAD_REQUEST,
Json(json!({
"error": {
"message": format!(
"Invalid 'metadata': expected object or null but got {}",
other
),
"type": "invalid_request_error",
"param": "metadata",
"code": "metadata_invalid_type"
}
})),
)
.into_response();
}
};
let merged_metadata = match patch {
Patch::NoChange => {
return (StatusCode::OK, Json(conversation_to_json(&existing))).into_response();
}
Patch::ClearAll => None,
Patch::Merge(upd) => {
let mut merged = existing.metadata.clone().unwrap_or_default();
let previous = merged.len();
for (k, v) in upd.into_iter() {
if v.is_null() {
merged.remove(&k);
} else {
merged.insert(k, v);
}
}
let updated = merged.len();
if updated > MAX_METADATA_PROPERTIES {
return (
StatusCode::BAD_REQUEST,
Json(json!({
"error": {
"message": format!(
"Invalid 'metadata': too many properties after update. Max {} ({} -> {}).",
MAX_METADATA_PROPERTIES, previous, updated
),
"type": "invalid_request_error",
"param": "metadata",
"code": "metadata_max_properties_exceeded",
"extra": {
"previous_property_count": previous,
"updated_property_count": updated
}
}
})),
)
.into_response();
}
if merged.is_empty() {
None
} else {
Some(merged)
}
}
};
match self
.conversation_storage
.update_conversation(&id, merged_metadata)
.await
{
Ok(Some(conv)) => (StatusCode::OK, Json(conversation_to_json(&conv))).into_response(),
Ok(None) => (
StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": format!("Conversation with id '{}' not found.", conversation_id),
"type": "invalid_request_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response(),
Err(err) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({
"error": {
"message": err.to_string(),
"type": "internal_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response(),
}
}
async fn delete_conversation(
&self,
_headers: Option<&HeaderMap>,
conversation_id: &str,
) -> Response {
let id: ConversationId = conversation_id.to_string().into();
match self.conversation_storage.delete_conversation(&id).await {
Ok(true) => (
StatusCode::OK,
Json(json!({
"id": conversation_id,
"object": "conversation.deleted",
"deleted": true
})),
)
.into_response(),
Ok(false) => (
StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": format!("Conversation with id '{}' not found.", conversation_id),
"type": "invalid_request_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response(),
Err(err) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({
"error": {
"message": err.to_string(),
"type": "internal_error",
"param": Value::Null,
"code": Value::Null
}
})),
)
.into_response(),
}
}
fn router_type(&self) -> &'static str {
"openai"
}
}
// Maximum number of properties allowed in conversation metadata (align with server)
const MAX_METADATA_PROPERTIES: usize = 16;
fn conversation_to_json(conversation: &Conversation) -> Value {
json!({
"id": conversation.id.0,
"object": "conversation",
"created_at": conversation.created_at.timestamp(),
"metadata": to_value(&conversation.metadata).unwrap_or(Value::Null),
})
}

47
sgl-router/src/routers/mod.rs
View File

@@ -13,6 +13,7 @@ use crate::protocols::spec::{
ChatCompletionRequest, CompletionRequest, EmbeddingRequest, GenerateRequest, RerankRequest,
ResponsesGetParams, ResponsesRequest,
};
use serde_json::Value;
pub mod factory;
pub mod grpc;
@@ -126,6 +127,52 @@ pub trait RouterTrait: Send + Sync + Debug {
model_id: Option<&str>,
) -> Response;
// Conversations API
async fn create_conversation(&self, _headers: Option<&HeaderMap>, _body: &Value) -> Response {
(
StatusCode::NOT_IMPLEMENTED,
"Conversations create endpoint not implemented",
)
.into_response()
}
async fn get_conversation(
&self,
_headers: Option<&HeaderMap>,
_conversation_id: &str,
) -> Response {
(
StatusCode::NOT_IMPLEMENTED,
"Conversations get endpoint not implemented",
)
.into_response()
}
async fn update_conversation(
&self,
_headers: Option<&HeaderMap>,
_conversation_id: &str,
_body: &Value,
) -> Response {
(
StatusCode::NOT_IMPLEMENTED,
"Conversations update endpoint not implemented",
)
.into_response()
}
async fn delete_conversation(
&self,
_headers: Option<&HeaderMap>,
_conversation_id: &str,
) -> Response {
(
StatusCode::NOT_IMPLEMENTED,
"Conversations delete endpoint not implemented",
)
.into_response()
}
/// Get router type name
fn router_type(&self) -> &'static str;

78
sgl-router/src/routers/router_manager.rs
View File

@@ -20,6 +20,7 @@ use axum::{
response::{IntoResponse, Response},
};
use dashmap::DashMap;
use serde_json::Value;
use std::sync::Arc;
use tracing::{debug, info, warn};
@@ -511,6 +512,83 @@ impl RouterTrait for RouterManager {
fn router_type(&self) -> &'static str {
"manager"
}
// Conversations API delegates
async fn create_conversation(&self, headers: Option<&HeaderMap>, body: &Value) -> Response {
let router = self.select_router_for_request(headers, None);
if let Some(router) = router {
router.create_conversation(headers, body).await
} else {
(
StatusCode::NOT_FOUND,
"No router available to create conversation",
)
.into_response()
}
}
async fn get_conversation(
&self,
headers: Option<&HeaderMap>,
conversation_id: &str,
) -> Response {
let router = self.select_router_for_request(headers, None);
if let Some(router) = router {
router.get_conversation(headers, conversation_id).await
} else {
(
StatusCode::NOT_FOUND,
format!(
"No router available to get conversation '{}'",
conversation_id
),
)
.into_response()
}
}
async fn update_conversation(
&self,
headers: Option<&HeaderMap>,
conversation_id: &str,
body: &Value,
) -> Response {
let router = self.select_router_for_request(headers, None);
if let Some(router) = router {
router
.update_conversation(headers, conversation_id, body)
.await
} else {
(
StatusCode::NOT_FOUND,
format!(
"No router available to update conversation '{}'",
conversation_id
),
)
.into_response()
}
}
async fn delete_conversation(
&self,
headers: Option<&HeaderMap>,
conversation_id: &str,
) -> Response {
let router = self.select_router_for_request(headers, None);
if let Some(router) = router {
router.delete_conversation(headers, conversation_id).await
} else {
(
StatusCode::NOT_FOUND,
format!(
"No router available to delete conversation '{}'",
conversation_id
),
)
.into_response()
}
}
}
impl std::fmt::Debug for RouterManager {