EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router] consolidate health endpoints and flush cache (#10876)

Browse Source
This commit is contained in:
Simo Lin
2025-09-24 18:23:21 -04:00
committed by GitHub
parent 4a87ba217f
commit 7a06ef984d
12 changed files with 220 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
sgl-router/src/routers/http/openai_router.rs
View File

@@ -20,13 +20,7 @@ use axum::{
use bytes::Bytes;
use futures_util::StreamExt;
use serde_json::{json, to_value, Value};
use std::{
any::Any,
borrow::Cow,
collections::HashMap,
io,
sync::atomic::{AtomicBool, Ordering},
};
use std::{any::Any, borrow::Cow, collections::HashMap, io, sync::atomic::AtomicBool};
use tokio::sync::mpsc;
use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::{error, info, warn};
@@ -777,7 +771,7 @@ impl super::super::RouterTrait for OpenAIRouter {
self
}
async fn health(&self, _req: Request<Body>) -> Response {
async fn health_generate(&self, _req: Request<Body>) -> Response {
// Simple upstream probe: GET {base}/v1/models without auth
let url = format!("{}/v1/models", self.base_url);
match self
@@ -808,11 +802,6 @@ impl super::super::RouterTrait for OpenAIRouter {
}
}
async fn health_generate(&self, _req: Request<Body>) -> Response {
// For OpenAI, health_generate is the same as health
self.health(_req).await
}
async fn get_server_info(&self, _req: Request<Body>) -> Response {
let info = json!({
"router_type": "openai",
@@ -1307,14 +1296,6 @@ impl super::super::RouterTrait for OpenAIRouter {
}
}
async fn flush_cache(&self) -> Response {
(
StatusCode::FORBIDDEN,
"flush_cache not supported for OpenAI router",
)
.into_response()
}
async fn get_worker_loads(&self) -> Response {
(
StatusCode::FORBIDDEN,
@@ -1327,14 +1308,6 @@ impl super::super::RouterTrait for OpenAIRouter {
"openai"
}
fn readiness(&self) -> Response {
if self.healthy.load(Ordering::Acquire) && self.circuit_breaker.can_execute() {
(StatusCode::OK, "Ready").into_response()
} else {
(StatusCode::SERVICE_UNAVAILABLE, "Not ready").into_response()
}
}
async fn route_embeddings(
&self,
_headers: Option<&HeaderMap>,