[router] Add OpenAI backend support - core function (#10254)

This commit is contained in:
Keyang Ru
2025-09-11 14:13:51 -07:00
committed by GitHub
parent ab795ae840
commit dee197e11b
12 changed files with 1158 additions and 16 deletions

View File

@@ -101,6 +101,11 @@ pub enum RoutingMode {
#[serde(skip_serializing_if = "Option::is_none")]
decode_policy: Option<PolicyConfig>,
},
#[serde(rename = "openai")]
OpenAI {
/// OpenAI-compatible API base(s), provided via worker URLs
worker_urls: Vec<String>,
},
}
impl RoutingMode {
@@ -116,6 +121,8 @@ impl RoutingMode {
decode_urls,
..
} => prefill_urls.len() + decode_urls.len(),
// OpenAI mode represents a single upstream
RoutingMode::OpenAI { .. } => 1,
}
}
@@ -380,6 +387,7 @@ impl RouterConfig {
match self.mode {
RoutingMode::Regular { .. } => "regular",
RoutingMode::PrefillDecode { .. } => "prefill_decode",
RoutingMode::OpenAI { .. } => "openai",
}
}

View File

@@ -95,6 +95,20 @@ impl ConfigValidator {
Self::validate_policy(d_policy)?;
}
}
RoutingMode::OpenAI { worker_urls } => {
// Require exactly one worker URL for OpenAI router
if worker_urls.len() != 1 {
return Err(ConfigError::ValidationFailed {
reason: "OpenAI mode requires exactly one --worker-urls entry".to_string(),
});
}
// Validate URL format
if let Err(e) = url::Url::parse(&worker_urls[0]) {
return Err(ConfigError::ValidationFailed {
reason: format!("Invalid OpenAI worker URL '{}': {}", &worker_urls[0], e),
});
}
}
}
Ok(())
}
@@ -243,6 +257,12 @@ impl ConfigValidator {
});
}
}
RoutingMode::OpenAI { .. } => {
// OpenAI mode doesn't use service discovery
return Err(ConfigError::ValidationFailed {
reason: "OpenAI mode does not support service discovery".to_string(),
});
}
}
Ok(())