[router] Support multiple worker URLs for OpenAI router (#11723)

This commit is contained in:
Keyang Ru
2025-10-22 09:27:58 -07:00
committed by GitHub
parent 1d097aac87
commit 77258ce039
9 changed files with 426 additions and 150 deletions

View File

@@ -8,8 +8,8 @@ use serde_json::Value;
// Import shared types from common module
use super::common::{
default_true, ChatLogProbs, GenerationRequest, PromptTokenUsageInfo, StringOrArray, ToolChoice,
UsageInfo,
default_model, default_true, ChatLogProbs, GenerationRequest, PromptTokenUsageInfo,
StringOrArray, ToolChoice, UsageInfo,
};
// ============================================================================
@@ -452,9 +452,9 @@ pub struct ResponsesRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<HashMap<String, Value>>,
/// Model to use (optional to match vLLM)
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
/// Model to use
#[serde(default = "default_model")]
pub model: String,
/// Optional conversation id to persist input/output as items
#[serde(skip_serializing_if = "Option::is_none")]
@@ -565,7 +565,7 @@ impl Default for ResponsesRequest {
max_output_tokens: None,
max_tool_calls: None,
metadata: None,
model: None,
model: default_model(),
conversation: None,
parallel_tool_calls: None,
previous_response_id: None,
@@ -598,7 +598,7 @@ impl GenerationRequest for ResponsesRequest {
}
fn get_model(&self) -> Option<&str> {
self.model.as_deref()
Some(self.model.as_str())
}
fn extract_text_for_routing(&self) -> String {