EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router][grpc] Refactor chat handler in grpc/ to use centralized orchestrator (#11314)

Browse Source 
Co-authored-by: Simo Lin <linsimo.mark@gmail.com>
This commit is contained in:
Chang Su
2025-10-07 20:50:20 -07:00
committed by GitHub
parent 4b4dc132fa
commit edd86b8853
9 changed files with 2900 additions and 2490 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
sgl-router/src/protocols/spec.rs
View File

@@ -2066,6 +2066,40 @@ impl GenerationRequest for GenerateRequest {
}
}
// TODO(generate): Define GenerateResponse and GenerateChoice structs
//
// Required for pipeline generate response processing (see grpc/pipeline.rs:931-964)
//
// #[derive(Debug, Clone, Serialize, Deserialize)]
// pub struct GenerateResponse {
// pub id: String,
// pub object: String, // "text.completion"
// pub created: u64,
// pub model: String,
// pub choices: Vec<GenerateChoice>,
// #[serde(skip_serializing_if = "Option::is_none")]
// pub usage: Option<Usage>,
// #[serde(skip_serializing_if = "Option::is_none")]
// pub system_fingerprint: Option<String>,
// }
//
// #[derive(Debug, Clone, Serialize, Deserialize)]
// pub struct GenerateChoice {
// pub index: u32,
// pub text: String,
// #[serde(skip_serializing_if = "Option::is_none")]
// pub output_ids: Option<Vec<u32>>,
// #[serde(skip_serializing_if = "Option::is_none")]
// pub finish_reason: Option<String>,
// #[serde(skip_serializing_if = "Option::is_none")]
// pub logprobs: Option<TopLogprobs>,
// #[serde(skip_serializing_if = "Option::is_none")]
// pub matched_stop: Option<Value>,
// }
//
// Note: Verify if similar structs already exist elsewhere before implementing.
// May need streaming variant (GenerateStreamResponse) as well.
// Constants for rerank API
pub const DEFAULT_MODEL_NAME: &str = "default";