EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router] forward stream_options in request (#7860)

Browse Source
This commit is contained in:
Shuaiyi Zhang
2025-07-08 23:03:38 +08:00
committed by GitHub
parent 653b873b91
commit 51ae40306a
3 changed files with 21 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
sgl-router/benches/request_processing.rs
View File

@@ -59,6 +59,7 @@ fn create_sample_chat_completion_request() -> ChatCompletionRequest {
top_p: Some(1.0), top_p: Some(1.0),
n: Some(1), n: Some(1),
stream: false, stream: false,
stream_options: None,
stop: None, stop: None,
presence_penalty: Some(0.0), presence_penalty: Some(0.0),
frequency_penalty: Some(0.0), frequency_penalty: Some(0.0),
@@ -86,6 +87,7 @@ fn create_sample_completion_request() -> CompletionRequest {
top_p: Some(1.0), top_p: Some(1.0),
n: Some(1), n: Some(1),
stream: false, stream: false,
stream_options: None,
logprobs: None, logprobs: None,
echo: false, echo: false,
stop: None, stop: None,
@@ -130,6 +132,7 @@ fn create_large_chat_completion_request() -> ChatCompletionRequest {
top_p: Some(0.95), top_p: Some(0.95),
n: Some(1), n: Some(1),
stream: false, stream: false,
stream_options: None,
stop: None, stop: None,
presence_penalty: Some(0.1), presence_penalty: Some(0.1),
frequency_penalty: Some(0.1), frequency_penalty: Some(0.1),

14
sgl-router/src/openai_api_types.rs
View File

@@ -52,6 +52,10 @@ pub struct CompletionRequest {
#[serde(default)] #[serde(default)]
pub stream: bool, pub stream: bool,
/// Options for streaming response
#[serde(skip_serializing_if = "Option::is_none")]
pub stream_options: Option<StreamOptions>,
/// Include the log probabilities on the logprobs most likely tokens /// Include the log probabilities on the logprobs most likely tokens
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub logprobs: Option<u32>, pub logprobs: Option<u32>,
@@ -132,6 +136,10 @@ pub struct ChatCompletionRequest {
#[serde(default)] #[serde(default)]
pub stream: bool, pub stream: bool,
/// Options for streaming response
#[serde(skip_serializing_if = "Option::is_none")]
pub stream_options: Option<StreamOptions>,
/// Up to 4 sequences where the API will stop generating further tokens /// Up to 4 sequences where the API will stop generating further tokens
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub stop: Option<StringOrArray>, pub stop: Option<StringOrArray>,
@@ -258,6 +266,12 @@ pub struct ImageUrl {
pub detail: Option<String>, // "auto", "low", or "high" pub detail: Option<String>, // "auto", "low", or "high"
} }
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct StreamOptions {
#[serde(skip_serializing_if = "Option::is_none")]
pub include_usage: Option<bool>,
}
#[derive(Debug, Clone, Deserialize, Serialize)] #[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(tag = "type")] #[serde(tag = "type")]
pub enum ResponseFormat { pub enum ResponseFormat {

4
sgl-router/tests/benchmark_integration.rs
View File

@@ -50,6 +50,7 @@ fn test_benchmark_request_creation() {
top_p: Some(1.0), top_p: Some(1.0),
n: Some(1), n: Some(1),
stream: false, stream: false,
stream_options: None,
stop: None, stop: None,
presence_penalty: Some(0.0), presence_penalty: Some(0.0),
frequency_penalty: Some(0.0), frequency_penalty: Some(0.0),
@@ -75,6 +76,7 @@ fn test_benchmark_request_creation() {
top_p: Some(1.0), top_p: Some(1.0),
n: Some(1), n: Some(1),
stream: false, stream: false,
stream_options: None,
logprobs: None, logprobs: None,
echo: false, echo: false,
stop: None, stop: None,
@@ -143,6 +145,7 @@ fn test_benchmark_request_adaptation() {
top_p: Some(1.0), top_p: Some(1.0),
n: Some(1), n: Some(1),
stream: false, stream: false,
stream_options: None,
stop: None, stop: None,
presence_penalty: Some(0.0), presence_penalty: Some(0.0),
frequency_penalty: Some(0.0), frequency_penalty: Some(0.0),
@@ -168,6 +171,7 @@ fn test_benchmark_request_adaptation() {
top_p: Some(1.0), top_p: Some(1.0),
n: Some(1), n: Some(1),
stream: false, stream: false,
stream_options: None,
logprobs: None, logprobs: None,
echo: false, echo: false,
stop: None, stop: None,