EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router][grpc] Refine streaming processes (#11277)

Browse Source
This commit is contained in:
Chang Su
2025-10-06 15:15:01 -07:00
committed by GitHub
parent 748f86f3de
commit b07c9c76c5
9 changed files with 77 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sgl-router/src/reasoning_parser/parsers/base.rs
View File

@@ -187,6 +187,10 @@ impl ReasoningParser for BaseReasoningParser {
fn model_type(&self) -> &str {
&self.model_type
}
fn is_in_reasoning(&self) -> bool {
self.in_reasoning
}
}
#[cfg(test)]

4
sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs
View File

@@ -55,6 +55,10 @@ impl ReasoningParser for DeepSeekR1Parser {
fn model_type(&self) -> &str {
self.base.model_type()
}
fn is_in_reasoning(&self) -> bool {
self.base.is_in_reasoning()
}
}
#[cfg(test)]

4
sgl-router/src/reasoning_parser/parsers/glm45.rs
View File

@@ -54,6 +54,10 @@ impl ReasoningParser for Glm45Parser {
fn model_type(&self) -> &str {
self.base.model_type()
}
fn is_in_reasoning(&self) -> bool {
self.base.is_in_reasoning()
}
}
#[cfg(test)]

4
sgl-router/src/reasoning_parser/parsers/kimi.rs
View File

@@ -54,6 +54,10 @@ impl ReasoningParser for KimiParser {
fn model_type(&self) -> &str {
self.base.model_type()
}
fn is_in_reasoning(&self) -> bool {
self.base.is_in_reasoning()
}
}
#[cfg(test)]

8
sgl-router/src/reasoning_parser/parsers/qwen3.rs
View File

@@ -55,6 +55,10 @@ impl ReasoningParser for Qwen3Parser {
fn model_type(&self) -> &str {
self.base.model_type()
}
fn is_in_reasoning(&self) -> bool {
self.base.is_in_reasoning()
}
}
/// QwenThinking parser - variant that assumes reasoning from start.
@@ -106,6 +110,10 @@ impl ReasoningParser for QwenThinkingParser {
fn model_type(&self) -> &str {
self.base.model_type()
}
fn is_in_reasoning(&self) -> bool {
self.base.is_in_reasoning()
}
}
#[cfg(test)]

4
sgl-router/src/reasoning_parser/parsers/step3.rs
View File

@@ -54,6 +54,10 @@ impl ReasoningParser for Step3Parser {
fn model_type(&self) -> &str {
self.base.model_type()
}
fn is_in_reasoning(&self) -> bool {
self.base.is_in_reasoning()
}
}
#[cfg(test)]

5
sgl-router/src/reasoning_parser/traits.rs
View File

@@ -69,6 +69,11 @@ pub trait ReasoningParser: Send + Sync {
/// Get the model type this parser is designed for.
fn model_type(&self) -> &str;
/// Check if the parser is currently in reasoning mode.
///
/// Returns true if the parser is currently parsing reasoning content.
fn is_in_reasoning(&self) -> bool;
}
/// Error types for reasoning parsing operations.

38
sgl-router/src/routers/grpc/pd_router.rs
View File

@@ -922,27 +922,31 @@ impl GrpcPDRouter {
stream_buffer.push_str(&delta);
// Reasoning content handling
if separate_reasoning {
let (normal_text, reasoning_chunk) = router.process_reasoning_stream(
&delta,
index,
&mut reasoning_parsers,
&request_id,
&model,
created,
);
let in_reasoning = if separate_reasoning {
let (normal_text, reasoning_chunk, in_reasoning) = router
.process_reasoning_stream(
&delta,
index,
&mut reasoning_parsers,
&request_id,
&model,
created,
);
if let Some(chunk) = reasoning_chunk {
tx.send(Ok(bytes::Bytes::from(Self::format_sse_chunk(&chunk))))
.map_err(|_| "Failed to send reasoning chunk".to_string())?;
}
delta = normal_text;
}
in_reasoning
} else {
false
};
// Tool call handling
let tool_choice_enabled =
!matches!(tool_choice, Some(ToolChoice::Value(ToolChoiceValue::None)));
if tool_choice_enabled && tools.is_some() {
if !in_reasoning && tool_choice_enabled && tools.is_some() {
let (should_skip, tool_chunks) = router
.process_tool_calls_stream(
&delta,
@@ -1173,16 +1177,18 @@ impl GrpcPDRouter {
request_id: &str,
model: &str,
created: u64,
) -> (String, Option<ChatCompletionStreamResponse>) {
) -> (String, Option<ChatCompletionStreamResponse>, bool) {
// Get or create parser for this index
reasoning_parsers
.entry(index)
.or_insert_with(|| self.reasoning_parser_factory.get_pooled(model));
if let Some(pooled_parser) = reasoning_parsers.get(&index) {
let parse_result = {
let (parse_result, in_reasoning) = {
let mut parser = pooled_parser.lock().unwrap();
parser.parse_reasoning_streaming_incremental(delta)
let result = parser.parse_reasoning_streaming_incremental(delta);
let in_reasoning = parser.is_in_reasoning();
(result, in_reasoning)
};
match parse_result {
@@ -1214,7 +1220,7 @@ impl GrpcPDRouter {
} else {
None
};
return (normal_text, chunk);
return (normal_text, chunk, in_reasoning);
}
Err(e) => {
warn!("Reasoning parsing error: {}", e);
@@ -1222,7 +1228,7 @@ impl GrpcPDRouter {
}
}
(delta.to_string(), None)
(delta.to_string(), None, false)
}
/// Helper: Process tool calls in streaming mode

38
sgl-router/src/routers/grpc/router.rs
View File

@@ -494,16 +494,18 @@ impl GrpcRouter {
request_id: &str,
model: &str,
created: u64,
) -> (String, Option<ChatCompletionStreamResponse>) {
) -> (String, Option<ChatCompletionStreamResponse>, bool) {
// Get or create parser for this index
reasoning_parsers
.entry(index)
.or_insert_with(|| self.reasoning_parser_factory.get_pooled(model));
if let Some(pooled_parser) = reasoning_parsers.get(&index) {
let parse_result = {
let (parse_result, in_reasoning) = {
let mut parser = pooled_parser.lock().unwrap();
parser.parse_reasoning_streaming_incremental(delta)
let result = parser.parse_reasoning_streaming_incremental(delta);
let in_reasoning = parser.is_in_reasoning();
(result, in_reasoning)
};
match parse_result {
@@ -535,7 +537,7 @@ impl GrpcRouter {
} else {
None
};
return (normal_text, chunk);
return (normal_text, chunk, in_reasoning);
}
Err(e) => {
warn!("Reasoning parsing error: {}", e);
@@ -543,7 +545,7 @@ impl GrpcRouter {
}
}
(delta.to_string(), None)
(delta.to_string(), None, false)
}
/// Helper: Process tool calls in streaming mode
@@ -901,27 +903,31 @@ impl GrpcRouter {
stream_buffer.push_str(&delta);
// Reasoning content handling
if separate_reasoning {
let (normal_text, reasoning_chunk) = router.process_reasoning_stream(
&delta,
index,
&mut reasoning_parsers,
&request_id,
&model,
created,
);
let in_reasoning = if separate_reasoning {
let (normal_text, reasoning_chunk, in_reasoning) = router
.process_reasoning_stream(
&delta,
index,
&mut reasoning_parsers,
&request_id,
&model,
created,
);
if let Some(chunk) = reasoning_chunk {
tx.send(Ok(Bytes::from(Self::format_sse_chunk(&chunk))))
.map_err(|_| "Failed to send reasoning chunk".to_string())?;
}
delta = normal_text;
}
in_reasoning
} else {
false
};
// Tool call handling
let tool_choice_enabled =
!matches!(tool_choice, Some(ToolChoice::Value(ToolChoiceValue::None)));
if tool_choice_enabled && tools.is_some() {
if !in_reasoning && tool_choice_enabled && tools.is_some() {
let (should_skip, tool_chunks) = router
.process_tool_calls_stream(
&delta,