EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router][tool call] Support normal content extraction before tool call (streaming) (#11038)

Browse Source
This commit is contained in:
Chang Su
2025-09-28 19:46:06 -07:00
committed by GitHub
parent 2e7633982c
commit dba751a896
8 changed files with 157 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
sgl-router/src/tool_parser/parsers/deepseek_parser.rs
View File

@@ -154,8 +154,18 @@ impl ToolParser for DeepSeekParser {
// Check for tool markers
if !self.has_tool_markers(&state.buffer) {
// No markers found, return as incomplete
return Ok(StreamResult::Incomplete);
// No tool markers detected - return all buffered content as normal text
let normal_text = std::mem::take(&mut state.buffer);
return Ok(StreamResult::NormalText(normal_text));
}
// Check for text before tool markers and extract it as normal text
if let Some(marker_pos) = state.buffer.find("<｜tool▁calls▁begin｜>") {
if marker_pos > 0 {
// We have text before the tool marker - extract it as normal text
let normal_text: String = state.buffer.drain(..marker_pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
// Look for start of tool calls
@@ -220,7 +230,7 @@ impl ToolParser for DeepSeekParser {
});
}
Err(_) => {
// Can't parse yet, keep buffering
// Can't parse yet, continue waiting for more data
}
}
}

14
sgl-router/src/tool_parser/parsers/glm4_moe_parser.rs
View File

@@ -177,8 +177,18 @@ impl ToolParser for Glm4MoeParser {
// Check for tool markers
if !self.has_tool_markers(&state.buffer) {
// No markers found, return as incomplete
return Ok(StreamResult::Incomplete);
// No tool markers detected - return all buffered content as normal text
let normal_text = std::mem::take(&mut state.buffer);
return Ok(StreamResult::NormalText(normal_text));
}
// Check for text before tool markers and extract it as normal text
if let Some(marker_pos) = state.buffer.find("<tool_call>") {
if marker_pos > 0 {
// We have text before the tool marker - extract it as normal text
let normal_text: String = state.buffer.drain(..marker_pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
// Look for start of tool call

73
sgl-router/src/tool_parser/parsers/json_parser.rs
View File

@@ -227,10 +227,34 @@ impl JsonParser {
}
// Check for any start token
self.token_config
let has_start_token = self
.token_config
.start_tokens
.iter()
.any(|token| text.contains(token))
.any(|token| text.contains(token));
// Also check if we have what looks like JSON even without start token
// This handles cases where we've already processed the start token
// and are working on subsequent tools
has_start_token || (text.trim_start().starts_with('{') && text.contains(r#""name""#))
}
/// Check if text might contain a partial start token (for streaming)
fn has_partial_start_token(&self, text: &str) -> bool {
if self.token_config.start_tokens.is_empty() {
return false;
}
// Check if the end of the buffer could be the start of any start token
for start_token in &self.token_config.start_tokens {
for i in 1..start_token.len() {
let token_prefix = &start_token[..i];
if text.ends_with(token_prefix) {
return true;
}
}
}
false
}
}
@@ -382,8 +406,42 @@ impl ToolParser for JsonParser {
// Check if we have potential tool calls
if !self.has_tool_markers(&state.buffer) {
// No tool markers, return as incomplete
return Ok(StreamResult::Incomplete);
if self.has_partial_start_token(&state.buffer) {
// We might be in the middle of receiving a start token, wait for more data
return Ok(StreamResult::Incomplete);
}
// No tool markers and no partial tokens - return all buffered content as normal text
let normal_text = std::mem::take(&mut state.buffer);
return Ok(StreamResult::NormalText(normal_text));
}
// Check for text before tool markers and extract it as normal text
if !self.token_config.start_tokens.is_empty() {
let start_token = &self.token_config.start_tokens[0];
if !start_token.is_empty() {
if let Some(marker_pos) = state.buffer.find(start_token) {
if marker_pos > 0 {
// We have text before the tool marker - extract it as normal text
let normal_text: String = state.buffer.drain(..marker_pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
}
} else {
// For JSON without start tokens, look for the start of JSON structure
// Find whichever comes first: '{' or '['
let brace_pos = state.buffer.find('{');
let bracket_pos = state.buffer.find('[');
let json_pos = brace_pos.iter().chain(bracket_pos.iter()).min().copied();
if let Some(pos) = json_pos {
if pos > 0 {
// We have text before JSON structure - extract it as normal text
let normal_text: String = state.buffer.drain(..pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
}
// Extract JSON content first to check for separators
@@ -407,9 +465,8 @@ impl ToolParser for JsonParser {
// We need to figure out how much to remove from the original buffer
// Find where the separator is in the original buffer and remove up to and including it
if let Some(sep_in_original) = state.buffer.find(separator.as_str()) {
let remaining =
state.buffer[sep_in_original + separator.len()..].to_string();
state.buffer = remaining;
// Remove processed content up to and including separator
state.buffer.drain(..=sep_in_original + separator.len() - 1);
}
// Return the first tool as complete
@@ -518,7 +575,7 @@ impl ToolParser for JsonParser {
}
Err(_) => {
// Failed to parse even as partial JSON
// Keep buffering
// Continue waiting for more data
}
}

19
sgl-router/src/tool_parser/parsers/kimik2_parser.rs
View File

@@ -152,9 +152,22 @@ impl ToolParser for KimiK2Parser {
self.has_tool_markers(&state.buffer) || state.buffer.contains("<|tool_call_begin|>");
if !has_tool_call {
// No markers found, clear buffer and return
state.buffer.clear();
return Ok(StreamResult::Incomplete);
// No tool markers detected - return all buffered content as normal text
let normal_text = std::mem::take(&mut state.buffer);
return Ok(StreamResult::NormalText(normal_text));
}
// Check for text before tool markers and extract it as normal text
let marker1_pos = state.buffer.find("<|tool_calls_section_begin|>");
let marker2_pos = state.buffer.find("<|tool_call_begin|>");
let marker_pos = marker1_pos.iter().chain(marker2_pos.iter()).min().copied();
if let Some(pos) = marker_pos {
if pos > 0 {
// We have text before the tool marker - extract it as normal text
let normal_text: String = state.buffer.drain(..pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
// Try to match streaming pattern

16
sgl-router/src/tool_parser/parsers/llama_parser.rs
View File

@@ -75,16 +75,18 @@ impl ToolParser for LlamaParser {
chunk: &str,
state: &mut ParseState,
) -> ToolParserResult<StreamResult> {
// Try with the python_tag parser first
// First, try with the configured json_parser (which handles python_tag)
let result = self.json_parser.parse_incremental(chunk, state).await?;
// If we get Incomplete and buffer starts with '{', might be plain JSON
if matches!(result, StreamResult::Incomplete) && state.buffer.trim_start().starts_with('{')
{
// Check if we have python_tag in the buffer
if !state.buffer.contains("<|python_tag|>") {
// Likely plain JSON, create temporary parser
// If we get Incomplete and no python_tag in buffer, might be plain JSON
if matches!(result, StreamResult::Incomplete) {
let trimmed = state.buffer.trim_start();
if trimmed.starts_with('{') && !state.buffer.contains("<|python_tag|>") {
// Likely plain JSON, try with a plain parser
// Note: We need to be careful not to double-add the chunk
let plain_parser = JsonParser::new();
// The chunk was already added to state.buffer by json_parser above
// So we call with empty string to just process what's in the buffer
return plain_parser.parse_incremental("", state).await;
}
}

13
sgl-router/src/tool_parser/parsers/mistral_parser.rs
View File

@@ -195,7 +195,18 @@ impl ToolParser for MistralParser {
// Check if we have the start marker
if !self.has_tool_markers(&state.buffer) {
return Ok(StreamResult::Incomplete);
// No tool markers detected - return all buffered content as normal text
let normal_text = std::mem::take(&mut state.buffer);
return Ok(StreamResult::NormalText(normal_text));
}
// Check for text before [TOOL_CALLS] and extract it as normal text
if let Some(marker_pos) = state.buffer.find("[TOOL_CALLS]") {
if marker_pos > 0 {
// We have text before the tool marker - extract it as normal text
let normal_text: String = state.buffer.drain(..marker_pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
// Try to extract complete JSON array

20
sgl-router/src/tool_parser/parsers/qwen_parser.rs
View File

@@ -190,7 +190,18 @@ impl ToolParser for QwenParser {
// Check if we have the start marker
if !self.has_tool_markers(&state.buffer) {
return Ok(StreamResult::Incomplete);
// No tool markers detected - return all buffered content as normal text
let normal_text = std::mem::take(&mut state.buffer);
return Ok(StreamResult::NormalText(normal_text));
}
// Check for text before tool markers and extract it as normal text
if let Some(marker_pos) = state.buffer.find("<tool_call>") {
if marker_pos > 0 {
// We have text before the tool marker - extract it as normal text
let normal_text: String = state.buffer.drain(..marker_pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
// Find start and end positions
@@ -212,7 +223,12 @@ impl ToolParser for QwenParser {
}
}
Err(_) => {
// JSON parsing failed, might be incomplete
// JSON parsing failed, might be incomplete or malformed
// If we have what looks like a complete tool call block, treat as normal text
if state.buffer[start_pos..end_pos].contains("\n</tool_call>") {
let malformed_text: String = state.buffer.drain(..end_pos).collect();
return Ok(StreamResult::NormalText(malformed_text));
}
}
}
} else {

14
sgl-router/src/tool_parser/parsers/step3_parser.rs
View File

@@ -209,8 +209,18 @@ impl ToolParser for Step3Parser {
// Check for tool markers
if !self.has_tool_markers(&state.buffer) {
// No markers found, return as incomplete
return Ok(StreamResult::Incomplete);
// No tool markers detected - return all buffered content as normal text
let normal_text = std::mem::take(&mut state.buffer);
return Ok(StreamResult::NormalText(normal_text));
}
// Check for text before tool markers and extract it as normal text
if let Some(marker_pos) = state.buffer.find("<｜tool_calls_begin｜>") {
if marker_pos > 0 {
// We have text before the tool marker - extract it as normal text
let normal_text: String = state.buffer.drain(..marker_pos).collect();
return Ok(StreamResult::NormalText(normal_text));
}
}
// Look for start of tool calls