EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router][tool call] Improve normal content extraction and error handling (non-stream) (#11050)

Browse Source
This commit is contained in:
Chang Su
2025-09-29 00:19:30 -07:00
committed by GitHub
parent 11965b0daf
commit af4ab65606
21 changed files with 306 additions and 1077 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
sgl-router/tests/tool_parser_deepseek.rs
View File

@@ -13,8 +13,9 @@ async fn test_deepseek_complete_parsing() {
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>
The weather in Tokyo is..."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Let me help you with that.\n");
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -140,25 +141,6 @@ async fn test_deepseek_malformed_json_handling() {
assert_eq!(tools[0].function.name, "valid");
}
#[tokio::test]
async fn test_normal_text_extraction() {
let parser = DeepSeekParser::new();
// Python extracts text before tool calls as normal_text
let input = r#"Let me help you with that.
<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
```json
{"location": "Tokyo"}
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
// TODO: Verify normal text extraction when parser returns it
// In Python: normal_text = "Let me help you with that."
}
#[tokio::test]
async fn test_multiple_tool_calls() {
let parser = DeepSeekParser::new();

27
sgl-router/tests/tool_parser_fallback.rs
View File

@@ -111,19 +111,25 @@ async fn test_mistral_parser_invalid_format_returns_as_normal_text() {
async fn test_deepseek_parser_invalid_format_returns_as_normal_text() {
let parser = DeepSeekParser::new();
// Invalid JSON after emoji marker
let input = r#"🤔[{"name": "test", "arguments": malformed}]"#;
// Invalid JSON in tool call
let input = r#"Some text<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>test
```json
{"name": "test", "arguments": malformed}
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
assert_eq!(normal_text, input); // Should preserve original text when parsing fails
// Emoji but no JSON array
let input = "🤔 Just thinking about this problem...";
// Missing function marker
let input = r#"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>notfunction<｜tool▁sep｜>test
```json
{"x": 1}
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
assert_eq!(normal_text, input); // Should return original text
assert_eq!(normal_text, input); // Should return original text when parsing fails
// No emoji marker at all
// No tool markers at all
let input = "Regular response without any special markers.";
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
@@ -148,9 +154,8 @@ That's all!"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1); // Should extract the valid tool
assert_eq!(tools[0].function.name, "valid_tool");
// Normal text should contain the text around the valid tool call
assert!(normal_text.contains("Let me help you"));
assert!(normal_text.contains("That's all!"));
// Normal text should contain text before the first tool call
assert_eq!(normal_text, "Let me help you with that.\n");
}
#[tokio::test]
@@ -208,8 +213,8 @@ async fn test_unicode_and_special_chars_in_failed_parsing() {
</tool_call>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
// Should handle Unicode properly in the fallback text
assert!(!normal_text.is_empty() || normal_text == input);
// Should handle Unicode properly in the fallback text - malformed content should be preserved
assert_eq!(normal_text, input);
// Special characters that might confuse parsers
let input = r#"Response: <tool_call>{"name": "test\n\t", "arguments": {"]}"}</tool_call>"#;

9
sgl-router/tests/tool_parser_glm4_moe.rs
View File

@@ -15,8 +15,9 @@ async fn test_glm4_complete_parsing() {
</tool_call>
The weather will be..."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Let me search for that.\n");
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -39,8 +40,9 @@ async fn test_glm4_multiple_tools() {
<arg_value>zh</arg_value>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
@@ -62,8 +64,9 @@ async fn test_glm4_type_conversion() {
<arg_value>string value</arg_value>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["count"], 42);

66
sgl-router/tests/tool_parser_json.rs
View File

@@ -3,15 +3,16 @@
//! Tests for the JSON parser which handles OpenAI, Claude, and generic JSON formats
use serde_json::json;
use sglang_router_rs::tool_parser::{JsonParser, ToolParser};
use sglang_router_rs::tool_parser::{JsonParser, TokenConfig, ToolParser};
#[tokio::test]
async fn test_simple_json_tool_call() {
let parser = JsonParser::new();
let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -21,13 +22,14 @@ async fn test_simple_json_tool_call() {
#[tokio::test]
async fn test_json_array_of_tools() {
let parser = JsonParser::new();
let input = r#"[
let input = r#"Hello, here are the results: [
{"name": "get_weather", "arguments": {"location": "SF"}},
{"name": "search", "arguments": {"query": "news"}}
]"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "Hello, here are the results: ");
assert_eq!(tools[0].function.name, "get_weather");
assert_eq!(tools[1].function.name, "search");
}
@@ -37,8 +39,9 @@ async fn test_json_with_parameters_key() {
let parser = JsonParser::new();
let input = r#"{"name": "calculate", "parameters": {"x": 10, "y": 20}}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "calculate");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -51,8 +54,12 @@ async fn test_json_extraction_from_text() {
let parser = JsonParser::new();
let input = r#"I'll help you with that. {"name": "search", "arguments": {"query": "rust"}} Let me search for that."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(
normal_text,
"I'll help you with that. Let me search for that."
);
assert_eq!(tools[0].function.name, "search");
}
@@ -73,8 +80,9 @@ async fn test_json_with_nested_objects() {
}
}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "update_config");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -87,8 +95,9 @@ async fn test_json_with_special_characters() {
let parser = JsonParser::new();
let input = r#"{"name": "echo", "arguments": {"text": "Line 1\nLine 2\tTabbed", "path": "C:\\Users\\test"}}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Line 1\nLine 2\tTabbed");
@@ -100,8 +109,9 @@ async fn test_json_with_unicode() {
let parser = JsonParser::new();
let input = r#"{"name": "translate", "arguments": {"text": "Hello 世界 🌍", "emoji": "😊"}}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Hello 世界 🌍");
@@ -113,8 +123,9 @@ async fn test_json_empty_arguments() {
let parser = JsonParser::new();
let input = r#"{"name": "ping", "arguments": {}}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "ping");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -127,8 +138,12 @@ async fn test_json_invalid_format() {
// Missing closing brace
let input = r#"{"name": "test", "arguments": {"key": "value""#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
assert_eq!(
normal_text,
"{\"name\": \"test\", \"arguments\": {\"key\": \"value\""
);
// Not JSON at all
let input = "This is just plain text";
@@ -145,3 +160,32 @@ async fn test_json_format_detection() {
assert!(!parser.detect_format("plain text"));
assert!(!parser.detect_format(r#"{"key": "value"}"#)); // No name field
}
#[tokio::test]
async fn test_parse_with_wrapper_tokens() {
let parser = JsonParser::with_config(TokenConfig {
start_tokens: vec!["<tool>".to_string()],
end_tokens: vec!["</tool>".to_string()],
separator: ", ".to_string(),
});
let input = r#"<tool>{"name": "test", "arguments": {}}</tool>"#;
let (normal_text, tool_calls) = parser.parse_complete(input).await.unwrap();
assert_eq!(tool_calls.len(), 1);
assert_eq!(tool_calls[0].function.name, "test");
assert_eq!(normal_text, ""); // Wrapper tokens with no extra text
}
#[tokio::test]
async fn test_parse_with_start_token_invalid_json() {
let parser = JsonParser::with_config(TokenConfig {
start_tokens: vec!["<|python_tag|>".to_string()],
end_tokens: vec!["".to_string()],
separator: ";".to_string(),
});
let input = r#"Hello world <|python_tag|>this is not valid json at all"#;
let (normal_text, tool_calls) = parser.parse_complete(input).await.unwrap();
assert_eq!(tool_calls.len(), 0);
assert_eq!(normal_text, input); // Should return entire original text when JSON parsing fails
}

16
sgl-router/tests/tool_parser_kimik2.rs
View File

@@ -12,8 +12,9 @@ async fn test_kimik2_complete_parsing() {
<|tool_calls_section_end|>
The weather in Tokyo is..."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Let me help you with that.\n");
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -30,8 +31,9 @@ async fn test_kimik2_multiple_tools() {
<|tool_call_begin|>functions.translate:1<|tool_call_argument_begin|>{"text": "Hello", "to": "ja"}<|tool_call_end|>
<|tool_calls_section_end|>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
@@ -44,8 +46,9 @@ async fn test_kimik2_with_whitespace() {
<|tool_call_begin|> functions.test:0 <|tool_call_argument_begin|> {"key": "value", "num": 42} <|tool_call_end|>
<|tool_calls_section_end|>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "test");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -117,8 +120,9 @@ async fn test_kimik2_sequential_indices() {
<|tool_call_begin|>functions.third:2<|tool_call_argument_begin|>{"param": "c"}<|tool_call_end|>
<|tool_calls_section_end|>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 3);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "first");
assert_eq!(tools[1].function.name, "second");
assert_eq!(tools[2].function.name, "third");
@@ -134,12 +138,12 @@ async fn test_function_index_extraction() {
<|tool_call_begin|>functions.calc:1<|tool_call_argument_begin|>{"x": 10}<|tool_call_end|>
<|tool_calls_section_end|>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "Text before tool calls.\n");
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "calc");
// TODO: Verify indices are preserved: 0 and 1
// TODO: Verify normal text = "Text before tool calls."
}
#[tokio::test]

6
sgl-router/tests/tool_parser_llama.rs
View File

@@ -36,8 +36,9 @@ async fn test_llama_with_text_before() {
let parser = LlamaParser::new();
let input = r#"Let me help you with that. <|python_tag|>{"name": "get_time", "arguments": {"timezone": "UTC"}}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Let me help you with that. ");
assert_eq!(tools[0].function.name, "get_time");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -99,8 +100,9 @@ async fn test_llama_invalid_json_after_tag() {
let parser = LlamaParser::new();
let input = r#"<|python_tag|>{"name": invalid}"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
assert_eq!(normal_text, "<|python_tag|>{\"name\": invalid}");
}
#[tokio::test]

12
sgl-router/tests/tool_parser_mistral.rs
View File

@@ -11,8 +11,9 @@ async fn test_mistral_single_tool() {
let input = r#"Let me search for that.
[TOOL_CALLS] [{"name": "search_web", "arguments": {"query": "latest news", "max_results": 5}}]"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Let me search for that.\n");
assert_eq!(tools[0].function.name, "search_web");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -29,8 +30,9 @@ async fn test_mistral_multiple_tools() {
{"name": "search_news", "arguments": {"query": "AI developments", "limit": 10}}
]"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "I'll help you with both tasks.\n");
assert_eq!(tools[0].function.name, "get_weather");
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -47,8 +49,9 @@ async fn test_mistral_nested_json() {
let input = r#"Processing complex data.
[TOOL_CALLS] [{"name": "process_data", "arguments": {"config": {"nested": {"value": [1, 2, 3]}}, "enabled": true}}]"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Processing complex data.\n");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["config"]["nested"]["value"], json!([1, 2, 3]));
@@ -146,8 +149,9 @@ async fn test_mistral_real_world_output() {
Let me execute these searches for you."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "I'll search for information about Rust programming and check the weather in San Francisco.\n\n");
assert_eq!(tools[0].function.name, "web_search");
assert_eq!(tools[1].function.name, "get_weather");
}

3
sgl-router/tests/tool_parser_pythonic.rs
View File

@@ -165,8 +165,9 @@ async fn test_pythonic_real_world_llama4() {
These functions will provide the information you need."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 3);
assert_eq!(normal_text, "I'll help you with multiple tasks. Let me search for information and perform calculations.\n\n\n\nThese functions will provide the information you need.");
assert_eq!(tools[0].function.name, "web_search");
assert_eq!(tools[1].function.name, "calculate");
assert_eq!(tools[2].function.name, "get_weather");

12
sgl-router/tests/tool_parser_qwen.rs
View File

@@ -32,8 +32,9 @@ async fn test_qwen_multiple_sequential_tools() {
{"name": "translate", "arguments": {"text": "Hello", "to": "zh"}}
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "Let me help you with that.\n");
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
@@ -79,8 +80,9 @@ Now I'll translate something.
</tool_call>
Done!"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "First, let me search for information.\n");
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
@@ -171,8 +173,12 @@ Let me also calculate something for you:
These tools will provide the information you need."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(
normal_text,
"I'll help you search for information and perform calculations.\n\n"
);
assert_eq!(tools[0].function.name, "web_search");
assert_eq!(tools[1].function.name, "calculator");

6
sgl-router/tests/tool_parser_step3.rs
View File

@@ -15,8 +15,9 @@ async fn test_step3_complete_parsing() {
<｜tool_calls_end｜>
Here are the results..."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Let me help you.\n");
assert_eq!(tools[0].function.name, "search");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
@@ -174,8 +175,9 @@ async fn test_steptml_format() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>Text after."#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Text before.\n");
assert_eq!(tools[0].function.name, "search");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();

9
sgl-router/tests/tool_parser_wrapper_tokens.rs
View File

@@ -118,8 +118,12 @@ async fn test_json_extraction_without_wrapper_tokens() {
And here is some text after.
"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(
normal_text,
"\n Here is some text before the JSON.\n \n And here is some text after.\n "
);
assert_eq!(tools[0].function.name, "search");
}
@@ -143,8 +147,9 @@ async fn test_json_with_multiline_wrapper_content() {
```
Done!"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "format_code");
}