[router][tool call] Clean up redundant detect_format and has_tool_markers (#11270)
This commit is contained in:
@@ -108,13 +108,13 @@ fn test_deepseek_format_detection() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
// Should detect DeepSeek format
|
||||
assert!(parser.detect_format("<|tool▁calls▁begin|>"));
|
||||
assert!(parser.detect_format("text with <|tool▁calls▁begin|> marker"));
|
||||
assert!(parser.has_tool_markers("<|tool▁calls▁begin|>"));
|
||||
assert!(parser.has_tool_markers("text with <|tool▁calls▁begin|> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.detect_format("[TOOL_CALLS]"));
|
||||
assert!(!parser.detect_format("<tool_call>"));
|
||||
assert!(!parser.detect_format("plain text"));
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<tool_call>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -117,13 +117,13 @@ fn test_glm4_format_detection() {
|
||||
let parser = Glm4MoeParser::new();
|
||||
|
||||
// Should detect GLM-4 format
|
||||
assert!(parser.detect_format("<tool_call>"));
|
||||
assert!(parser.detect_format("text with <tool_call> marker"));
|
||||
assert!(parser.has_tool_markers("<tool_call>"));
|
||||
assert!(parser.has_tool_markers("text with <tool_call> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.detect_format("[TOOL_CALLS]"));
|
||||
assert!(!parser.detect_format("<|tool▁calls▁begin|>"));
|
||||
assert!(!parser.detect_format("plain text"));
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<|tool▁calls▁begin|>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -109,14 +109,14 @@ fn test_gpt_oss_format_detection() {
|
||||
let parser = GptOssParser::new();
|
||||
|
||||
// Should detect GPT-OSS format
|
||||
assert!(parser.detect_format("<|channel|>commentary to="));
|
||||
assert!(parser.detect_format("<|channel|>commentary"));
|
||||
assert!(parser.detect_format("text with <|channel|>commentary to= marker"));
|
||||
assert!(parser.has_tool_markers("<|channel|>commentary to="));
|
||||
assert!(parser.has_tool_markers("<|channel|>commentary"));
|
||||
assert!(parser.has_tool_markers("text with <|channel|>commentary to= marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.detect_format("[TOOL_CALLS]"));
|
||||
assert!(!parser.detect_format("<tool_call>"));
|
||||
assert!(!parser.detect_format("plain text"));
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<tool_call>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -155,7 +155,7 @@ async fn test_json_invalid_format() {
|
||||
async fn test_json_format_detection() {
|
||||
let parser = JsonParser::new();
|
||||
|
||||
assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#));
|
||||
assert!(parser.detect_format(r#"[{"name": "test"}]"#));
|
||||
assert!(!parser.detect_format("plain text"));
|
||||
assert!(parser.has_tool_markers(r#"{"name": "test", "arguments": {}}"#));
|
||||
assert!(parser.has_tool_markers(r#"[{"name": "test"}]"#));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
@@ -98,14 +98,13 @@ fn test_kimik2_format_detection() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
// Should detect Kimi K2 format
|
||||
assert!(parser.detect_format("<|tool_calls_section_begin|>"));
|
||||
assert!(parser.detect_format("<|tool_call_begin|>"));
|
||||
assert!(parser.detect_format("text with <|tool_calls_section_begin|> marker"));
|
||||
assert!(parser.has_tool_markers("<|tool_calls_section_begin|>"));
|
||||
assert!(parser.has_tool_markers("text with <|tool_calls_section_begin|> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.detect_format("[TOOL_CALLS]"));
|
||||
assert!(!parser.detect_format("<tool_call>"));
|
||||
assert!(!parser.detect_format("plain text"));
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<tool_call>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -116,10 +116,10 @@ async fn test_llama_empty_arguments() {
|
||||
async fn test_llama_format_detection() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
assert!(parser.detect_format(r#"<|python_tag|>{"name": "test"}"#));
|
||||
assert!(parser.detect_format(r#"{"name": "test", "parameters": {}}"#));
|
||||
assert!(!parser.detect_format("plain text"));
|
||||
assert!(!parser.detect_format(r#"{"key": "value"}"#)); // No name field
|
||||
assert!(parser.has_tool_markers(r#"<|python_tag|>{"name": "test"}"#));
|
||||
assert!(parser.has_tool_markers(r#"{"name": "test", "parameters": {}}"#));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
assert!(!parser.has_tool_markers(r#"{"key": "value"}"#)); // No name field
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -96,10 +96,10 @@ async fn test_mistral_with_brackets_in_strings() {
|
||||
async fn test_mistral_format_detection() {
|
||||
let parser = MistralParser::new();
|
||||
|
||||
assert!(parser.detect_format("[TOOL_CALLS] ["));
|
||||
assert!(parser.detect_format("Some text [TOOL_CALLS] ["));
|
||||
assert!(!parser.detect_format("Just plain text"));
|
||||
assert!(!parser.detect_format("[{\"name\": \"test\"}]")); // JSON array without TOOL_CALLS
|
||||
assert!(parser.has_tool_markers("[TOOL_CALLS] ["));
|
||||
assert!(parser.has_tool_markers("Some text [TOOL_CALLS] ["));
|
||||
assert!(!parser.has_tool_markers("Just plain text"));
|
||||
assert!(!parser.has_tool_markers("[{\"name\": \"test\"}]")); // JSON array without TOOL_CALLS
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -125,10 +125,10 @@ async fn test_pythonic_empty_arguments() {
|
||||
async fn test_pythonic_format_detection() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
assert!(!parser.detect_format("[function_name(")); // Incomplete
|
||||
assert!(parser.detect_format("[get_weather(city=\"NYC\")]"));
|
||||
assert!(!parser.detect_format("Just plain text"));
|
||||
assert!(!parser.detect_format("{\"name\": \"test\"}")); // JSON
|
||||
assert!(!parser.has_tool_markers("[function_name(")); // Incomplete
|
||||
assert!(parser.has_tool_markers("[get_weather(city=\"NYC\")]"));
|
||||
assert!(!parser.has_tool_markers("Just plain text"));
|
||||
assert!(!parser.has_tool_markers("{\"name\": \"test\"}")); // JSON
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -120,10 +120,10 @@ async fn test_qwen_with_newlines_in_strings() {
|
||||
async fn test_qwen_format_detection() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
assert!(parser.detect_format("<tool_call>"));
|
||||
assert!(parser.detect_format("Some text <tool_call>\n{"));
|
||||
assert!(!parser.detect_format("Just plain text"));
|
||||
assert!(!parser.detect_format("{\"name\": \"test\"}")); // Plain JSON
|
||||
assert!(parser.has_tool_markers("<tool_call>"));
|
||||
assert!(parser.has_tool_markers("Some text <tool_call>\n{"));
|
||||
assert!(!parser.has_tool_markers("Just plain text"));
|
||||
assert!(!parser.has_tool_markers("{\"name\": \"test\"}")); // Plain JSON
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -111,13 +111,13 @@ fn test_step3_format_detection() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
// Should detect Step3 format
|
||||
assert!(parser.detect_format("<|tool_calls_begin|>"));
|
||||
assert!(parser.detect_format("text with <|tool_calls_begin|> marker"));
|
||||
assert!(parser.has_tool_markers("<|tool_calls_begin|>"));
|
||||
assert!(parser.has_tool_markers("text with <|tool_calls_begin|> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.detect_format("[TOOL_CALLS]"));
|
||||
assert!(!parser.detect_format("<tool_call>"));
|
||||
assert!(!parser.detect_format("plain text"));
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<tool_call>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user