初始化项目，由ModelHub XC社区提供模型
Model: karthik-2905/AL1-model-B Source: Original Platform
This commit is contained in:
ModelHub XC
210
eval/tools_base.json
Normal file
210
eval/tools_base.json
Normal file
@@ -0,0 +1,210 @@
|
||||
{
|
||||
"summary": {
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"adapter": null,
|
||||
"n": 12,
|
||||
"name_acc": 0.5,
|
||||
"full_acc": 0.5
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"query": "Show me the contents of config.py",
|
||||
"expected": {
|
||||
"name": "read_file",
|
||||
"arguments": {
|
||||
"path": "config.py"
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"name": "read_file",
|
||||
"arguments": {
|
||||
"path": "config.py"
|
||||
}
|
||||
},
|
||||
"raw": "<tool_call>\n{\"name\": \"read_file\", \"arguments\": {\"path\": \"config.py\"}}\n</tool_call>",
|
||||
"name_ok": true,
|
||||
"full_ok": true
|
||||
},
|
||||
{
|
||||
"query": "What files are in the src directory?",
|
||||
"expected": {
|
||||
"name": "list_dir",
|
||||
"arguments": {
|
||||
"path": "src"
|
||||
}
|
||||
},
|
||||
"got": null,
|
||||
"raw": "The files in the `src` directory are: `index.html`, `styles.css`, and `script.js`.",
|
||||
"name_ok": false,
|
||||
"full_ok": false
|
||||
},
|
||||
{
|
||||
"query": "Create a file notes.txt containing the text hello world",
|
||||
"expected": {
|
||||
"name": "write_file",
|
||||
"arguments": {
|
||||
"path": "notes.txt",
|
||||
"content": "hello world"
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"name": "write_file",
|
||||
"arguments": {
|
||||
"path": "notes.txt",
|
||||
"content": "hello world"
|
||||
}
|
||||
},
|
||||
"raw": "<tool_call>\n{\"name\": \"write_file\", \"arguments\": {\"path\": \"notes.txt\", \"content\": \"hello world\"}}\n</tool_call>",
|
||||
"name_ok": true,
|
||||
"full_ok": true
|
||||
},
|
||||
{
|
||||
"query": "Run the test suite using pytest",
|
||||
"expected": {
|
||||
"name": "run_bash",
|
||||
"arguments": {
|
||||
"command": "pytest"
|
||||
}
|
||||
},
|
||||
"got": null,
|
||||
"raw": "I cannot run a test suite using pytest directly. However, you can run tests using the `pytest` command in the terminal. Please make sure you have the `pytest` package installed and the test files are in the correct directory.",
|
||||
"name_ok": false,
|
||||
"full_ok": false
|
||||
},
|
||||
{
|
||||
"query": "In app.py replace 5000 with 8080",
|
||||
"expected": {
|
||||
"name": "edit_file",
|
||||
"arguments": {
|
||||
"path": "app.py",
|
||||
"old_string": "5000",
|
||||
"new_string": "8080"
|
||||
}
|
||||
},
|
||||
"got": null,
|
||||
"raw": "None of the provided functions can be used to replace a number in a file. The functions available are for reading, writing, editing, listing, and running shell commands.",
|
||||
"name_ok": false,
|
||||
"full_ok": false
|
||||
},
|
||||
{
|
||||
"query": "Read the README.md file for me",
|
||||
"expected": {
|
||||
"name": "read_file",
|
||||
"arguments": {
|
||||
"path": "README.md"
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"name": "read_file",
|
||||
"arguments": {
|
||||
"path": "README.md"
|
||||
}
|
||||
},
|
||||
"raw": "<tool_call>\n{\"name\": \"read_file\", \"arguments\": {\"path\": \"README.md\"}}\n</tool_call>",
|
||||
"name_ok": true,
|
||||
"full_ok": true
|
||||
},
|
||||
{
|
||||
"query": "List everything in the current directory .",
|
||||
"expected": {
|
||||
"name": "list_dir",
|
||||
"arguments": {
|
||||
"path": "."
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"name": "list_dir",
|
||||
"arguments": {
|
||||
"path": "."
|
||||
}
|
||||
},
|
||||
"raw": "<tool_call>\n{\"name\": \"list_dir\", \"arguments\": {\"path\": \".\"}}\n</tool_call>",
|
||||
"name_ok": true,
|
||||
"full_ok": true
|
||||
},
|
||||
{
|
||||
"query": "Execute git status",
|
||||
"expected": {
|
||||
"name": "run_bash",
|
||||
"arguments": {
|
||||
"command": "git status"
|
||||
}
|
||||
},
|
||||
"got": null,
|
||||
"raw": "I don't have access to the git command line interface, but you can run the following command in your terminal to execute git status:\n\ngit status",
|
||||
"name_ok": false,
|
||||
"full_ok": false
|
||||
},
|
||||
{
|
||||
"query": "Make a new file hello.py that prints hi",
|
||||
"expected": {
|
||||
"name": "write_file",
|
||||
"arguments": {
|
||||
"path": "hello.py",
|
||||
"content": "print('hi')"
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"name": "write_file",
|
||||
"arguments": {
|
||||
"path": "hello.py",
|
||||
"content": "print('hi')"
|
||||
}
|
||||
},
|
||||
"raw": "<tool_call>\n{\"name\": \"write_file\", \"arguments\": {\"path\": \"hello.py\", \"content\": \"print('hi')\"}}\n</tool_call>",
|
||||
"name_ok": true,
|
||||
"full_ok": true
|
||||
},
|
||||
{
|
||||
"query": "Open and show src/utils.py",
|
||||
"expected": {
|
||||
"name": "read_file",
|
||||
"arguments": {
|
||||
"path": "src/utils.py"
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"name": "list_dir",
|
||||
"arguments": {
|
||||
"path": "src"
|
||||
}
|
||||
},
|
||||
"raw": "<tool_call>\n{\"name\": \"list_dir\", \"arguments\": {\"path\": \"src\"}}\n</tool_call>",
|
||||
"name_ok": false,
|
||||
"full_ok": false
|
||||
},
|
||||
{
|
||||
"query": "In main.py change the word foo to bar",
|
||||
"expected": {
|
||||
"name": "edit_file",
|
||||
"arguments": {
|
||||
"path": "main.py",
|
||||
"old_string": "foo",
|
||||
"new_string": "bar"
|
||||
}
|
||||
},
|
||||
"got": null,
|
||||
"raw": "None of the provided functions can be used to change the word \"foo\" to \"bar\" in a file. The available functions are for reading, writing, editing, listing, and running shell commands.",
|
||||
"name_ok": false,
|
||||
"full_ok": false
|
||||
},
|
||||
{
|
||||
"query": "Show the entries in the tests folder",
|
||||
"expected": {
|
||||
"name": "list_dir",
|
||||
"arguments": {
|
||||
"path": "tests"
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"name": "list_dir",
|
||||
"arguments": {
|
||||
"path": "tests"
|
||||
}
|
||||
},
|
||||
"raw": "<tool_call>\n{\"name\": \"list_dir\", \"arguments\": {\"path\": \"tests\"}}\n</tool_call>",
|
||||
"name_ok": true,
|
||||
"full_ok": true
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user