AL1-model-B/eval/tools_sft1ep.json

{
  "summary": {
    "model": "Qwen/Qwen3-0.6B",
    "adapter": "adapters/sft-lora-1ep",
    "n": 12,
    "name_acc": 1.0,
    "full_acc": 0.917
  },
  "results": [
    {
      "query": "Show me the contents of config.py",
      "expected": {
        "name": "read_file",
        "arguments": {
          "path": "config.py"
        }
      },
      "got": {
        "name": "read_file",
        "arguments": {
          "path": "config.py"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"read_file\", \"arguments\": {\"path\": \"config.py\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "What files are in the src directory?",
      "expected": {
        "name": "list_dir",
        "arguments": {
          "path": "src"
        }
      },
      "got": {
        "name": "list_dir",
        "arguments": {
          "path": "src"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"list_dir\", \"arguments\": {\"path\": \"src\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "Create a file notes.txt containing the text hello world",
      "expected": {
        "name": "write_file",
        "arguments": {
          "path": "notes.txt",
          "content": "hello world"
        }
      },
      "got": {
        "name": "write_file",
        "arguments": {
          "path": "notes.txt",
          "content": "hello world"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"write_file\", \"arguments\": {\"path\": \"notes.txt\", \"content\": \"hello world\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "Run the test suite using pytest",
      "expected": {
        "name": "run_bash",
        "arguments": {
          "command": "pytest"
        }
      },
      "got": {
        "name": "run_bash",
        "arguments": {
          "command": "pytest"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"run_bash\", \"arguments\": {\"command\": \"pytest\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "In app.py replace 5000 with 8080",
      "expected": {
        "name": "edit_file",
        "arguments": {
          "path": "app.py",
          "old_string": "5000",
          "new_string": "8080"
        }
      },
      "got": {
        "name": "edit_file",
        "arguments": {
          "path": "app.py",
          "old_string": "5000",
          "new_string": "8080"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"edit_file\", \"arguments\": {\"path\": \"app.py\", \"old_string\": \"5000\", \"new_string\": \"8080\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "Read the README.md file for me",
      "expected": {
        "name": "read_file",
        "arguments": {
          "path": "README.md"
        }
      },
      "got": {
        "name": "read_file",
        "arguments": {
          "path": "README.md"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"read_file\", \"arguments\": {\"path\": \"README.md\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "List everything in the current directory .",
      "expected": {
        "name": "list_dir",
        "arguments": {
          "path": "."
        }
      },
      "got": {
        "name": "list_dir",
        "arguments": {
          "path": "."
        }
      },
      "raw": "<tool_call>\n{\"name\": \"list_dir\", \"arguments\": {\"path\": \".\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "Execute git status",
      "expected": {
        "name": "run_bash",
        "arguments": {
          "command": "git status"
        }
      },
      "got": {
        "name": "run_bash",
        "arguments": {
          "command": "git status"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"run_bash\", \"arguments\": {\"command\": \"git status\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "Make a new file hello.py that prints hi",
      "expected": {
        "name": "write_file",
        "arguments": {
          "path": "hello.py",
          "content": "print('hi')"
        }
      },
      "got": {
        "name": "write_file",
        "arguments": {
          "path": "hello.py",
          "content": "hi"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"write_file\", \"arguments\": {\"path\": \"hello.py\", \"content\": \"hi\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": false
    },
    {
      "query": "Open and show src/utils.py",
      "expected": {
        "name": "read_file",
        "arguments": {
          "path": "src/utils.py"
        }
      },
      "got": {
        "name": "read_file",
        "arguments": {
          "path": "src/utils.py"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"read_file\", \"arguments\": {\"path\": \"src/utils.py\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "In main.py change the word foo to bar",
      "expected": {
        "name": "edit_file",
        "arguments": {
          "path": "main.py",
          "old_string": "foo",
          "new_string": "bar"
        }
      },
      "got": {
        "name": "edit_file",
        "arguments": {
          "path": "main.py",
          "old_string": "foo",
          "new_string": "bar"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"edit_file\", \"arguments\": {\"path\": \"main.py\", \"old_string\": \"foo\", \"new_string\": \"bar\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    },
    {
      "query": "Show the entries in the tests folder",
      "expected": {
        "name": "list_dir",
        "arguments": {
          "path": "tests"
        }
      },
      "got": {
        "name": "list_dir",
        "arguments": {
          "path": "tests"
        }
      },
      "raw": "<tool_call>\n{\"name\": \"list_dir\", \"arguments\": {\"path\": \"tests\"}}\n</tool_call>",
      "name_ok": true,
      "full_ok": true
    }
  ]
}