初始化项目，由ModelHub XC社区提供模型
Model: ConeML/coneml-348m-alpha-polish900 Source: Original Platform
This commit is contained in:
ModelHub XC
492
evals/chat_activation_focus600_gpu.json
Normal file
492
evals/chat_activation_focus600_gpu.json
Normal file
@@ -0,0 +1,492 @@
|
||||
{
|
||||
"chat_format": "User:\\n...\\nAssistant:\\n",
|
||||
"ckpt": "runs/v10_348m_cone_sft_focus600/sft_ckpt_0000300.pt",
|
||||
"code": {
|
||||
"N": 6,
|
||||
"ast_rate": 1.0,
|
||||
"body_rate": 0.0,
|
||||
"ci95_exec": [
|
||||
0.0,
|
||||
0.3903430336530645
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"ast_ok": true,
|
||||
"body_ok": false,
|
||||
"code": "def add_one(n):\n \"\"\"Return n plus one.\"\"\"\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return n + 1",
|
||||
"name": "add_one",
|
||||
"return_ok": false
|
||||
},
|
||||
{
|
||||
"ast_ok": true,
|
||||
"body_ok": false,
|
||||
"code": "def reverse_string(s):\n \"\"\"Return s reversed.\"\"\"\n",
|
||||
"exec_ok": false,
|
||||
"generated": "return s[::-1]",
|
||||
"name": "reverse_string",
|
||||
"return_ok": false
|
||||
},
|
||||
{
|
||||
"ast_ok": true,
|
||||
"body_ok": false,
|
||||
"code": "def count_vowels(s):\n \"\"\"Return the number of vowels in s.\"\"\"\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return sum(1 for ch in s.lower() if ch in 'aeiou')",
|
||||
"name": "count_vowels",
|
||||
"return_ok": false
|
||||
},
|
||||
{
|
||||
"ast_ok": true,
|
||||
"body_ok": false,
|
||||
"code": "def is_even(n):\n \"\"\"Return True if n is even.\"\"\"\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return n % 2 == 0",
|
||||
"name": "is_even",
|
||||
"return_ok": false
|
||||
},
|
||||
{
|
||||
"ast_ok": true,
|
||||
"body_ok": false,
|
||||
"code": "def sum_list(items):\n \"\"\"Return the sum of the numbers in items.\"\"\"\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return sum(items)",
|
||||
"name": "sum_list",
|
||||
"return_ok": false
|
||||
},
|
||||
{
|
||||
"ast_ok": true,
|
||||
"body_ok": false,
|
||||
"code": "def first_item(items):\n \"\"\"Return the first item in a non-empty list.\"\"\"\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return items[0]",
|
||||
"name": "first_item",
|
||||
"return_ok": false
|
||||
}
|
||||
],
|
||||
"exec_rate": 0.0,
|
||||
"return_rate": 0.0
|
||||
},
|
||||
"device": "cuda",
|
||||
"generated_at": "2026-06-22T20:54:24Z",
|
||||
"math": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 0.28125,
|
||||
"ci95_anywhere": [
|
||||
0.2106120712149715,
|
||||
0.36463580199045076
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"final": true,
|
||||
"first": true,
|
||||
"gen": " 6",
|
||||
"gold": 6,
|
||||
"prompt": "Sam has 4 apples. Lily has 2 more apples than Sam. How many apples does Lily have? Return only the number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 2",
|
||||
"gold": 4,
|
||||
"prompt": "Mia had 3 red marbles and found 2 blue marbles. Then she gave 1 marble away. Now Mia has how many marbles? Return only the number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 32",
|
||||
"gold": 20,
|
||||
"prompt": "What is 68 - 48? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 196",
|
||||
"gold": 214,
|
||||
"prompt": "What is 118 + 96? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 105",
|
||||
"gold": 113,
|
||||
"prompt": "What is 109 + 4? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 284",
|
||||
"gold": 1110,
|
||||
"prompt": "What is 74 * 15? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 103",
|
||||
"gold": 55,
|
||||
"prompt": "What is 97 - 42? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 75",
|
||||
"gold": 51,
|
||||
"prompt": "What is 102 - 51? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 18",
|
||||
"gold": 2,
|
||||
"prompt": "What is 22 - 20? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 54",
|
||||
"gold": 58,
|
||||
"prompt": "What is 26 + 32? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 365",
|
||||
"gold": 1469,
|
||||
"prompt": "What is 113 * 13? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 44",
|
||||
"gold": 43,
|
||||
"prompt": "What is 43 * 1? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 109",
|
||||
"gold": 103,
|
||||
"prompt": "What is 111 - 8? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 110",
|
||||
"gold": 122,
|
||||
"prompt": "What is 79 + 43? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"final": true,
|
||||
"first": true,
|
||||
"gen": " 120",
|
||||
"gold": 120,
|
||||
"prompt": "What is 60 * 2? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 155",
|
||||
"gold": 151,
|
||||
"prompt": "What is 72 + 79? Return only the final number."
|
||||
}
|
||||
],
|
||||
"final_numeric_accuracy": 0.28125,
|
||||
"first_numeric_accuracy": 0.28125
|
||||
},
|
||||
"step": 300,
|
||||
"transitive": {
|
||||
"depth_1": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 1.0,
|
||||
"chance": 0.5,
|
||||
"ci95_first": [
|
||||
0.9708620041018919,
|
||||
0.9999999999999999
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Cara",
|
||||
"first_ok": true,
|
||||
"gen": " Cara",
|
||||
"gold": "Cara",
|
||||
"prompt": "Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Hugo. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Hugo",
|
||||
"first_ok": true,
|
||||
"gen": " Hugo",
|
||||
"gold": "Hugo",
|
||||
"prompt": "Dana is taller than Hugo. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Gina is taller than Dana. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Gina is taller than Dana. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Cara",
|
||||
"first_ok": true,
|
||||
"gen": " Cara",
|
||||
"gold": "Cara",
|
||||
"prompt": "Cara is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Cara is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
}
|
||||
],
|
||||
"first_name_accuracy": 1.0
|
||||
},
|
||||
"depth_2": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 1.0,
|
||||
"chance": 0.3333333333333333,
|
||||
"ci95_first": [
|
||||
0.9708620041018919,
|
||||
0.9999999999999999
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Evan",
|
||||
"first_ok": true,
|
||||
"gen": " Evan",
|
||||
"gold": "Evan",
|
||||
"prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Anna",
|
||||
"first_ok": true,
|
||||
"gen": " Anna",
|
||||
"gold": "Anna",
|
||||
"prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Ben",
|
||||
"first_ok": true,
|
||||
"gen": " Ben",
|
||||
"gold": "Ben",
|
||||
"prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Anna",
|
||||
"first_ok": true,
|
||||
"gen": " Anna",
|
||||
"gold": "Anna",
|
||||
"prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Cara",
|
||||
"first_ok": true,
|
||||
"gen": " Cara",
|
||||
"gold": "Cara",
|
||||
"prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
|
||||
}
|
||||
],
|
||||
"first_name_accuracy": 1.0
|
||||
},
|
||||
"depth_3": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 1.0,
|
||||
"chance": 0.25,
|
||||
"ci95_first": [
|
||||
0.9708620041018919,
|
||||
0.9999999999999999
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Ben",
|
||||
"first_ok": true,
|
||||
"gen": " Ben",
|
||||
"gold": "Ben",
|
||||
"prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Ben",
|
||||
"first_ok": true,
|
||||
"gen": " Ben",
|
||||
"gold": "Ben",
|
||||
"prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Hugo",
|
||||
"first_ok": true,
|
||||
"gen": " Hugo",
|
||||
"gold": "Hugo",
|
||||
"prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Hugo",
|
||||
"first_ok": true,
|
||||
"gen": " Hugo",
|
||||
"gold": "Hugo",
|
||||
"prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Anna",
|
||||
"first_ok": true,
|
||||
"gen": " Anna",
|
||||
"gold": "Anna",
|
||||
"prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
}
|
||||
],
|
||||
"first_name_accuracy": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
497
evals/chat_activation_polish900_gpu.json
Normal file
497
evals/chat_activation_polish900_gpu.json
Normal file
@@ -0,0 +1,497 @@
|
||||
{
|
||||
"chat_format": "User:\\n...\\nAssistant:\\n",
|
||||
"ckpt": "runs/v10_348m_cone_sft_polish900/sft_ckpt_0000300.pt",
|
||||
"code": {
|
||||
"N": 6,
|
||||
"ast_rate": 0.16666666666666666,
|
||||
"body_rate": 0.16666666666666666,
|
||||
"ci95_exec": [
|
||||
0.03005258587173032,
|
||||
0.563509436563646
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"ast_ok": false,
|
||||
"body_ok": false,
|
||||
"code": "def add_one(n):\n \"\"\"Return n plus one.\"\"\"\n return n + 1\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return n + 1",
|
||||
"name": "add_one",
|
||||
"return_ok": false,
|
||||
"syntax_error": "unindent does not match any outer indentation level (<unknown>, line 3)"
|
||||
},
|
||||
{
|
||||
"ast_ok": true,
|
||||
"body_ok": true,
|
||||
"code": "def reverse_string(s):\n \"\"\"Return s reversed.\"\"\"\n return s[::-1]\n",
|
||||
"exec_ok": true,
|
||||
"generated": "return s[::-1]",
|
||||
"name": "reverse_string",
|
||||
"return_ok": true
|
||||
},
|
||||
{
|
||||
"ast_ok": false,
|
||||
"body_ok": false,
|
||||
"code": "def count_vowels(s):\n \"\"\"Return the number of vowels in s.\"\"\"\n return sum(1 for ch in s.lower() if ch in 'aeiou')\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return sum(1 for ch in s.lower() if ch in 'aeiou')",
|
||||
"name": "count_vowels",
|
||||
"return_ok": false,
|
||||
"syntax_error": "unindent does not match any outer indentation level (<unknown>, line 3)"
|
||||
},
|
||||
{
|
||||
"ast_ok": false,
|
||||
"body_ok": false,
|
||||
"code": "def is_even(n):\n \"\"\"Return True if n is even.\"\"\"\n return n % 2 == 0\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return n % 2 == 0",
|
||||
"name": "is_even",
|
||||
"return_ok": false,
|
||||
"syntax_error": "unindent does not match any outer indentation level (<unknown>, line 3)"
|
||||
},
|
||||
{
|
||||
"ast_ok": false,
|
||||
"body_ok": false,
|
||||
"code": "def sum_list(items):\n \"\"\"Return the sum of the numbers in items.\"\"\"\n return sum(items)\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return sum(items)",
|
||||
"name": "sum_list",
|
||||
"return_ok": false,
|
||||
"syntax_error": "unindent does not match any outer indentation level (<unknown>, line 3)"
|
||||
},
|
||||
{
|
||||
"ast_ok": false,
|
||||
"body_ok": false,
|
||||
"code": "def first_item(items):\n \"\"\"Return the first item in a non-empty list.\"\"\"\n return items[0]\n",
|
||||
"exec_ok": false,
|
||||
"generated": " return items[0]",
|
||||
"name": "first_item",
|
||||
"return_ok": false,
|
||||
"syntax_error": "unindent does not match any outer indentation level (<unknown>, line 3)"
|
||||
}
|
||||
],
|
||||
"exec_rate": 0.16666666666666666,
|
||||
"return_rate": 0.16666666666666666
|
||||
},
|
||||
"device": "cuda",
|
||||
"generated_at": "2026-06-22T21:44:35Z",
|
||||
"math": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 0.359375,
|
||||
"ci95_anywhere": [
|
||||
0.281465985317146,
|
||||
0.44547907602919684
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"final": true,
|
||||
"first": true,
|
||||
"gen": " 6",
|
||||
"gold": 6,
|
||||
"prompt": "Sam has 4 apples. Lily has 2 more apples than Sam. How many apples does Lily have? Return only the number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 3",
|
||||
"gold": 4,
|
||||
"prompt": "Mia had 3 red marbles and found 2 blue marbles. Then she gave 1 marble away. Now Mia has how many marbles? Return only the number."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"final": true,
|
||||
"first": true,
|
||||
"gen": " 20",
|
||||
"gold": 20,
|
||||
"prompt": "What is 68 - 48? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 224",
|
||||
"gold": 214,
|
||||
"prompt": "What is 118 + 96? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 111",
|
||||
"gold": 113,
|
||||
"prompt": "What is 109 + 4? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 342",
|
||||
"gold": 1110,
|
||||
"prompt": "What is 74 * 15? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 53",
|
||||
"gold": 55,
|
||||
"prompt": "What is 97 - 42? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 57",
|
||||
"gold": 51,
|
||||
"prompt": "What is 102 - 51? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"final": true,
|
||||
"first": true,
|
||||
"gen": " 2",
|
||||
"gold": 2,
|
||||
"prompt": "What is 22 - 20? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 54",
|
||||
"gold": 58,
|
||||
"prompt": "What is 26 + 32? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 7895",
|
||||
"gold": 1469,
|
||||
"prompt": "What is 113 * 13? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"final": true,
|
||||
"first": true,
|
||||
"gen": " 43",
|
||||
"gold": 43,
|
||||
"prompt": "What is 43 * 1? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 109",
|
||||
"gold": 103,
|
||||
"prompt": "What is 111 - 8? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 150",
|
||||
"gold": 122,
|
||||
"prompt": "What is 79 + 43? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"final": true,
|
||||
"first": true,
|
||||
"gen": " 120",
|
||||
"gold": 120,
|
||||
"prompt": "What is 60 * 2? Return only the final number."
|
||||
},
|
||||
{
|
||||
"anywhere": false,
|
||||
"final": false,
|
||||
"first": false,
|
||||
"gen": " 161",
|
||||
"gold": 151,
|
||||
"prompt": "What is 72 + 79? Return only the final number."
|
||||
}
|
||||
],
|
||||
"final_numeric_accuracy": 0.359375,
|
||||
"first_numeric_accuracy": 0.359375
|
||||
},
|
||||
"step": 300,
|
||||
"transitive": {
|
||||
"depth_1": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 1.0,
|
||||
"chance": 0.5,
|
||||
"ci95_first": [
|
||||
0.9708620041018919,
|
||||
0.9999999999999999
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Cara",
|
||||
"first_ok": true,
|
||||
"gen": " Cara",
|
||||
"gold": "Cara",
|
||||
"prompt": "Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Hugo. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Hugo",
|
||||
"first_ok": true,
|
||||
"gen": " Hugo",
|
||||
"gold": "Hugo",
|
||||
"prompt": "Dana is taller than Hugo. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Gina is taller than Dana. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Gina is taller than Dana. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Cara",
|
||||
"first_ok": true,
|
||||
"gen": " Cara",
|
||||
"gold": "Cara",
|
||||
"prompt": "Cara is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Cara is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
}
|
||||
],
|
||||
"first_name_accuracy": 1.0
|
||||
},
|
||||
"depth_2": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 1.0,
|
||||
"chance": 0.3333333333333333,
|
||||
"ci95_first": [
|
||||
0.9708620041018919,
|
||||
0.9999999999999999
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Evan",
|
||||
"first_ok": true,
|
||||
"gen": " Evan",
|
||||
"gold": "Evan",
|
||||
"prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Anna",
|
||||
"first_ok": true,
|
||||
"gen": " Anna",
|
||||
"gold": "Anna",
|
||||
"prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Ben",
|
||||
"first_ok": true,
|
||||
"gen": " Ben",
|
||||
"gold": "Ben",
|
||||
"prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Anna",
|
||||
"first_ok": true,
|
||||
"gen": " Anna",
|
||||
"gold": "Anna",
|
||||
"prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Dana",
|
||||
"first_ok": true,
|
||||
"gen": " Dana",
|
||||
"gold": "Dana",
|
||||
"prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Cara",
|
||||
"first_ok": true,
|
||||
"gen": " Cara",
|
||||
"gold": "Cara",
|
||||
"prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
|
||||
}
|
||||
],
|
||||
"first_name_accuracy": 1.0
|
||||
},
|
||||
"depth_3": {
|
||||
"N": 128,
|
||||
"answer_anywhere_rate": 1.0,
|
||||
"chance": 0.25,
|
||||
"ci95_first": [
|
||||
0.9708620041018919,
|
||||
0.9999999999999999
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Ben",
|
||||
"first_ok": true,
|
||||
"gen": " Ben",
|
||||
"gold": "Ben",
|
||||
"prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Ben",
|
||||
"first_ok": true,
|
||||
"gen": " Ben",
|
||||
"gold": "Ben",
|
||||
"prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Hugo",
|
||||
"first_ok": true,
|
||||
"gen": " Hugo",
|
||||
"gold": "Hugo",
|
||||
"prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Gina",
|
||||
"first_ok": true,
|
||||
"gen": " Gina",
|
||||
"gold": "Gina",
|
||||
"prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Hugo",
|
||||
"first_ok": true,
|
||||
"gen": " Hugo",
|
||||
"gold": "Hugo",
|
||||
"prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Anna",
|
||||
"first_ok": true,
|
||||
"gen": " Anna",
|
||||
"gold": "Anna",
|
||||
"prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
||||
},
|
||||
{
|
||||
"anywhere": true,
|
||||
"first_name": "Finn",
|
||||
"first_ok": true,
|
||||
"gen": " Finn",
|
||||
"gold": "Finn",
|
||||
"prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
||||
}
|
||||
],
|
||||
"first_name_accuracy": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
1351
evals/diag_focus600_raw_gpu.json
Normal file
1351
evals/diag_focus600_raw_gpu.json
Normal file
File diff suppressed because it is too large
Load Diff
1351
evals/diag_polish900_raw_gpu.json
Normal file
1351
evals/diag_polish900_raw_gpu.json
Normal file
File diff suppressed because it is too large
Load Diff
1351
evals/diag_postsft_sft300_gpu.json
Normal file
1351
evals/diag_postsft_sft300_gpu.json
Normal file
File diff suppressed because it is too large
Load Diff
1351
evals/v10_diagnostic_probe_0210000_gpu_matched_sft300.json
Normal file
1351
evals/v10_diagnostic_probe_0210000_gpu_matched_sft300.json
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user