493 lines
15 KiB
JSON
493 lines
15 KiB
JSON
{
|
|
"chat_format": "User:\\n...\\nAssistant:\\n",
|
|
"ckpt": "runs/v10_348m_cone_sft_focus600/sft_ckpt_0000300.pt",
|
|
"code": {
|
|
"N": 6,
|
|
"ast_rate": 1.0,
|
|
"body_rate": 0.0,
|
|
"ci95_exec": [
|
|
0.0,
|
|
0.3903430336530645
|
|
],
|
|
"examples": [
|
|
{
|
|
"ast_ok": true,
|
|
"body_ok": false,
|
|
"code": "def add_one(n):\n \"\"\"Return n plus one.\"\"\"\n",
|
|
"exec_ok": false,
|
|
"generated": " return n + 1",
|
|
"name": "add_one",
|
|
"return_ok": false
|
|
},
|
|
{
|
|
"ast_ok": true,
|
|
"body_ok": false,
|
|
"code": "def reverse_string(s):\n \"\"\"Return s reversed.\"\"\"\n",
|
|
"exec_ok": false,
|
|
"generated": "return s[::-1]",
|
|
"name": "reverse_string",
|
|
"return_ok": false
|
|
},
|
|
{
|
|
"ast_ok": true,
|
|
"body_ok": false,
|
|
"code": "def count_vowels(s):\n \"\"\"Return the number of vowels in s.\"\"\"\n",
|
|
"exec_ok": false,
|
|
"generated": " return sum(1 for ch in s.lower() if ch in 'aeiou')",
|
|
"name": "count_vowels",
|
|
"return_ok": false
|
|
},
|
|
{
|
|
"ast_ok": true,
|
|
"body_ok": false,
|
|
"code": "def is_even(n):\n \"\"\"Return True if n is even.\"\"\"\n",
|
|
"exec_ok": false,
|
|
"generated": " return n % 2 == 0",
|
|
"name": "is_even",
|
|
"return_ok": false
|
|
},
|
|
{
|
|
"ast_ok": true,
|
|
"body_ok": false,
|
|
"code": "def sum_list(items):\n \"\"\"Return the sum of the numbers in items.\"\"\"\n",
|
|
"exec_ok": false,
|
|
"generated": " return sum(items)",
|
|
"name": "sum_list",
|
|
"return_ok": false
|
|
},
|
|
{
|
|
"ast_ok": true,
|
|
"body_ok": false,
|
|
"code": "def first_item(items):\n \"\"\"Return the first item in a non-empty list.\"\"\"\n",
|
|
"exec_ok": false,
|
|
"generated": " return items[0]",
|
|
"name": "first_item",
|
|
"return_ok": false
|
|
}
|
|
],
|
|
"exec_rate": 0.0,
|
|
"return_rate": 0.0
|
|
},
|
|
"device": "cuda",
|
|
"generated_at": "2026-06-22T20:54:24Z",
|
|
"math": {
|
|
"N": 128,
|
|
"answer_anywhere_rate": 0.28125,
|
|
"ci95_anywhere": [
|
|
0.2106120712149715,
|
|
0.36463580199045076
|
|
],
|
|
"examples": [
|
|
{
|
|
"anywhere": true,
|
|
"final": true,
|
|
"first": true,
|
|
"gen": " 6",
|
|
"gold": 6,
|
|
"prompt": "Sam has 4 apples. Lily has 2 more apples than Sam. How many apples does Lily have? Return only the number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 2",
|
|
"gold": 4,
|
|
"prompt": "Mia had 3 red marbles and found 2 blue marbles. Then she gave 1 marble away. Now Mia has how many marbles? Return only the number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 32",
|
|
"gold": 20,
|
|
"prompt": "What is 68 - 48? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 196",
|
|
"gold": 214,
|
|
"prompt": "What is 118 + 96? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 105",
|
|
"gold": 113,
|
|
"prompt": "What is 109 + 4? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 284",
|
|
"gold": 1110,
|
|
"prompt": "What is 74 * 15? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 103",
|
|
"gold": 55,
|
|
"prompt": "What is 97 - 42? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 75",
|
|
"gold": 51,
|
|
"prompt": "What is 102 - 51? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 18",
|
|
"gold": 2,
|
|
"prompt": "What is 22 - 20? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 54",
|
|
"gold": 58,
|
|
"prompt": "What is 26 + 32? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 365",
|
|
"gold": 1469,
|
|
"prompt": "What is 113 * 13? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 44",
|
|
"gold": 43,
|
|
"prompt": "What is 43 * 1? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 109",
|
|
"gold": 103,
|
|
"prompt": "What is 111 - 8? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 110",
|
|
"gold": 122,
|
|
"prompt": "What is 79 + 43? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"final": true,
|
|
"first": true,
|
|
"gen": " 120",
|
|
"gold": 120,
|
|
"prompt": "What is 60 * 2? Return only the final number."
|
|
},
|
|
{
|
|
"anywhere": false,
|
|
"final": false,
|
|
"first": false,
|
|
"gen": " 155",
|
|
"gold": 151,
|
|
"prompt": "What is 72 + 79? Return only the final number."
|
|
}
|
|
],
|
|
"final_numeric_accuracy": 0.28125,
|
|
"first_numeric_accuracy": 0.28125
|
|
},
|
|
"step": 300,
|
|
"transitive": {
|
|
"depth_1": {
|
|
"N": 128,
|
|
"answer_anywhere_rate": 1.0,
|
|
"chance": 0.5,
|
|
"ci95_first": [
|
|
0.9708620041018919,
|
|
0.9999999999999999
|
|
],
|
|
"examples": [
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Dana",
|
|
"first_ok": true,
|
|
"gen": " Dana",
|
|
"gold": "Dana",
|
|
"prompt": "Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Cara",
|
|
"first_ok": true,
|
|
"gen": " Cara",
|
|
"gold": "Cara",
|
|
"prompt": "Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Dana",
|
|
"first_ok": true,
|
|
"gen": " Dana",
|
|
"gold": "Dana",
|
|
"prompt": "Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Gina",
|
|
"first_ok": true,
|
|
"gen": " Gina",
|
|
"gold": "Gina",
|
|
"prompt": "Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Dana",
|
|
"first_ok": true,
|
|
"gen": " Dana",
|
|
"gold": "Dana",
|
|
"prompt": "Dana is taller than Hugo. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Hugo",
|
|
"first_ok": true,
|
|
"gen": " Hugo",
|
|
"gold": "Hugo",
|
|
"prompt": "Dana is taller than Hugo. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Gina",
|
|
"first_ok": true,
|
|
"gen": " Gina",
|
|
"gold": "Gina",
|
|
"prompt": "Gina is taller than Dana. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Dana",
|
|
"first_ok": true,
|
|
"gen": " Dana",
|
|
"gold": "Dana",
|
|
"prompt": "Gina is taller than Dana. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Cara",
|
|
"first_ok": true,
|
|
"gen": " Cara",
|
|
"gold": "Cara",
|
|
"prompt": "Cara is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Finn",
|
|
"first_ok": true,
|
|
"gen": " Finn",
|
|
"gold": "Finn",
|
|
"prompt": "Cara is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
|
}
|
|
],
|
|
"first_name_accuracy": 1.0
|
|
},
|
|
"depth_2": {
|
|
"N": 128,
|
|
"answer_anywhere_rate": 1.0,
|
|
"chance": 0.3333333333333333,
|
|
"ci95_first": [
|
|
0.9708620041018919,
|
|
0.9999999999999999
|
|
],
|
|
"examples": [
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Evan",
|
|
"first_ok": true,
|
|
"gen": " Evan",
|
|
"gold": "Evan",
|
|
"prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Anna",
|
|
"first_ok": true,
|
|
"gen": " Anna",
|
|
"gold": "Anna",
|
|
"prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Ben",
|
|
"first_ok": true,
|
|
"gen": " Ben",
|
|
"gold": "Ben",
|
|
"prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Anna",
|
|
"first_ok": true,
|
|
"gen": " Anna",
|
|
"gold": "Anna",
|
|
"prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Finn",
|
|
"first_ok": true,
|
|
"gen": " Finn",
|
|
"gold": "Finn",
|
|
"prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Gina",
|
|
"first_ok": true,
|
|
"gen": " Gina",
|
|
"gold": "Gina",
|
|
"prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Dana",
|
|
"first_ok": true,
|
|
"gen": " Dana",
|
|
"gold": "Dana",
|
|
"prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Finn",
|
|
"first_ok": true,
|
|
"gen": " Finn",
|
|
"gold": "Finn",
|
|
"prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Gina",
|
|
"first_ok": true,
|
|
"gen": " Gina",
|
|
"gold": "Gina",
|
|
"prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Cara",
|
|
"first_ok": true,
|
|
"gen": " Cara",
|
|
"gold": "Cara",
|
|
"prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
|
|
}
|
|
],
|
|
"first_name_accuracy": 1.0
|
|
},
|
|
"depth_3": {
|
|
"N": 128,
|
|
"answer_anywhere_rate": 1.0,
|
|
"chance": 0.25,
|
|
"ci95_first": [
|
|
0.9708620041018919,
|
|
0.9999999999999999
|
|
],
|
|
"examples": [
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Finn",
|
|
"first_ok": true,
|
|
"gen": " Finn",
|
|
"gold": "Finn",
|
|
"prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Ben",
|
|
"first_ok": true,
|
|
"gen": " Ben",
|
|
"gold": "Ben",
|
|
"prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Ben",
|
|
"first_ok": true,
|
|
"gen": " Ben",
|
|
"gold": "Ben",
|
|
"prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Finn",
|
|
"first_ok": true,
|
|
"gen": " Finn",
|
|
"gold": "Finn",
|
|
"prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Hugo",
|
|
"first_ok": true,
|
|
"gen": " Hugo",
|
|
"gold": "Hugo",
|
|
"prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Gina",
|
|
"first_ok": true,
|
|
"gen": " Gina",
|
|
"gold": "Gina",
|
|
"prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Hugo",
|
|
"first_ok": true,
|
|
"gen": " Hugo",
|
|
"gold": "Hugo",
|
|
"prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Finn",
|
|
"first_ok": true,
|
|
"gen": " Finn",
|
|
"gold": "Finn",
|
|
"prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Anna",
|
|
"first_ok": true,
|
|
"gen": " Anna",
|
|
"gold": "Anna",
|
|
"prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the tallest is who? Return only the name."
|
|
},
|
|
{
|
|
"anywhere": true,
|
|
"first_name": "Finn",
|
|
"first_ok": true,
|
|
"gen": " Finn",
|
|
"gold": "Finn",
|
|
"prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the shortest is who? Return only the name."
|
|
}
|
|
],
|
|
"first_name_accuracy": 1.0
|
|
}
|
|
}
|
|
}
|