1352 lines
51 KiB
JSON
1352 lines
51 KiB
JSON
{
|
||
"generated_at": "2026-06-22T20:13:15Z",
|
||
"ckpt": "runs/v10_348m_cone_sft_210k_clean20k/sft_ckpt_0000300.pt",
|
||
"step": 300,
|
||
"device": "cuda",
|
||
"threads": 8,
|
||
"diagnostics": {
|
||
"transitive": {
|
||
"depth_1": {
|
||
"N": 128,
|
||
"chance": 0.5,
|
||
"first_name_accuracy": 0.328125,
|
||
"answer_anywhere_rate": 0.390625,
|
||
"ci95_first_name": [
|
||
0.25282858521157203,
|
||
0.41343760087840264
|
||
],
|
||
"by_type": {
|
||
"tallest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.484375,
|
||
"answer_anywhere_rate": 0.484375,
|
||
"ci95_first_name": [
|
||
0.36633348565783896,
|
||
0.6041860775600096
|
||
]
|
||
},
|
||
"shortest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.171875,
|
||
"answer_anywhere_rate": 0.296875,
|
||
"ci95_first_name": [
|
||
0.0987764126731623,
|
||
0.2821344149016591
|
||
]
|
||
}
|
||
},
|
||
"examples": [
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Finn is taller than Cara. Of all of them, the tallest is",
|
||
"gold": "Finn",
|
||
"gen": " Leo.\n\nAnswer: Leo is the tallest.\nConfidence: high",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Finn is taller than Cara. Of all of them, the shortest is",
|
||
"gold": "Cara",
|
||
"gen": " Leo.\n\nAnswer: Leo is the shortest animal.\nConfidence:",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Hugo is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Hugo",
|
||
"gen": " Pierre.\n\nLet's double-check: We can verify this by checking",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Hugo is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Leo, and the tallest is Leo.\n\nAnswer: Leo is the tallest",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Maria.\n\nAnswer: Maria\nConfidence: high\nConfidence rationale",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Maria, and the tallest is Anna.\n\nAnswer: Maria is taller than",
|
||
"first_name": "Anna",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Ben is taller than Evan. Of all of them, the tallest is",
|
||
"gold": "Ben",
|
||
"gen": " Ben, and the tallest is Evan.\n\nAnswer: Ben is taller",
|
||
"first_name": "Ben",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Ben is taller than Evan. Of all of them, the shortest is",
|
||
"gold": "Evan",
|
||
"gen": " Ben, then Evan, then Alex.\n\nAnswer: Ben is the",
|
||
"first_name": "Ben",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n\nAnswer: Cara is taller than Ben\nConfidence",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Cara.\n\nAnswer: Cara is the shortest student.\nConf",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Cara. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Cara.\n\nAnswer: Cara is taller than both Bella and Bella",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Cara. Of all of them, the shortest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n\nAnswer: Cara is the shortest student.\nConf",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
}
|
||
]
|
||
},
|
||
"depth_2": {
|
||
"N": 128,
|
||
"chance": 0.3333333333333333,
|
||
"first_name_accuracy": 0.359375,
|
||
"answer_anywhere_rate": 0.375,
|
||
"ci95_first_name": [
|
||
0.281465985317146,
|
||
0.44547907602919684
|
||
],
|
||
"by_type": {
|
||
"tallest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.4375,
|
||
"answer_anywhere_rate": 0.4375,
|
||
"ci95_first_name": [
|
||
0.32293817065427327,
|
||
0.5591400822171212
|
||
]
|
||
},
|
||
"shortest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.28125,
|
||
"answer_anywhere_rate": 0.3125,
|
||
"ci95_first_name": [
|
||
0.18593226105484584,
|
||
0.40134162399503504
|
||
]
|
||
}
|
||
},
|
||
"examples": [
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Evan is taller than Finn. Finn is taller than Dana. Of all of them, the tallest is",
|
||
"gold": "Evan",
|
||
"gen": " Evan.\n\nAnswer: Evan is the tallest.\nConfidence",
|
||
"first_name": "Evan",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Evan is taller than Finn. Finn is taller than Dana. Of all of them, the shortest is",
|
||
"gold": "Dana",
|
||
"gen": " Evan.\n\nAnswer: Evan is the shortest.\nConfidence",
|
||
"first_name": "Evan",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Finn. Finn is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n\nReasoning: First, we map the ordered chain",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Finn. Finn is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Cara.\n\nAnswer: Cara is the shortest.\nConfidence",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Evan. Evan is taller than Hugo. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Evan.\n\nAnswer: Evan\nConfidence: high\nConf",
|
||
"first_name": "Evan",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Evan. Evan is taller than Hugo. Of all of them, the shortest is",
|
||
"gold": "Hugo",
|
||
"gen": " Evan, the tallest.\n\nAnswer: Evan is the tallest.",
|
||
"first_name": "Evan",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Evan is taller than Gina. Gina is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Evan",
|
||
"gen": " Evan.\n\nAnswer: Evan is the tallest.\nConfidence",
|
||
"first_name": "Evan",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Evan is taller than Gina. Gina is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Anna.\n\nAnswer: Anna\nConfidence: high\nConfidence rationale",
|
||
"first_name": "Anna",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Cara. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Cara.\n\nAnswer: Cara is the tallest.\nConfidence",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Cara. Of all of them, the shortest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n\nAnswer: Cara\nConfidence: high\nConf",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Dana. Dana is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Leo.\n\nReasoning: First, we list the ordered chain:",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Dana. Dana is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Ben. This confirms the order.\n\nSo the answer is Ben.\n",
|
||
"first_name": "Ben",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
}
|
||
]
|
||
},
|
||
"depth_3": {
|
||
"N": 128,
|
||
"chance": 0.25,
|
||
"first_name_accuracy": 0.125,
|
||
"answer_anywhere_rate": 0.15625,
|
||
"ci95_first_name": [
|
||
0.07842577339785958,
|
||
0.19342772352572143
|
||
],
|
||
"by_type": {
|
||
"tallest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.1875,
|
||
"answer_anywhere_rate": 0.1875,
|
||
"ci95_first_name": [
|
||
0.11064534880922097,
|
||
0.29974591554775176
|
||
]
|
||
},
|
||
"shortest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.0625,
|
||
"answer_anywhere_rate": 0.125,
|
||
"ci95_first_name": [
|
||
0.024570803674792932,
|
||
0.14997696642496888
|
||
]
|
||
}
|
||
},
|
||
"examples": [
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Evan. Evan is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Maria.\n\nReasoning: We are given three statements: (1",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Evan. Evan is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Maria.\n\nReasoning: We are given three ordered chain values:",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Hugo. Hugo is taller than Gina. Gina is taller than Dana. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n\nReasoning: First, we list the ordered chain",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Hugo. Hugo is taller than Gina. Gina is taller than Dana. Of all of them, the shortest is",
|
||
"gold": "Dana",
|
||
"gen": " Gerhard.\n\nReasoning: First, we list the ordered",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Evan. Evan is taller than Ben. Ben is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Evan.\n\nAnswer: Evan is the tallest.\nConfidence",
|
||
"first_name": "Evan",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Evan. Evan is taller than Ben. Ben is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Evan, the tallest.\n\nReasoning: First, we list",
|
||
"first_name": "Evan",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Gina. Gina is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Anna.\n\nReasoning: First, we list the ordered chain:",
|
||
"first_name": "Anna",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Gina. Gina is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Anna.\n\nReasoning: First, list the ordered chain: Anna",
|
||
"first_name": "Anna",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Anna is taller than Cara. Cara is taller than Gina. Gina is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Anna",
|
||
"gen": " Cara.\n\nReasoning: We are given three ordered chain values",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Anna is taller than Cara. Cara is taller than Gina. Gina is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Cara. This confirms that Cara is the tallest.\n\nSo the",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Ben. Ben is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Anna.\n\nReasoning: First, we list the ordered chain:",
|
||
"first_name": "Anna",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Ben. Ben is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Ben. This confirms the order. Ben is the shortest. Confidence: high Confidence",
|
||
"first_name": "Ben",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"math_extraction": {
|
||
"N": 256,
|
||
"by_source": {
|
||
"corpus_v3/L2/math-drill-v1/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.234375,
|
||
"final_numeric_accuracy": 0.21875,
|
||
"answer_anywhere_rate": 0.3125,
|
||
"first_line_answer_anywhere_rate": 0.28125,
|
||
"ci95_answer_anywhere": [
|
||
0.21230909502265743,
|
||
0.43392566359152623
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "Add 7 and 2.",
|
||
"gold": "9",
|
||
"gen": " 13",
|
||
"first_line": "13",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "83 + 49 =",
|
||
"gold": "132",
|
||
"gen": " 102\n\nAnswer: 102",
|
||
"first_line": "102",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "What's 23 plus 91?",
|
||
"gold": "114",
|
||
"gen": " 235",
|
||
"first_line": "235",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "8 minus 1 equals what?",
|
||
"gold": "7",
|
||
"gen": " 5",
|
||
"first_line": "5",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Divide: 65 ÷ 11",
|
||
"gold": "5 remainder 10",
|
||
"gen": " 65",
|
||
"first_line": "65",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "? × 3 = 18",
|
||
"gold": "6",
|
||
"gen": " 18",
|
||
"first_line": "18",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_v3/L2/math-k6-v2/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.078125,
|
||
"final_numeric_accuracy": 0.1875,
|
||
"answer_anywhere_rate": 0.4375,
|
||
"first_line_answer_anywhere_rate": 0.40625,
|
||
"ci95_answer_anywhere": [
|
||
0.32293817065427327,
|
||
0.5591400822171212
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "Draw a clock showing 3:00. What time is it? Write the time in numbers.",
|
||
"gold": "3:00",
|
||
"gen": " The time is 3:00.",
|
||
"first_line": "The time is 3:00.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"problem": "Mia has 8 stickers. She gets 5 more stickers. How many stickers does Mia have now?",
|
||
"gold": "13",
|
||
"gen": " Start with 8. Add 5: 8 + 5 = 13. Use counting on: 8 + 3 = 11, 11 + 2 = 13, 13 + 1 = 14. So 8 + 5 = 14.\n\nLet's double-",
|
||
"first_line": "Start with 8. Add 5: 8 + 5 = 13. Use counting on: 8 + 3 = 11, 11 + 2 = 13, 13 + 1 = 14. So 8 + 5 = 14.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"problem": "Draw a circle. Then draw a line to cut the circle into two equal shares. How many halves do you have?",
|
||
"gold": "2 halves",
|
||
"gen": " You have one half.",
|
||
"first_line": "You have one half.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Mia has 8 stickers and gets 6 more stickers. How many stickers does Mia have now?",
|
||
"gold": "14",
|
||
"gen": " 8 + 6 = 14. Use the standard algorithm: Start with 8. Add 6: 8 + 6 = 14. This matches the previous result. 14 Confidence: high Confidence rationale: The problem is straightforward with a single correct answer. The",
|
||
"first_line": "8 + 6 = 14. Use the standard algorithm: Start with 8. Add 6: 8 + 6 = 14. This matches the previous result. 14 Confidence: high Confidence rationale: The problem",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"problem": "Sarah has 37 stickers. She gets 8 more stickers from her friend. How many stickers does Sarah have now?",
|
||
"gold": "45 stickers",
|
||
"gen": " 37\n\nLet's double-check: We can add the new stickers to the original count: 37 + 8. First, add the ones: 7 + 8 = 15. Then add the tens: 3 + 2 = 5. So",
|
||
"first_line": "37",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Draw a triangle that is red and a triangle that is blue. How are they the same? How are they different?",
|
||
"gold": "They are the same because they both have three sides. They are different because one is red and one is blue.",
|
||
"gen": " They are the same because they both have three sides and three angles. They are different because one is red and one is blue.",
|
||
"first_line": "They are the same because they both have three sides and three angles. They are different because one is red and one is blue.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_v3/L2/math-7-12-v2/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.015625,
|
||
"final_numeric_accuracy": 0.03125,
|
||
"answer_anywhere_rate": 0.078125,
|
||
"first_line_answer_anywhere_rate": 0.078125,
|
||
"ci95_answer_anywhere": [
|
||
0.03383067809543547,
|
||
0.1701975287864777
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "A rectangular prism has a length of 8 cm, a width of 5 cm, and a height of 3 cm. What is its surface area?",
|
||
"gold": "158",
|
||
"gen": " Surface area of a rectangular prism is the sum of the areas of all six faces. The formula is SA = 2lw + 2lh + 2wh. Substituting l=8, w=5, h=3: SA = 2",
|
||
"first_line": "Surface area of a rectangular prism is the sum of the areas of all six faces. The formula is SA = 2lw + 2lh + 2wh. Substituting l=8, w=5, h=3: SA = 2",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "A recipe for homemade lemonade calls for 3/4 cup of lemon juice for every 2 1/2 cups of water. What is the unit rate of cups of lemon juice per cup of water?",
|
||
"gold": "3/10 cup of lemon juice per cup of water",
|
||
"gen": " The unit rate is 3/4 cup of lemon juice per 1 cup of water. Use this unit rate to find how many cups of lemon juice are needed for 2 1/2 cups of water. (2 1/2) / (",
|
||
"first_line": "The unit rate is 3/4 cup of lemon juice per 1 cup of water. Use this unit rate to find how many cups of lemon juice are needed for 2 1/2 cups of water. (2 1/2) ",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "A school wants to estimate the average number of hours per week that its 800 students spend on homework. They randomly select 40 students and ask them. The sample average is 6.2 hours with a standard deviation of 1.5 hou",
|
||
"gold": "More reliable, because a larger sample size reduces variability and gives a more precise estimate of the population average.",
|
||
"gen": " The estimate of the population average is more reliable if the sample size is large enough. A sample of only 40 students is too small to reliably reflect the entire population, so the sample average may be biased. The larger sample size increases the margin",
|
||
"first_line": "The estimate of the population average is more reliable if the sample size is large enough. A sample of only 40 students is too small to reliably reflect the en",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "The table shows the number of cups of flour needed to make a certain number of batches of cookies. Determine the constant of proportionality (cups per batch) and write an equation relating cups (c) to batches (b).\n\n| Bat",
|
||
"gold": "Constant of proportionality = 2.5 cups per batch; equation: c = 2.5b",
|
||
"gen": " The constant of proportionality is 5, because 5 cups of flour per batch corresponds to 2 batches. The equation is c = 5b.\n\nLet's double-check: Check by using the equation c = 5b. For b =",
|
||
"first_line": "The constant of proportionality is 5, because 5 cups of flour per batch corresponds to 2 batches. The equation is c = 5b.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Simplify the expression: 3(2x + 4) - 2(3x - 1).",
|
||
"gold": "14",
|
||
"gen": " 6x + 12\nConfidence: high\nConfidence rationale: The problem preserves the exact same algebraic simplification steps (distributive property, combining like terms) as the original, only changing the variable names and the specific numbers. The verification",
|
||
"first_line": "6x + 12",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "A graph shows a line passing through the origin and the point (4, 10). What is the constant of proportionality (unit rate) represented by this graph?",
|
||
"gold": "2.5",
|
||
"gen": " The constant of proportionality is the slope of the line, which is 10. The answer is 10.",
|
||
"first_line": "The constant of proportionality is the slope of the line, which is 10. The answer is 10.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_v3/L2/math_lite_naturalistic_v1/canonical.jsonl": {
|
||
"N": 0,
|
||
"first_numeric_accuracy": 0.0,
|
||
"final_numeric_accuracy": 0.0,
|
||
"answer_anywhere_rate": 0.0,
|
||
"first_line_answer_anywhere_rate": 0.0,
|
||
"ci95_answer_anywhere": [
|
||
0.0,
|
||
0.0
|
||
],
|
||
"examples": []
|
||
},
|
||
"corpus_v3/L2/gold_k6_algebraic_thinking_v1/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.046875,
|
||
"final_numeric_accuracy": 0.09375,
|
||
"answer_anywhere_rate": 0.125,
|
||
"first_line_answer_anywhere_rate": 0.125,
|
||
"ci95_answer_anywhere": [
|
||
0.064721653926208,
|
||
0.22774786330215924
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "Use the table or number-line information. A growing pattern begins with term table [(1, 12), (2, 18), (3, 24)]. Find term 4. Check by substituting the answer back.",
|
||
"gold": "30.",
|
||
"gen": " 4",
|
||
"first_line": "4",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the table or number-line information. A pattern has rule value = 6n + 5. Which term number has value 47? Explain the inverse operation you use.",
|
||
"gold": "7.",
|
||
"gen": " 47",
|
||
"first_line": "47",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the equation information. A pattern has rule value = 2n + 1. Which term number has value 9? Check by substituting the answer back.",
|
||
"gold": "4.",
|
||
"gen": " 9",
|
||
"first_line": "9",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the equation information. Find y so the equality is true: 10 * 4 = 10 * y. Check the answer in the original equation.",
|
||
"gold": "4.",
|
||
"gen": " 10y",
|
||
"first_line": "10y",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the model information. Use a simple model for this situation: A balance model has left pan 2x + 7 and right pan 43 using beads. Find the missing number. Explain the inverse operation you use. Use x for the unknown, b",
|
||
"gold": "x = 18.",
|
||
"gen": " The missing number is 11.",
|
||
"first_line": "The missing number is 11.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the story information. Rosa starts with 6 books. After giving away an unknown number, 3 are left. Write an equation and find the missing number. Check by substituting the answer back. Use x for the unknown, but keep ",
|
||
"gold": "x = 3.",
|
||
"gen": " Rosa has 6 - 3 = 3 books remaining. The answer is 3.",
|
||
"first_line": "Rosa has 6 - 3 = 3 books remaining. The answer is 3.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"overall": {
|
||
"first_numeric_accuracy": 0.09375,
|
||
"final_numeric_accuracy": 0.1328125,
|
||
"answer_anywhere_rate": 0.23828125,
|
||
"first_line_answer_anywhere_rate": 0.22265625,
|
||
"ci95_answer_anywhere": [
|
||
0.19020459483719204,
|
||
0.2940966102046489
|
||
]
|
||
},
|
||
"examples": [
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Add 7 and 2.",
|
||
"gold": "9",
|
||
"gen": " 13",
|
||
"first_line": "13",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "83 + 49 =",
|
||
"gold": "132",
|
||
"gen": " 102\n\nAnswer: 102",
|
||
"first_line": "102",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What's 23 plus 91?",
|
||
"gold": "114",
|
||
"gen": " 235",
|
||
"first_line": "235",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "8 minus 1 equals what?",
|
||
"gold": "7",
|
||
"gen": " 5",
|
||
"first_line": "5",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Divide: 65 ÷ 11",
|
||
"gold": "5 remainder 10",
|
||
"gen": " 65",
|
||
"first_line": "65",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "? × 3 = 18",
|
||
"gold": "6",
|
||
"gen": " 18",
|
||
"first_line": "18",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Sum 780 and 389.",
|
||
"gold": "1169",
|
||
"gen": " 700",
|
||
"first_line": "700",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Compute the sum: 236 + 328.",
|
||
"gold": "564",
|
||
"gen": " 236",
|
||
"first_line": "236",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What's 32 plus 24?",
|
||
"gold": "56",
|
||
"gen": " 64",
|
||
"first_line": "64",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Evaluate 939 + 350.",
|
||
"gold": "1289",
|
||
"gen": " 938",
|
||
"first_line": "938",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "7 + 7 = ?",
|
||
"gold": "14",
|
||
"gen": " 18\n\nSo the total is 18.\n\nLet's double-check: We can verify by grouping the numbers differently. Notice that 7 + 7 = 14. Then add the remaining 6: 14 + 6 = 20. The",
|
||
"first_line": "18",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Divide 16 by 9.",
|
||
"gold": "1 remainder 7",
|
||
"gen": " 10\n\nSo the quotient is 10 and the remainder is 0.\n\nLet's double-check: Verification by multiplying the quotient by the divisor: 10 * 9 = 90. This confirms that 16 + 9 = 25, so",
|
||
"first_line": "10",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What is 4 × 7?",
|
||
"gold": "28",
|
||
"gen": " 28\n\nSo the product is 28.\n\nLet's double-check: Verification by reverse division: if 4 × 7 = 28, then 28 ÷ 7 should equal 4. Indeed 28 ÷ 7 = 4. This confirms 4 ×",
|
||
"first_line": "28",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "11 × 9 = ?",
|
||
"gold": "99",
|
||
"gen": " 108\n\nLet's double-check: The stated result 108 matches the answer field for this drill.\n\nSo the answer is 108.\nI'm confident in this answer.",
|
||
"first_line": "108",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Compute 7 × 6.",
|
||
"gold": "42",
|
||
"gen": " 42",
|
||
"first_line": "42",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Compute 68 + 92.",
|
||
"gold": "160",
|
||
"gen": " 176",
|
||
"first_line": "176",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Solve: 198 + 903",
|
||
"gold": "1101",
|
||
"gen": " 19812",
|
||
"first_line": "19812",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Add: 4244 + 3127",
|
||
"gold": "7371",
|
||
"gen": " 6875",
|
||
"first_line": "6875",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What is 9 × 12?",
|
||
"gold": "108",
|
||
"gen": " 108\n\nLet's double-check: The stated result 108 matches the answer field for this drill.\n\nSo the answer is 108.\nI'm confident in this answer.",
|
||
"first_line": "108",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Divide: 18 ÷ 8",
|
||
"gold": "2 remainder 2",
|
||
"gen": " 2\n\nSo the answer is 2.\nI'm confident in this answer.",
|
||
"first_line": "2",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
}
|
||
]
|
||
},
|
||
"code_first_function": {
|
||
"N": 6,
|
||
"ast_rate": 1.0,
|
||
"non_docstring_body_rate": 0.0,
|
||
"return_rate": 0.0,
|
||
"exec_pass_rate": 0.0,
|
||
"ci95_exec": [
|
||
0.0,
|
||
0.3903430336530645
|
||
],
|
||
"examples": [
|
||
{
|
||
"name": "add_one",
|
||
"prompt": "def add_one(n):",
|
||
"generated": "\ndef test_add_one():\n assert add_one(1) == 2\n\ndef test_add_two():\n assert add_two(1) == 3\n\ndef test_add_three():\n assert add_three(1) == 4\n\ndef test_add_four():\n assert add_four(1) == 5\n\ndef test_add_five():\n assert add_five(1) == 6\n\n",
|
||
"first_function": "def add_one(n):\n \"\"\"Return n plus one.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "reverse_string",
|
||
"prompt": "def reverse_string(s):",
|
||
"generated": "",
|
||
"first_function": "def reverse_string(s):\n \"\"\"Return s reversed.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "count_vowels",
|
||
"prompt": "def count_vowels(s):",
|
||
"generated": "\ndef count_digits(s):\n \"\"\"Return the number of digits in s.\"\"\"\n\ndef is_digit(s):\n \"\"\"Return true if s is a digit.\"\"\"\n\ndef is_lowercase(s):\n \"\"\"Return true if s is a lowercase string.\"\"\"\n\ndef is_uppercase(s):\n \"\"\"Return true if s is a uppercase string.\"\"\"\n\ndef is_lowercase_lowercase(s):\n ",
|
||
"first_function": "def count_vowels(s):\n \"\"\"Return the number of vowels in s.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "is_even",
|
||
"prompt": "def is_even(n):",
|
||
"generated": "",
|
||
"first_function": "def is_even(n):\n \"\"\"Return True if n is even.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "sum_list",
|
||
"prompt": "def sum_list(items):",
|
||
"generated": "",
|
||
"first_function": "def sum_list(items):\n \"\"\"Return the sum of the numbers in items.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "first_item",
|
||
"prompt": "def first_item(items):",
|
||
"generated": "",
|
||
"first_function": "def first_item(items):\n \"\"\"Return the first item in a non-empty list.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_direct_answer_audit": {
|
||
"manifest": "runs/v10_348m_cone/manifest_live.yaml",
|
||
"by_component": [
|
||
{
|
||
"id": "L0_tom_semantic_topup_v1",
|
||
"layer": "L0",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L0_tom_semantic_equalizer_v1",
|
||
"layer": "L0",
|
||
"N": 4336,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L0_5_cosmopedia_l1_reroute_v1",
|
||
"layer": "L0_5",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0012
|
||
},
|
||
{
|
||
"id": "L1_naturalistic_reasoning_v2",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0016
|
||
},
|
||
{
|
||
"id": "L1_naturalistic_reasoning_v1",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0022
|
||
},
|
||
{
|
||
"id": "L1_logic-primitives-v2",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0062
|
||
},
|
||
{
|
||
"id": "L1_logic-primitives-v1",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.002
|
||
},
|
||
{
|
||
"id": "L1_logic-books-bulk-v1",
|
||
"layer": "L1",
|
||
"N": 918,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.007625272331154684
|
||
},
|
||
{
|
||
"id": "L1_cac-compact-v1",
|
||
"layer": "L1",
|
||
"N": 500,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_math-7-12-v2",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0002
|
||
},
|
||
{
|
||
"id": "L2_math-k6-v2",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0028
|
||
},
|
||
{
|
||
"id": "L2_math_lite_naturalistic_v1",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_math-drill-v1",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_gold_k6_algebraic_thinking_v1",
|
||
"layer": "L2",
|
||
"N": 4800,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 1.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_entry_subjects_gold_v1",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0004
|
||
},
|
||
{
|
||
"id": "L2_intro_books_gold_clean_v1",
|
||
"layer": "L2",
|
||
"N": 2634,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.013287775246772968
|
||
},
|
||
{
|
||
"id": "L2_math-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 196,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.01020408163265306
|
||
},
|
||
{
|
||
"id": "L2_algebra-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 369,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.02168021680216802
|
||
},
|
||
{
|
||
"id": "L2_geometry-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 172,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.040697674418604654
|
||
},
|
||
{
|
||
"id": "L2_trig-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 111,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_precalculus-source-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 124,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.016129032258064516
|
||
},
|
||
{
|
||
"id": "primer_math",
|
||
"layer": "L2",
|
||
"N": 170,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L3_python_no_eval_overlap_clean_v1",
|
||
"layer": "L3",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.005
|
||
},
|
||
{
|
||
"id": "L3_code_lane_clean_v3",
|
||
"layer": "L3",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0044
|
||
},
|
||
{
|
||
"id": "L3_process_reasoning_books_v1",
|
||
"layer": "L3",
|
||
"N": 797,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.002509410288582183
|
||
},
|
||
{
|
||
"id": "L3_algorithms-bulk-v1",
|
||
"layer": "L3",
|
||
"N": 298,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.006711409395973154
|
||
},
|
||
{
|
||
"id": "L3_data-science-bulk-clean-v1",
|
||
"layer": "L3",
|
||
"N": 294,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L3_algorithms-books-new-clean-v1",
|
||
"layer": "L3",
|
||
"N": 35,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L3_machine-learning-books-bulk-v1",
|
||
"layer": "L3",
|
||
"N": 6,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "primer_code_disciplines",
|
||
"layer": "L3",
|
||
"N": 620,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "agentic_agent-flan-v1",
|
||
"layer": "L4",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0816
|
||
},
|
||
{
|
||
"id": "primer_agentic",
|
||
"layer": "L4",
|
||
"N": 125,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.016
|
||
}
|
||
],
|
||
"by_layer": {
|
||
"L0": {
|
||
"N": 9336,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
"L0_5": {
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0012
|
||
},
|
||
"L1": {
|
||
"N": 21418,
|
||
"output_answer_rate": 0.490241852647306,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0031282099168923336
|
||
},
|
||
"L2": {
|
||
"N": 33576,
|
||
"output_answer_rate": 0.5947700738622825,
|
||
"answer_key_rate": 0.14295925661186562,
|
||
"answer_marker_rate": 0.002114605670717179
|
||
},
|
||
"L3": {
|
||
"N": 12050,
|
||
"output_answer_rate": 0.051452282157676346,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.004232365145228216
|
||
},
|
||
"L4": {
|
||
"N": 5125,
|
||
"output_answer_rate": 0.024390243902439025,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.08
|
||
}
|
||
}
|
||
}
|
||
},
|
||
"status": "complete"
|
||
}
|