1352 lines
48 KiB
JSON
1352 lines
48 KiB
JSON
{
|
||
"generated_at": "2026-06-22T20:54:19Z",
|
||
"ckpt": "runs/v10_348m_cone_sft_focus600/sft_ckpt_0000300.pt",
|
||
"step": 300,
|
||
"device": "cuda",
|
||
"threads": 8,
|
||
"diagnostics": {
|
||
"transitive": {
|
||
"depth_1": {
|
||
"N": 128,
|
||
"chance": 0.5,
|
||
"first_name_accuracy": 0.46875,
|
||
"answer_anywhere_rate": 0.46875,
|
||
"ci95_first_name": [
|
||
0.38447328031492234,
|
||
0.5548478444287094
|
||
],
|
||
"by_type": {
|
||
"tallest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.6875,
|
||
"answer_anywhere_rate": 0.6875,
|
||
"ci95_first_name": [
|
||
0.5660743364084737,
|
||
0.7876909049773426
|
||
]
|
||
},
|
||
"shortest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.25,
|
||
"answer_anywhere_rate": 0.25,
|
||
"ci95_first_name": [
|
||
0.16014791974174092,
|
||
0.36816509174383727
|
||
]
|
||
}
|
||
},
|
||
"examples": [
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Finn is taller than Cara. Of all of them, the tallest is",
|
||
"gold": "Finn",
|
||
"gen": " Cara.\n \n\nAnswer: Cara",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Finn is taller than Cara. Of all of them, the shortest is",
|
||
"gold": "Cara",
|
||
"gen": " Leo.\n\nAnswer: Leo",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Hugo is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Hugo",
|
||
"gen": " Pierre.",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Hugo is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Ben.\n\nAnswer: Ben",
|
||
"first_name": "Ben",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Gina.\n \n\nAnswer: Gina",
|
||
"first_name": "Gina",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Leo.\n\nSo the answer is Leo.",
|
||
"first_name": "",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Ben is taller than Evan. Of all of them, the tallest is",
|
||
"gold": "Ben",
|
||
"gen": " Ben.\n\nAnswer: Ben",
|
||
"first_name": "Ben",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Ben is taller than Evan. Of all of them, the shortest is",
|
||
"gold": "Evan",
|
||
"gen": " Ben.",
|
||
"first_name": "Ben",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Cara.",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Cara. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Cara.\n \n\nAnswer: Cara",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Cara. Of all of them, the shortest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n \n\nAnswer: Cara",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
}
|
||
]
|
||
},
|
||
"depth_2": {
|
||
"N": 128,
|
||
"chance": 0.3333333333333333,
|
||
"first_name_accuracy": 0.6171875,
|
||
"answer_anywhere_rate": 0.6171875,
|
||
"ci95_first_name": [
|
||
0.5307306786554781,
|
||
0.6968151035559028
|
||
],
|
||
"by_type": {
|
||
"tallest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.546875,
|
||
"answer_anywhere_rate": 0.546875,
|
||
"ci95_first_name": [
|
||
0.42573382520679776,
|
||
0.6627074851396563
|
||
]
|
||
},
|
||
"shortest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.6875,
|
||
"answer_anywhere_rate": 0.6875,
|
||
"ci95_first_name": [
|
||
0.5660743364084737,
|
||
0.7876909049773426
|
||
]
|
||
}
|
||
},
|
||
"examples": [
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Evan is taller than Finn. Finn is taller than Dana. Of all of them, the tallest is",
|
||
"gold": "Evan",
|
||
"gen": " Evan.\n \n\nReasoning: First, we are given that",
|
||
"first_name": "Evan",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Evan is taller than Finn. Finn is taller than Dana. Of all of them, the shortest is",
|
||
"gold": "Dana",
|
||
"gen": " Dana.\n\nReasoning: First, we list the three heights:",
|
||
"first_name": "Dana",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Finn. Finn is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n\nReasoning: First, we map the ordered chain",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Finn. Finn is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Cara.\n\nReasoning: First, we list the three heights",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Evan. Evan is taller than Hugo. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Hugo.\n \n\nAnswer: Hugo",
|
||
"first_name": "Hugo",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Evan. Evan is taller than Hugo. Of all of them, the shortest is",
|
||
"gold": "Hugo",
|
||
"gen": " Hugo.\n \n\nAnswer: Hugo",
|
||
"first_name": "Hugo",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Evan is taller than Gina. Gina is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Evan",
|
||
"gen": " Evan.\n \n\nReasoning: First, we list the three",
|
||
"first_name": "Evan",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Evan is taller than Gina. Gina is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Anna.\n\nVerification: Consider a different set: {x, y",
|
||
"first_name": "Anna",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Cara. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Cara.\n \n\nAnswer: Cara",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Cara. Of all of them, the shortest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n \n\nReasoning: First, list the three heights",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Dana. Dana is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Gina.\n \n\nReasoning: First, identify the three people:",
|
||
"first_name": "Gina",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Dana. Dana is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Ben. This confirms Ben is the shortest. Ben Confidence: high Confidence rationale:",
|
||
"first_name": "Ben",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
}
|
||
]
|
||
},
|
||
"depth_3": {
|
||
"N": 128,
|
||
"chance": 0.25,
|
||
"first_name_accuracy": 0.4375,
|
||
"answer_anywhere_rate": 0.4375,
|
||
"ci95_first_name": [
|
||
0.35462168452480924,
|
||
0.5240205649624543
|
||
],
|
||
"by_type": {
|
||
"tallest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.359375,
|
||
"answer_anywhere_rate": 0.359375,
|
||
"ci95_first_name": [
|
||
0.2528823053221461,
|
||
0.48179376363849163
|
||
]
|
||
},
|
||
"shortest": {
|
||
"N": 64,
|
||
"first_name_accuracy": 0.515625,
|
||
"answer_anywhere_rate": 0.515625,
|
||
"ci95_first_name": [
|
||
0.3958139224399903,
|
||
0.6336665143421609
|
||
]
|
||
}
|
||
},
|
||
"examples": [
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Evan. Evan is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Gina",
|
||
"gen": " Gina.\n \n\nLet's double-check: We can verify this by",
|
||
"first_name": "Gina",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Gina is taller than Anna. Anna is taller than Evan. Evan is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Ben.\n\nReasoning: First, list the ordered chain: Anna",
|
||
"first_name": "Ben",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Hugo. Hugo is taller than Gina. Gina is taller than Dana. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n \n\nReasoning: First, we list the three",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Hugo. Hugo is taller than Gina. Gina is taller than Dana. Of all of them, the shortest is",
|
||
"gold": "Dana",
|
||
"gen": " Dana. This matches the conclusion derived by the full chain of reasoning.\n\n",
|
||
"first_name": "Dana",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Evan. Evan is taller than Ben. Ben is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n \n\nReasoning: First, we list the ordered",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Evan. Evan is taller than Ben. Ben is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Evan?\n\nReasoning: First, list the ordered chain:",
|
||
"first_name": "Evan",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Gina. Gina is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n \n\nReasoning: First, we list the three",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Gina. Gina is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Anna.\n \n\nReasoning: First, list the ordered chain:",
|
||
"first_name": "Anna",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Anna is taller than Cara. Cara is taller than Gina. Gina is taller than Ben. Of all of them, the tallest is",
|
||
"gold": "Anna",
|
||
"gen": " Cara.\n \n\nReasoning: First, we list the ordered",
|
||
"first_name": "Cara",
|
||
"first_name_ok": false,
|
||
"answer_anywhere": false
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Anna is taller than Cara. Cara is taller than Gina. Gina is taller than Ben. Of all of them, the shortest is",
|
||
"gold": "Ben",
|
||
"gen": " Ben.\n\nAnswer: Ben",
|
||
"first_name": "Ben",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "tallest",
|
||
"prompt": "Cara is taller than Ben. Ben is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is",
|
||
"gold": "Cara",
|
||
"gen": " Cara.\n \n\nReasoning: First, we list the ordered",
|
||
"first_name": "Cara",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
},
|
||
{
|
||
"type": "shortest",
|
||
"prompt": "Cara is taller than Ben. Ben is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is",
|
||
"gold": "Anna",
|
||
"gen": " Anna.\n\nReasoning: First, list the ordered chain: Ben",
|
||
"first_name": "Anna",
|
||
"first_name_ok": true,
|
||
"answer_anywhere": true
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"math_extraction": {
|
||
"N": 256,
|
||
"by_source": {
|
||
"corpus_v3/L2/math-drill-v1/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.328125,
|
||
"final_numeric_accuracy": 0.328125,
|
||
"answer_anywhere_rate": 0.328125,
|
||
"first_line_answer_anywhere_rate": 0.328125,
|
||
"ci95_answer_anywhere": [
|
||
0.2257039576708776,
|
||
0.45001123772545737
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "Add 7 and 2.",
|
||
"gold": "9",
|
||
"gen": " 15",
|
||
"first_line": "15",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "83 + 49 =",
|
||
"gold": "132",
|
||
"gen": " 128",
|
||
"first_line": "128",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "What's 23 plus 91?",
|
||
"gold": "114",
|
||
"gen": " 104",
|
||
"first_line": "104",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "8 minus 1 equals what?",
|
||
"gold": "7",
|
||
"gen": " 3",
|
||
"first_line": "3",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Divide: 65 ÷ 11",
|
||
"gold": "5 remainder 10",
|
||
"gen": " 65",
|
||
"first_line": "65",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "? × 3 = 18",
|
||
"gold": "6",
|
||
"gen": " 18",
|
||
"first_line": "18",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_v3/L2/math-k6-v2/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.1875,
|
||
"final_numeric_accuracy": 0.359375,
|
||
"answer_anywhere_rate": 0.515625,
|
||
"first_line_answer_anywhere_rate": 0.484375,
|
||
"ci95_answer_anywhere": [
|
||
0.3958139224399903,
|
||
0.6336665143421609
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "Draw a clock showing 3:00. What time is it? Write the time in numbers.",
|
||
"gold": "3:00",
|
||
"gen": " 3:00",
|
||
"first_line": "3:00",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"problem": "Mia has 8 stickers. She gets 5 more stickers. How many stickers does Mia have now?",
|
||
"gold": "13",
|
||
"gen": " 8",
|
||
"first_line": "8",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Draw a circle. Then draw a line to cut the circle into two equal shares. How many halves do you have?",
|
||
"gold": "2 halves",
|
||
"gen": " 2",
|
||
"first_line": "2",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"problem": "Mia has 8 stickers and gets 6 more stickers. How many stickers does Mia have now?",
|
||
"gold": "14",
|
||
"gen": " 8 + 6 = 14",
|
||
"first_line": "8 + 6 = 14",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"problem": "Sarah has 37 stickers. She gets 8 more stickers from her friend. How many stickers does Sarah have now?",
|
||
"gold": "45 stickers",
|
||
"gen": " 37\nConfidence: high\nConfidence rationale: The problem uses straightforward addition of two small numbers, and the verification confirms the result with a different operation (subtraction).",
|
||
"first_line": "37",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Draw a triangle that is red and a triangle that is blue. How are they the same? How are they different?",
|
||
"gold": "They are the same because they both have three sides. They are different because one is red and one is blue.",
|
||
"gen": " They are the same because they both have three sides and three angles. They are different because one is red and one is blue.",
|
||
"first_line": "They are the same because they both have three sides and three angles. They are different because one is red and one is blue.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_v3/L2/math-7-12-v2/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.03125,
|
||
"final_numeric_accuracy": 0.015625,
|
||
"answer_anywhere_rate": 0.046875,
|
||
"first_line_answer_anywhere_rate": 0.046875,
|
||
"ci95_answer_anywhere": [
|
||
0.016068725432185196,
|
||
0.12899860788542522
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "A rectangular prism has a length of 8 cm, a width of 5 cm, and a height of 3 cm. What is its surface area?",
|
||
"gold": "158",
|
||
"gen": " Surface area of a rectangular prism is the sum of the areas of all six faces. The formula is 2lw + 2lh + 2wh. Substituting l=8, w=5, h=3 gives: 2(8×",
|
||
"first_line": "Surface area of a rectangular prism is the sum of the areas of all six faces. The formula is 2lw + 2lh + 2wh. Substituting l=8, w=5, h=3 gives: 2(8×",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "A recipe for homemade lemonade calls for 3/4 cup of lemon juice for every 2 1/2 cups of water. What is the unit rate of cups of lemon juice per cup of water?",
|
||
"gold": "3/10 cup of lemon juice per cup of water",
|
||
"gen": " 3/4 cup of lemon juice per 1 cup of water",
|
||
"first_line": "3/4 cup of lemon juice per 1 cup of water",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "A school wants to estimate the average number of hours per week that its 800 students spend on homework. They randomly select 40 students and ask them. The sample average is 6.2 hours with a standard deviation of 1.5 hou",
|
||
"gold": "More reliable, because a larger sample size reduces variability and gives a more precise estimate of the population average.",
|
||
"gen": " The estimate of the population average is more reliable if the sample size is large enough. A sample of only 40 students is too small to reliably reflect the entire population, so the estimate is unreliable.",
|
||
"first_line": "The estimate of the population average is more reliable if the sample size is large enough. A sample of only 40 students is too small to reliably reflect the en",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "The table shows the number of cups of flour needed to make a certain number of batches of cookies. Determine the constant of proportionality (cups per batch) and write an equation relating cups (c) to batches (b).\n\n| Bat",
|
||
"gold": "Constant of proportionality = 2.5 cups per batch; equation: c = 2.5b",
|
||
"gen": " c = 5b\n\nThis equation shows that for every batch of cookies, 5 cups of flour are needed.\n\nLet's double-check: Use the equation to check the given data: 2 batches of cookies = 10 cups,",
|
||
"first_line": "c = 5b",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Simplify the expression: 3(2x + 4) - 2(3x - 1).",
|
||
"gold": "14",
|
||
"gen": " 6x + 12\n\nLet's double-check: To verify, choose a value for x, say x = 5. Substituting into the original expression: 3(2*5 + 4) - 2(3*5 - 1",
|
||
"first_line": "6x + 12",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "A graph shows a line passing through the origin and the point (4, 10). What is the constant of proportionality (unit rate) represented by this graph?",
|
||
"gold": "2.5",
|
||
"gen": " The constant of proportionality is the slope of the line, which is 10.",
|
||
"first_line": "The constant of proportionality is the slope of the line, which is 10.",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_v3/L2/math_lite_naturalistic_v1/canonical.jsonl": {
|
||
"N": 0,
|
||
"first_numeric_accuracy": 0.0,
|
||
"final_numeric_accuracy": 0.0,
|
||
"answer_anywhere_rate": 0.0,
|
||
"first_line_answer_anywhere_rate": 0.0,
|
||
"ci95_answer_anywhere": [
|
||
0.0,
|
||
0.0
|
||
],
|
||
"examples": []
|
||
},
|
||
"corpus_v3/L2/gold_k6_algebraic_thinking_v1/canonical.jsonl": {
|
||
"N": 64,
|
||
"first_numeric_accuracy": 0.03125,
|
||
"final_numeric_accuracy": 0.0625,
|
||
"answer_anywhere_rate": 0.0625,
|
||
"first_line_answer_anywhere_rate": 0.0625,
|
||
"ci95_answer_anywhere": [
|
||
0.024570803674792932,
|
||
0.14997696642496888
|
||
],
|
||
"examples": [
|
||
{
|
||
"problem": "Use the table or number-line information. A growing pattern begins with term table [(1, 12), (2, 18), (3, 24)]. Find term 4. Check by substituting the answer back.",
|
||
"gold": "30.",
|
||
"gen": " 4",
|
||
"first_line": "4",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the table or number-line information. A pattern has rule value = 6n + 5. Which term number has value 47? Explain the inverse operation you use.",
|
||
"gold": "7.",
|
||
"gen": " 47",
|
||
"first_line": "47",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the equation information. A pattern has rule value = 2n + 1. Which term number has value 9? Check by substituting the answer back.",
|
||
"gold": "4.",
|
||
"gen": " 9",
|
||
"first_line": "9",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the equation information. Find y so the equality is true: 10 * 4 = 10 * y. Check the answer in the original equation.",
|
||
"gold": "4.",
|
||
"gen": " 40",
|
||
"first_line": "40",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the model information. Use a simple model for this situation: A balance model has left pan 2x + 7 and right pan 43 using beads. Find the missing number. Explain the inverse operation you use. Use x for the unknown, b",
|
||
"gold": "x = 18.",
|
||
"gen": " 19",
|
||
"first_line": "19",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"problem": "Use the story information. Rosa starts with 6 books. After giving away an unknown number, 3 are left. Write an equation and find the missing number. Check by substituting the answer back. Use x for the unknown, but keep ",
|
||
"gold": "x = 3.",
|
||
"gen": " 3",
|
||
"first_line": "3",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"overall": {
|
||
"first_numeric_accuracy": 0.14453125,
|
||
"final_numeric_accuracy": 0.19140625,
|
||
"answer_anywhere_rate": 0.23828125,
|
||
"first_line_answer_anywhere_rate": 0.23046875,
|
||
"ci95_answer_anywhere": [
|
||
0.19020459483719204,
|
||
0.2940966102046489
|
||
]
|
||
},
|
||
"examples": [
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Add 7 and 2.",
|
||
"gold": "9",
|
||
"gen": " 15",
|
||
"first_line": "15",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "83 + 49 =",
|
||
"gold": "132",
|
||
"gen": " 128",
|
||
"first_line": "128",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What's 23 plus 91?",
|
||
"gold": "114",
|
||
"gen": " 104",
|
||
"first_line": "104",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "8 minus 1 equals what?",
|
||
"gold": "7",
|
||
"gen": " 3",
|
||
"first_line": "3",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Divide: 65 ÷ 11",
|
||
"gold": "5 remainder 10",
|
||
"gen": " 65",
|
||
"first_line": "65",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "? × 3 = 18",
|
||
"gold": "6",
|
||
"gen": " 18",
|
||
"first_line": "18",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Sum 780 and 389.",
|
||
"gold": "1169",
|
||
"gen": " 800",
|
||
"first_line": "800",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Compute the sum: 236 + 328.",
|
||
"gold": "564",
|
||
"gen": " 233",
|
||
"first_line": "233",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What's 32 plus 24?",
|
||
"gold": "56",
|
||
"gen": " 64",
|
||
"first_line": "64",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Evaluate 939 + 350.",
|
||
"gold": "1289",
|
||
"gen": " 9880",
|
||
"first_line": "9880",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "7 + 7 = ?",
|
||
"gold": "14",
|
||
"gen": " 18",
|
||
"first_line": "18",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Divide 16 by 9.",
|
||
"gold": "1 remainder 7",
|
||
"gen": " 4",
|
||
"first_line": "4",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What is 4 × 7?",
|
||
"gold": "28",
|
||
"gen": " 28",
|
||
"first_line": "28",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "11 × 9 = ?",
|
||
"gold": "99",
|
||
"gen": " 81",
|
||
"first_line": "81",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Compute 7 × 6.",
|
||
"gold": "42",
|
||
"gen": " 42",
|
||
"first_line": "42",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Compute 68 + 92.",
|
||
"gold": "160",
|
||
"gen": " 190",
|
||
"first_line": "190",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Solve: 198 + 903",
|
||
"gold": "1101",
|
||
"gen": " 198",
|
||
"first_line": "198",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Add: 4244 + 3127",
|
||
"gold": "7371",
|
||
"gen": " 6875",
|
||
"first_line": "6875",
|
||
"first_numeric_match": false,
|
||
"final_numeric_match": false,
|
||
"answer_anywhere": false,
|
||
"first_line_answer_anywhere": false
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "What is 9 × 12?",
|
||
"gold": "108",
|
||
"gen": " 108",
|
||
"first_line": "108",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
},
|
||
{
|
||
"source": "corpus_v3/L2/math-drill-v1/canonical.jsonl",
|
||
"problem": "Divide: 18 ÷ 8",
|
||
"gold": "2 remainder 2",
|
||
"gen": " 2",
|
||
"first_line": "2",
|
||
"first_numeric_match": true,
|
||
"final_numeric_match": true,
|
||
"answer_anywhere": true,
|
||
"first_line_answer_anywhere": true
|
||
}
|
||
]
|
||
},
|
||
"code_first_function": {
|
||
"N": 6,
|
||
"ast_rate": 1.0,
|
||
"non_docstring_body_rate": 0.0,
|
||
"return_rate": 0.0,
|
||
"exec_pass_rate": 0.0,
|
||
"ci95_exec": [
|
||
0.0,
|
||
0.3903430336530645
|
||
],
|
||
"examples": [
|
||
{
|
||
"name": "add_one",
|
||
"prompt": "def add_one(n):",
|
||
"generated": "",
|
||
"first_function": "def add_one(n):\n \"\"\"Return n plus one.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "reverse_string",
|
||
"prompt": "def reverse_string(s):",
|
||
"generated": "",
|
||
"first_function": "def reverse_string(s):\n \"\"\"Return s reversed.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "count_vowels",
|
||
"prompt": "def count_vowels(s):",
|
||
"generated": "",
|
||
"first_function": "def count_vowels(s):\n \"\"\"Return the number of vowels in s.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "is_even",
|
||
"prompt": "def is_even(n):",
|
||
"generated": "",
|
||
"first_function": "def is_even(n):\n \"\"\"Return True if n is even.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "sum_list",
|
||
"prompt": "def sum_list(items):",
|
||
"generated": "",
|
||
"first_function": "def sum_list(items):\n \"\"\"Return the sum of the numbers in items.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
},
|
||
{
|
||
"name": "first_item",
|
||
"prompt": "def first_item(items):",
|
||
"generated": "",
|
||
"first_function": "def first_item(items):\n \"\"\"Return the first item in a non-empty list.\"\"\"\n",
|
||
"ast_ok": true,
|
||
"has_non_docstring_body": false,
|
||
"has_return": false,
|
||
"undefined_names": [],
|
||
"exec_ok": false
|
||
}
|
||
]
|
||
},
|
||
"corpus_direct_answer_audit": {
|
||
"manifest": "runs/v10_348m_cone/manifest_live.yaml",
|
||
"by_component": [
|
||
{
|
||
"id": "L0_tom_semantic_topup_v1",
|
||
"layer": "L0",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L0_tom_semantic_equalizer_v1",
|
||
"layer": "L0",
|
||
"N": 4336,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L0_5_cosmopedia_l1_reroute_v1",
|
||
"layer": "L0_5",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0012
|
||
},
|
||
{
|
||
"id": "L1_naturalistic_reasoning_v2",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0016
|
||
},
|
||
{
|
||
"id": "L1_naturalistic_reasoning_v1",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0022
|
||
},
|
||
{
|
||
"id": "L1_logic-primitives-v2",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0062
|
||
},
|
||
{
|
||
"id": "L1_logic-primitives-v1",
|
||
"layer": "L1",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.002
|
||
},
|
||
{
|
||
"id": "L1_logic-books-bulk-v1",
|
||
"layer": "L1",
|
||
"N": 918,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.007625272331154684
|
||
},
|
||
{
|
||
"id": "L1_cac-compact-v1",
|
||
"layer": "L1",
|
||
"N": 500,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_math-7-12-v2",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0002
|
||
},
|
||
{
|
||
"id": "L2_math-k6-v2",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0028
|
||
},
|
||
{
|
||
"id": "L2_math_lite_naturalistic_v1",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_math-drill-v1",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_gold_k6_algebraic_thinking_v1",
|
||
"layer": "L2",
|
||
"N": 4800,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 1.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_entry_subjects_gold_v1",
|
||
"layer": "L2",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0004
|
||
},
|
||
{
|
||
"id": "L2_intro_books_gold_clean_v1",
|
||
"layer": "L2",
|
||
"N": 2634,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.013287775246772968
|
||
},
|
||
{
|
||
"id": "L2_math-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 196,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.01020408163265306
|
||
},
|
||
{
|
||
"id": "L2_algebra-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 369,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.02168021680216802
|
||
},
|
||
{
|
||
"id": "L2_geometry-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 172,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.040697674418604654
|
||
},
|
||
{
|
||
"id": "L2_trig-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 111,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L2_precalculus-source-bulk-v1",
|
||
"layer": "L2",
|
||
"N": 124,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.016129032258064516
|
||
},
|
||
{
|
||
"id": "primer_math",
|
||
"layer": "L2",
|
||
"N": 170,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L3_python_no_eval_overlap_clean_v1",
|
||
"layer": "L3",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.005
|
||
},
|
||
{
|
||
"id": "L3_code_lane_clean_v3",
|
||
"layer": "L3",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0044
|
||
},
|
||
{
|
||
"id": "L3_process_reasoning_books_v1",
|
||
"layer": "L3",
|
||
"N": 797,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.002509410288582183
|
||
},
|
||
{
|
||
"id": "L3_algorithms-bulk-v1",
|
||
"layer": "L3",
|
||
"N": 298,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.006711409395973154
|
||
},
|
||
{
|
||
"id": "L3_data-science-bulk-clean-v1",
|
||
"layer": "L3",
|
||
"N": 294,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L3_algorithms-books-new-clean-v1",
|
||
"layer": "L3",
|
||
"N": 35,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "L3_machine-learning-books-bulk-v1",
|
||
"layer": "L3",
|
||
"N": 6,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "primer_code_disciplines",
|
||
"layer": "L3",
|
||
"N": 620,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
{
|
||
"id": "agentic_agent-flan-v1",
|
||
"layer": "L4",
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0816
|
||
},
|
||
{
|
||
"id": "primer_agentic",
|
||
"layer": "L4",
|
||
"N": 125,
|
||
"output_answer_rate": 1.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.016
|
||
}
|
||
],
|
||
"by_layer": {
|
||
"L0": {
|
||
"N": 9336,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0
|
||
},
|
||
"L0_5": {
|
||
"N": 5000,
|
||
"output_answer_rate": 0.0,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0012
|
||
},
|
||
"L1": {
|
||
"N": 21418,
|
||
"output_answer_rate": 0.490241852647306,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.0031282099168923336
|
||
},
|
||
"L2": {
|
||
"N": 33576,
|
||
"output_answer_rate": 0.5947700738622825,
|
||
"answer_key_rate": 0.14295925661186562,
|
||
"answer_marker_rate": 0.002114605670717179
|
||
},
|
||
"L3": {
|
||
"N": 12050,
|
||
"output_answer_rate": 0.051452282157676346,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.004232365145228216
|
||
},
|
||
"L4": {
|
||
"N": 5125,
|
||
"output_answer_rate": 0.024390243902439025,
|
||
"answer_key_rate": 0.0,
|
||
"answer_marker_rate": 0.08
|
||
}
|
||
}
|
||
}
|
||
},
|
||
"status": "complete"
|
||
}
|