{
  "chat_format": "User:\\n...\\nAssistant:\\n",
  "ckpt": "runs/v10_348m_cone_sft_focus600/sft_ckpt_0000300.pt",
  "code": {
    "N": 6,
    "ast_rate": 1.0,
    "body_rate": 0.0,
    "ci95_exec": [
      0.0,
      0.3903430336530645
    ],
    "examples": [
      {
        "ast_ok": true,
        "body_ok": false,
        "code": "def add_one(n):\n    \"\"\"Return n plus one.\"\"\"\n",
        "exec_ok": false,
        "generated": " return n + 1",
        "name": "add_one",
        "return_ok": false
      },
      {
        "ast_ok": true,
        "body_ok": false,
        "code": "def reverse_string(s):\n    \"\"\"Return s reversed.\"\"\"\n",
        "exec_ok": false,
        "generated": "return s[::-1]",
        "name": "reverse_string",
        "return_ok": false
      },
      {
        "ast_ok": true,
        "body_ok": false,
        "code": "def count_vowels(s):\n    \"\"\"Return the number of vowels in s.\"\"\"\n",
        "exec_ok": false,
        "generated": " return sum(1 for ch in s.lower() if ch in 'aeiou')",
        "name": "count_vowels",
        "return_ok": false
      },
      {
        "ast_ok": true,
        "body_ok": false,
        "code": "def is_even(n):\n    \"\"\"Return True if n is even.\"\"\"\n",
        "exec_ok": false,
        "generated": " return n % 2 == 0",
        "name": "is_even",
        "return_ok": false
      },
      {
        "ast_ok": true,
        "body_ok": false,
        "code": "def sum_list(items):\n    \"\"\"Return the sum of the numbers in items.\"\"\"\n",
        "exec_ok": false,
        "generated": " return sum(items)",
        "name": "sum_list",
        "return_ok": false
      },
      {
        "ast_ok": true,
        "body_ok": false,
        "code": "def first_item(items):\n    \"\"\"Return the first item in a non-empty list.\"\"\"\n",
        "exec_ok": false,
        "generated": " return items[0]",
        "name": "first_item",
        "return_ok": false
      }
    ],
    "exec_rate": 0.0,
    "return_rate": 0.0
  },
  "device": "cuda",
  "generated_at": "2026-06-22T20:54:24Z",
  "math": {
    "N": 128,
    "answer_anywhere_rate": 0.28125,
    "ci95_anywhere": [
      0.2106120712149715,
      0.36463580199045076
    ],
    "examples": [
      {
        "anywhere": true,
        "final": true,
        "first": true,
        "gen": " 6",
        "gold": 6,
        "prompt": "Sam has 4 apples. Lily has 2 more apples than Sam. How many apples does Lily have? Return only the number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 2",
        "gold": 4,
        "prompt": "Mia had 3 red marbles and found 2 blue marbles. Then she gave 1 marble away. Now Mia has how many marbles? Return only the number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 32",
        "gold": 20,
        "prompt": "What is 68 - 48? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 196",
        "gold": 214,
        "prompt": "What is 118 + 96? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 105",
        "gold": 113,
        "prompt": "What is 109 + 4? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 284",
        "gold": 1110,
        "prompt": "What is 74 * 15? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 103",
        "gold": 55,
        "prompt": "What is 97 - 42? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 75",
        "gold": 51,
        "prompt": "What is 102 - 51? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 18",
        "gold": 2,
        "prompt": "What is 22 - 20? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 54",
        "gold": 58,
        "prompt": "What is 26 + 32? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 365",
        "gold": 1469,
        "prompt": "What is 113 * 13? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 44",
        "gold": 43,
        "prompt": "What is 43 * 1? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 109",
        "gold": 103,
        "prompt": "What is 111 - 8? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 110",
        "gold": 122,
        "prompt": "What is 79 + 43? Return only the final number."
      },
      {
        "anywhere": true,
        "final": true,
        "first": true,
        "gen": " 120",
        "gold": 120,
        "prompt": "What is 60 * 2? Return only the final number."
      },
      {
        "anywhere": false,
        "final": false,
        "first": false,
        "gen": " 155",
        "gold": 151,
        "prompt": "What is 72 + 79? Return only the final number."
      }
    ],
    "final_numeric_accuracy": 0.28125,
    "first_numeric_accuracy": 0.28125
  },
  "step": 300,
  "transitive": {
    "depth_1": {
      "N": 128,
      "answer_anywhere_rate": 1.0,
      "chance": 0.5,
      "ci95_first": [
        0.9708620041018919,
        0.9999999999999999
      ],
      "examples": [
        {
          "anywhere": true,
          "first_name": "Dana",
          "first_ok": true,
          "gen": " Dana",
          "gold": "Dana",
          "prompt": "Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Cara",
          "first_ok": true,
          "gen": " Cara",
          "gold": "Cara",
          "prompt": "Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Dana",
          "first_ok": true,
          "gen": " Dana",
          "gold": "Dana",
          "prompt": "Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Gina",
          "first_ok": true,
          "gen": " Gina",
          "gold": "Gina",
          "prompt": "Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Dana",
          "first_ok": true,
          "gen": " Dana",
          "gold": "Dana",
          "prompt": "Dana is taller than Hugo. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Hugo",
          "first_ok": true,
          "gen": " Hugo",
          "gold": "Hugo",
          "prompt": "Dana is taller than Hugo. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Gina",
          "first_ok": true,
          "gen": " Gina",
          "gold": "Gina",
          "prompt": "Gina is taller than Dana. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Dana",
          "first_ok": true,
          "gen": " Dana",
          "gold": "Dana",
          "prompt": "Gina is taller than Dana. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Cara",
          "first_ok": true,
          "gen": " Cara",
          "gold": "Cara",
          "prompt": "Cara is taller than Finn. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Finn",
          "first_ok": true,
          "gen": " Finn",
          "gold": "Finn",
          "prompt": "Cara is taller than Finn. Of all of them, the shortest is who? Return only the name."
        }
      ],
      "first_name_accuracy": 1.0
    },
    "depth_2": {
      "N": 128,
      "answer_anywhere_rate": 1.0,
      "chance": 0.3333333333333333,
      "ci95_first": [
        0.9708620041018919,
        0.9999999999999999
      ],
      "examples": [
        {
          "anywhere": true,
          "first_name": "Evan",
          "first_ok": true,
          "gen": " Evan",
          "gold": "Evan",
          "prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Anna",
          "first_ok": true,
          "gen": " Anna",
          "gold": "Anna",
          "prompt": "Evan is taller than Dana. Dana is taller than Anna. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Ben",
          "first_ok": true,
          "gen": " Ben",
          "gold": "Ben",
          "prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Anna",
          "first_ok": true,
          "gen": " Anna",
          "gold": "Anna",
          "prompt": "Ben is taller than Evan. Evan is taller than Anna. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Finn",
          "first_ok": true,
          "gen": " Finn",
          "gold": "Finn",
          "prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Gina",
          "first_ok": true,
          "gen": " Gina",
          "gold": "Gina",
          "prompt": "Finn is taller than Hugo. Hugo is taller than Gina. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Dana",
          "first_ok": true,
          "gen": " Dana",
          "gold": "Dana",
          "prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Finn",
          "first_ok": true,
          "gen": " Finn",
          "gold": "Finn",
          "prompt": "Dana is taller than Hugo. Hugo is taller than Finn. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Gina",
          "first_ok": true,
          "gen": " Gina",
          "gold": "Gina",
          "prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Cara",
          "first_ok": true,
          "gen": " Cara",
          "gold": "Cara",
          "prompt": "Gina is taller than Dana. Dana is taller than Cara. Of all of them, the shortest is who? Return only the name."
        }
      ],
      "first_name_accuracy": 1.0
    },
    "depth_3": {
      "N": 128,
      "answer_anywhere_rate": 1.0,
      "chance": 0.25,
      "ci95_first": [
        0.9708620041018919,
        0.9999999999999999
      ],
      "examples": [
        {
          "anywhere": true,
          "first_name": "Finn",
          "first_ok": true,
          "gen": " Finn",
          "gold": "Finn",
          "prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Ben",
          "first_ok": true,
          "gen": " Ben",
          "gold": "Ben",
          "prompt": "Finn is taller than Dana. Dana is taller than Hugo. Hugo is taller than Ben. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Ben",
          "first_ok": true,
          "gen": " Ben",
          "gold": "Ben",
          "prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Finn",
          "first_ok": true,
          "gen": " Finn",
          "gold": "Finn",
          "prompt": "Ben is taller than Dana. Dana is taller than Evan. Evan is taller than Finn. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Hugo",
          "first_ok": true,
          "gen": " Hugo",
          "gold": "Hugo",
          "prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Gina",
          "first_ok": true,
          "gen": " Gina",
          "gold": "Gina",
          "prompt": "Hugo is taller than Evan. Evan is taller than Dana. Dana is taller than Gina. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Hugo",
          "first_ok": true,
          "gen": " Hugo",
          "gold": "Hugo",
          "prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Finn",
          "first_ok": true,
          "gen": " Finn",
          "gold": "Finn",
          "prompt": "Hugo is taller than Ben. Ben is taller than Dana. Dana is taller than Finn. Of all of them, the shortest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Anna",
          "first_ok": true,
          "gen": " Anna",
          "gold": "Anna",
          "prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the tallest is who? Return only the name."
        },
        {
          "anywhere": true,
          "first_name": "Finn",
          "first_ok": true,
          "gen": " Finn",
          "gold": "Finn",
          "prompt": "Anna is taller than Dana. Dana is taller than Gina. Gina is taller than Finn. Of all of them, the shortest is who? Return only the name."
        }
      ],
      "first_name_accuracy": 1.0
    }
  }
}