{
  "details": [
    {
      "difficulty": "hard",
      "problem_id": "arc196_a",
      "samples": [
        {
          "code_chars": 380,
          "completion_chars": 2880,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 327,
          "completion_chars": 4333,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 5950,
          "end": "format_error: final",
          "passed": false,
          "stop_seen": false,
          "tool_type": "final"
        },
        {
          "code_chars": 318,
          "completion_chars": 5576,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 316,
          "completion_chars": 6824,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 540,
          "completion_chars": 3408,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        }
      ]
    },
    {
      "difficulty": "hard",
      "problem_id": "abc398_f",
      "samples": [
        {
          "code_chars": 332,
          "completion_chars": 5783,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 325,
          "completion_chars": 4684,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 351,
          "completion_chars": 2437,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 194,
          "completion_chars": 4745,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 360,
          "completion_chars": 3907,
          "end": "ok",
          "passed": true,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 6143,
          "end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
          "passed": false,
          "stop_seen": false,
          "tool_type": null
        }
      ]
    },
    {
      "difficulty": "easy",
      "problem_id": "abc400_a",
      "samples": [
        {
          "code_chars": 180,
          "completion_chars": 982,
          "end": "ok",
          "passed": true,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 1062,
          "end": "format_error: final",
          "passed": false,
          "stop_seen": false,
          "tool_type": "final"
        },
        {
          "code_chars": 285,
          "completion_chars": 1636,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 264,
          "completion_chars": 1539,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 165,
          "completion_chars": 886,
          "end": "ok",
          "passed": true,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 175,
          "completion_chars": 940,
          "end": "ok",
          "passed": true,
          "stop_seen": true,
          "tool_type": "final"
        }
      ]
    },
    {
      "difficulty": "hard",
      "problem_id": "arc196_c",
      "samples": [
        {
          "code_chars": 0,
          "completion_chars": 9181,
          "end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
          "passed": false,
          "stop_seen": true,
          "tool_type": null
        },
        {
          "code_chars": 601,
          "completion_chars": 7205,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 371,
          "completion_chars": 3946,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 1489,
          "completion_chars": 8048,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 5614,
          "end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
          "passed": false,
          "stop_seen": false,
          "tool_type": null
        },
        {
          "code_chars": 125,
          "completion_chars": 7651,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        }
      ]
    },
    {
      "difficulty": "medium",
      "problem_id": "abc400_d",
      "samples": [
        {
          "code_chars": 2468,
          "completion_chars": 5103,
          "end": "timeout",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 3814,
          "end": "format_error: final",
          "passed": false,
          "stop_seen": false,
          "tool_type": "final"
        },
        {
          "code_chars": 1121,
          "completion_chars": 9244,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 3572,
          "end": "format_error: final",
          "passed": false,
          "stop_seen": false,
          "tool_type": "final"
        },
        {
          "code_chars": 1874,
          "completion_chars": 4560,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 3678,
          "end": "format_error: final",
          "passed": false,
          "stop_seen": false,
          "tool_type": "final"
        }
      ]
    },
    {
      "difficulty": "hard",
      "problem_id": "abc398_g",
      "samples": [
        {
          "code_chars": 820,
          "completion_chars": 3243,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 1127,
          "completion_chars": 5958,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 842,
          "completion_chars": 6255,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 326,
          "completion_chars": 6262,
          "end": "wrong_answer",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 869,
          "completion_chars": 4210,
          "end": "runtime_error",
          "passed": false,
          "stop_seen": true,
          "tool_type": "final"
        },
        {
          "code_chars": 0,
          "completion_chars": 3812,
          "end": "format_error: Could not parse 'Tool type' or 'Tool query' from the response. Ensure you output an <answer> block with 'Tool type:' and 'Tool query:'.",
          "passed": false,
          "stop_seen": false,
          "tool_type": null
        }
      ]
    }
  ],
  "eval_limit": 6,
  "max_new_tokens": 1024,
  "pass_at_1": 0.1111111111111111,
  "passed_samples": 4,
  "samples_per_problem": 6,
  "tag": "baseline",
  "temperature": 1.0,
  "tool_turn_limit": 1,
  "top_p": 0.95,
  "total_samples": 36
}