{
  "model": "appvoid/palmer-006",
  "tokenizer": "appvoid/palmer-006",
  "params": 91131072,
  "dataset": {
    "path": "/home/appvoid/arithmark-3.jsonl",
    "sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
    "items": 1000
  },
  "runtime": {
    "device": "cuda",
    "dtype": "torch.bfloat16",
    "batch_size": 32,
    "max_context": 1024,
    "attention_implementation": "auto",
    "compiled": false,
    "torch_version": "2.9.1+rocm7.2.0.git7e1940d4"
  },
  "primary_metric": "acc_norm",
  "results": {
    "arithmark-3": {
      "acc": 53.2,
      "acc_norm": 52.7,
      "raw_correct": 532,
      "norm_correct": 527,
      "total": 1000,
      "categories": {
        "elementary_school_math_continuation::addition::grades_1_2::easy": {
          "acc": 32.03125,
          "acc_norm": 31.25,
          "raw_correct": 41,
          "norm_correct": 40,
          "total": 128
        },
        "elementary_school_math_continuation::comparison::grades_2_3::medium": {
          "acc": 40.909090909090914,
          "acc_norm": 43.18181818181818,
          "raw_correct": 18,
          "norm_correct": 19,
          "total": 44
        },
        "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
          "acc": 47.91666666666667,
          "acc_norm": 45.83333333333333,
          "raw_correct": 23,
          "norm_correct": 22,
          "total": 48
        },
        "elementary_school_math_continuation::data::grades_2_3::easy": {
          "acc": 32.55813953488372,
          "acc_norm": 30.23255813953488,
          "raw_correct": 14,
          "norm_correct": 13,
          "total": 43
        },
        "elementary_school_math_continuation::division::grades_3_4::medium": {
          "acc": 87.03703703703704,
          "acc_norm": 87.03703703703704,
          "raw_correct": 47,
          "norm_correct": 47,
          "total": 54
        },
        "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
          "acc": 28.000000000000004,
          "acc_norm": 26.0,
          "raw_correct": 14,
          "norm_correct": 13,
          "total": 50
        },
        "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
          "acc": 94.23076923076923,
          "acc_norm": 94.23076923076923,
          "raw_correct": 49,
          "norm_correct": 49,
          "total": 52
        },
        "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
          "acc": 64.44444444444444,
          "acc_norm": 64.44444444444444,
          "raw_correct": 29,
          "norm_correct": 29,
          "total": 45
        },
        "elementary_school_math_continuation::measurement::grades_2_3::easy": {
          "acc": 68.42105263157895,
          "acc_norm": 67.10526315789474,
          "raw_correct": 52,
          "norm_correct": 51,
          "total": 76
        },
        "elementary_school_math_continuation::money::grades_3_4::medium": {
          "acc": 37.5,
          "acc_norm": 37.5,
          "raw_correct": 24,
          "norm_correct": 24,
          "total": 64
        },
        "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
          "acc": 81.08108108108108,
          "acc_norm": 81.08108108108108,
          "raw_correct": 60,
          "norm_correct": 60,
          "total": 74
        },
        "elementary_school_math_continuation::patterns::grades_3_4::medium": {
          "acc": 64.15094339622641,
          "acc_norm": 64.15094339622641,
          "raw_correct": 34,
          "norm_correct": 34,
          "total": 53
        },
        "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
          "acc": 37.60683760683761,
          "acc_norm": 37.60683760683761,
          "raw_correct": 44,
          "norm_correct": 44,
          "total": 117
        },
        "elementary_school_math_continuation::time::grades_2_3::easy": {
          "acc": 100.0,
          "acc_norm": 100.0,
          "raw_correct": 55,
          "norm_correct": 55,
          "total": 55
        },
        "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
          "acc": 26.08695652173913,
          "acc_norm": 26.08695652173913,
          "raw_correct": 12,
          "norm_correct": 12,
          "total": 46
        },
        "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
          "acc": 42.10526315789473,
          "acc_norm": 42.10526315789473,
          "raw_correct": 8,
          "norm_correct": 8,
          "total": 19
        },
        "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
          "acc": 25.0,
          "acc_norm": 21.875,
          "raw_correct": 8,
          "norm_correct": 7,
          "total": 32
        }
      },
      "timing": {
        "tokenization_seconds": 0.19381939000004422,
        "evaluation_seconds": 17.806661402999907,
        "examples_per_second": 56.15875864475802
      },
      "primary_metric": "acc_norm",
      "primary_acc": 52.7
    }
  }
}