157 lines
5.0 KiB
JSON
157 lines
5.0 KiB
JSON
{
|
|
"model": "appvoid/palmer-006",
|
|
"tokenizer": "appvoid/palmer-006",
|
|
"params": 91131072,
|
|
"dataset": {
|
|
"path": "/home/appvoid/arithmark-3.jsonl",
|
|
"sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
|
|
"items": 1000
|
|
},
|
|
"runtime": {
|
|
"device": "cuda",
|
|
"dtype": "torch.bfloat16",
|
|
"batch_size": 32,
|
|
"max_context": 1024,
|
|
"attention_implementation": "auto",
|
|
"compiled": false,
|
|
"torch_version": "2.9.1+rocm7.2.0.git7e1940d4"
|
|
},
|
|
"primary_metric": "acc_norm",
|
|
"results": {
|
|
"arithmark-3": {
|
|
"acc": 53.2,
|
|
"acc_norm": 52.7,
|
|
"raw_correct": 532,
|
|
"norm_correct": 527,
|
|
"total": 1000,
|
|
"categories": {
|
|
"elementary_school_math_continuation::addition::grades_1_2::easy": {
|
|
"acc": 32.03125,
|
|
"acc_norm": 31.25,
|
|
"raw_correct": 41,
|
|
"norm_correct": 40,
|
|
"total": 128
|
|
},
|
|
"elementary_school_math_continuation::comparison::grades_2_3::medium": {
|
|
"acc": 40.909090909090914,
|
|
"acc_norm": 43.18181818181818,
|
|
"raw_correct": 18,
|
|
"norm_correct": 19,
|
|
"total": 44
|
|
},
|
|
"elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
|
|
"acc": 47.91666666666667,
|
|
"acc_norm": 45.83333333333333,
|
|
"raw_correct": 23,
|
|
"norm_correct": 22,
|
|
"total": 48
|
|
},
|
|
"elementary_school_math_continuation::data::grades_2_3::easy": {
|
|
"acc": 32.55813953488372,
|
|
"acc_norm": 30.23255813953488,
|
|
"raw_correct": 14,
|
|
"norm_correct": 13,
|
|
"total": 43
|
|
},
|
|
"elementary_school_math_continuation::division::grades_3_4::medium": {
|
|
"acc": 87.03703703703704,
|
|
"acc_norm": 87.03703703703704,
|
|
"raw_correct": 47,
|
|
"norm_correct": 47,
|
|
"total": 54
|
|
},
|
|
"elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
|
|
"acc": 28.000000000000004,
|
|
"acc_norm": 26.0,
|
|
"raw_correct": 14,
|
|
"norm_correct": 13,
|
|
"total": 50
|
|
},
|
|
"elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
|
|
"acc": 94.23076923076923,
|
|
"acc_norm": 94.23076923076923,
|
|
"raw_correct": 49,
|
|
"norm_correct": 49,
|
|
"total": 52
|
|
},
|
|
"elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
|
|
"acc": 64.44444444444444,
|
|
"acc_norm": 64.44444444444444,
|
|
"raw_correct": 29,
|
|
"norm_correct": 29,
|
|
"total": 45
|
|
},
|
|
"elementary_school_math_continuation::measurement::grades_2_3::easy": {
|
|
"acc": 68.42105263157895,
|
|
"acc_norm": 67.10526315789474,
|
|
"raw_correct": 52,
|
|
"norm_correct": 51,
|
|
"total": 76
|
|
},
|
|
"elementary_school_math_continuation::money::grades_3_4::medium": {
|
|
"acc": 37.5,
|
|
"acc_norm": 37.5,
|
|
"raw_correct": 24,
|
|
"norm_correct": 24,
|
|
"total": 64
|
|
},
|
|
"elementary_school_math_continuation::multiplication::grades_3_4::medium": {
|
|
"acc": 81.08108108108108,
|
|
"acc_norm": 81.08108108108108,
|
|
"raw_correct": 60,
|
|
"norm_correct": 60,
|
|
"total": 74
|
|
},
|
|
"elementary_school_math_continuation::patterns::grades_3_4::medium": {
|
|
"acc": 64.15094339622641,
|
|
"acc_norm": 64.15094339622641,
|
|
"raw_correct": 34,
|
|
"norm_correct": 34,
|
|
"total": 53
|
|
},
|
|
"elementary_school_math_continuation::subtraction::grades_1_2::easy": {
|
|
"acc": 37.60683760683761,
|
|
"acc_norm": 37.60683760683761,
|
|
"raw_correct": 44,
|
|
"norm_correct": 44,
|
|
"total": 117
|
|
},
|
|
"elementary_school_math_continuation::time::grades_2_3::easy": {
|
|
"acc": 100.0,
|
|
"acc_norm": 100.0,
|
|
"raw_correct": 55,
|
|
"norm_correct": 55,
|
|
"total": 55
|
|
},
|
|
"elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
|
|
"acc": 26.08695652173913,
|
|
"acc_norm": 26.08695652173913,
|
|
"raw_correct": 12,
|
|
"norm_correct": 12,
|
|
"total": 46
|
|
},
|
|
"elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
|
|
"acc": 42.10526315789473,
|
|
"acc_norm": 42.10526315789473,
|
|
"raw_correct": 8,
|
|
"norm_correct": 8,
|
|
"total": 19
|
|
},
|
|
"elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
|
|
"acc": 25.0,
|
|
"acc_norm": 21.875,
|
|
"raw_correct": 8,
|
|
"norm_correct": 7,
|
|
"total": 32
|
|
}
|
|
},
|
|
"timing": {
|
|
"tokenization_seconds": 0.19381939000004422,
|
|
"evaluation_seconds": 17.806661402999907,
|
|
"examples_per_second": 56.15875864475802
|
|
},
|
|
"primary_metric": "acc_norm",
|
|
"primary_acc": 52.7
|
|
}
|
|
}
|
|
} |