初始化项目，由ModelHub XC社区提供模型
Model: appvoid/palmer-006 Source: Original Platform
This commit is contained in:
ModelHub XC
157
reports/appvoid_palmer-006_arithmark-3_results.json
Normal file
157
reports/appvoid_palmer-006_arithmark-3_results.json
Normal file
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"model": "appvoid/palmer-006",
|
||||
"tokenizer": "appvoid/palmer-006",
|
||||
"params": 91131072,
|
||||
"dataset": {
|
||||
"path": "/home/appvoid/arithmark-3.jsonl",
|
||||
"sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
|
||||
"items": 1000
|
||||
},
|
||||
"runtime": {
|
||||
"device": "cuda",
|
||||
"dtype": "torch.bfloat16",
|
||||
"batch_size": 32,
|
||||
"max_context": 1024,
|
||||
"attention_implementation": "auto",
|
||||
"compiled": false,
|
||||
"torch_version": "2.9.1+rocm7.2.0.git7e1940d4"
|
||||
},
|
||||
"primary_metric": "acc_norm",
|
||||
"results": {
|
||||
"arithmark-3": {
|
||||
"acc": 53.2,
|
||||
"acc_norm": 52.7,
|
||||
"raw_correct": 532,
|
||||
"norm_correct": 527,
|
||||
"total": 1000,
|
||||
"categories": {
|
||||
"elementary_school_math_continuation::addition::grades_1_2::easy": {
|
||||
"acc": 32.03125,
|
||||
"acc_norm": 31.25,
|
||||
"raw_correct": 41,
|
||||
"norm_correct": 40,
|
||||
"total": 128
|
||||
},
|
||||
"elementary_school_math_continuation::comparison::grades_2_3::medium": {
|
||||
"acc": 40.909090909090914,
|
||||
"acc_norm": 43.18181818181818,
|
||||
"raw_correct": 18,
|
||||
"norm_correct": 19,
|
||||
"total": 44
|
||||
},
|
||||
"elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
|
||||
"acc": 47.91666666666667,
|
||||
"acc_norm": 45.83333333333333,
|
||||
"raw_correct": 23,
|
||||
"norm_correct": 22,
|
||||
"total": 48
|
||||
},
|
||||
"elementary_school_math_continuation::data::grades_2_3::easy": {
|
||||
"acc": 32.55813953488372,
|
||||
"acc_norm": 30.23255813953488,
|
||||
"raw_correct": 14,
|
||||
"norm_correct": 13,
|
||||
"total": 43
|
||||
},
|
||||
"elementary_school_math_continuation::division::grades_3_4::medium": {
|
||||
"acc": 87.03703703703704,
|
||||
"acc_norm": 87.03703703703704,
|
||||
"raw_correct": 47,
|
||||
"norm_correct": 47,
|
||||
"total": 54
|
||||
},
|
||||
"elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
|
||||
"acc": 28.000000000000004,
|
||||
"acc_norm": 26.0,
|
||||
"raw_correct": 14,
|
||||
"norm_correct": 13,
|
||||
"total": 50
|
||||
},
|
||||
"elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
|
||||
"acc": 94.23076923076923,
|
||||
"acc_norm": 94.23076923076923,
|
||||
"raw_correct": 49,
|
||||
"norm_correct": 49,
|
||||
"total": 52
|
||||
},
|
||||
"elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
|
||||
"acc": 64.44444444444444,
|
||||
"acc_norm": 64.44444444444444,
|
||||
"raw_correct": 29,
|
||||
"norm_correct": 29,
|
||||
"total": 45
|
||||
},
|
||||
"elementary_school_math_continuation::measurement::grades_2_3::easy": {
|
||||
"acc": 68.42105263157895,
|
||||
"acc_norm": 67.10526315789474,
|
||||
"raw_correct": 52,
|
||||
"norm_correct": 51,
|
||||
"total": 76
|
||||
},
|
||||
"elementary_school_math_continuation::money::grades_3_4::medium": {
|
||||
"acc": 37.5,
|
||||
"acc_norm": 37.5,
|
||||
"raw_correct": 24,
|
||||
"norm_correct": 24,
|
||||
"total": 64
|
||||
},
|
||||
"elementary_school_math_continuation::multiplication::grades_3_4::medium": {
|
||||
"acc": 81.08108108108108,
|
||||
"acc_norm": 81.08108108108108,
|
||||
"raw_correct": 60,
|
||||
"norm_correct": 60,
|
||||
"total": 74
|
||||
},
|
||||
"elementary_school_math_continuation::patterns::grades_3_4::medium": {
|
||||
"acc": 64.15094339622641,
|
||||
"acc_norm": 64.15094339622641,
|
||||
"raw_correct": 34,
|
||||
"norm_correct": 34,
|
||||
"total": 53
|
||||
},
|
||||
"elementary_school_math_continuation::subtraction::grades_1_2::easy": {
|
||||
"acc": 37.60683760683761,
|
||||
"acc_norm": 37.60683760683761,
|
||||
"raw_correct": 44,
|
||||
"norm_correct": 44,
|
||||
"total": 117
|
||||
},
|
||||
"elementary_school_math_continuation::time::grades_2_3::easy": {
|
||||
"acc": 100.0,
|
||||
"acc_norm": 100.0,
|
||||
"raw_correct": 55,
|
||||
"norm_correct": 55,
|
||||
"total": 55
|
||||
},
|
||||
"elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
|
||||
"acc": 26.08695652173913,
|
||||
"acc_norm": 26.08695652173913,
|
||||
"raw_correct": 12,
|
||||
"norm_correct": 12,
|
||||
"total": 46
|
||||
},
|
||||
"elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
|
||||
"acc": 42.10526315789473,
|
||||
"acc_norm": 42.10526315789473,
|
||||
"raw_correct": 8,
|
||||
"norm_correct": 8,
|
||||
"total": 19
|
||||
},
|
||||
"elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
|
||||
"acc": 25.0,
|
||||
"acc_norm": 21.875,
|
||||
"raw_correct": 8,
|
||||
"norm_correct": 7,
|
||||
"total": 32
|
||||
}
|
||||
},
|
||||
"timing": {
|
||||
"tokenization_seconds": 0.19381939000004422,
|
||||
"evaluation_seconds": 17.806661402999907,
|
||||
"examples_per_second": 56.15875864475802
|
||||
},
|
||||
"primary_metric": "acc_norm",
|
||||
"primary_acc": 52.7
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user