171 lines
5.4 KiB
JSON
171 lines
5.4 KiB
JSON
{
|
|
"backend_settings": {
|
|
"apply_chat_template": true,
|
|
"backend": "vllm",
|
|
"batch_size": 32,
|
|
"dtype": "bfloat16",
|
|
"gpu_memory_utilization": 0.82,
|
|
"seed": "0,1234,1234,1234",
|
|
"temperature": "task-default 0",
|
|
"tensor_parallel_size": 1
|
|
},
|
|
"benchmark_files": {
|
|
"base": [
|
|
"gsm8k_base.json",
|
|
"ifeval_base.json",
|
|
"arc_challenge_base.json",
|
|
"mmlu_base.json"
|
|
],
|
|
"kosa": [
|
|
"gsm8k_kosa.json",
|
|
"ifeval_kosa.json",
|
|
"arc_challenge_kosa.json",
|
|
"mmlu_kosa.json"
|
|
]
|
|
},
|
|
"date": "2026-06-11T11:17:13.383746+00:00",
|
|
"harness": "lm-evaluation-harness 0.4.12",
|
|
"scored_rows": [
|
|
{
|
|
"arc_acc_norm": 0.5213310580204779,
|
|
"avg5": 0.7960271980535851,
|
|
"avg6": 0.7729557704902223,
|
|
"display_avg": 0.7729557704902223,
|
|
"display_avg_kind": "avg6",
|
|
"downloads": null,
|
|
"group": "kosa",
|
|
"gsm8k_flex": 0.8559514783927218,
|
|
"gsm8k_strict": 0.8423047763457164,
|
|
"ifeval_inst": 0.9028776978417267,
|
|
"ifeval_prompt": 0.8576709796672828,
|
|
"mmlu": 0.6575986326734083,
|
|
"mmlu_humanities": 0.5883103081827843,
|
|
"mmlu_other": 0.7219182491149019,
|
|
"mmlu_social": 0.7897302567435814,
|
|
"mmlu_stem": 0.5686647637170948,
|
|
"model": "kosa-labs/kosa-4B-it-v1",
|
|
"status": "scored"
|
|
},
|
|
{
|
|
"arc_acc_norm": 0.5170648464163823,
|
|
"avg5": 0.7854459095088681,
|
|
"avg6": 0.766511860924579,
|
|
"display_avg": 0.766511860924579,
|
|
"display_avg_kind": "avg6",
|
|
"downloads": 798,
|
|
"group": "same-base genuine accessible",
|
|
"gsm8k_flex": 0.8688400303260045,
|
|
"gsm8k_strict": 0.8483699772554966,
|
|
"ifeval_inst": 0.8741007194244604,
|
|
"ifeval_prompt": 0.8188539741219963,
|
|
"mmlu": 0.6718416180031335,
|
|
"mmlu_humanities": 0.6063761955366631,
|
|
"mmlu_other": 0.7296427421950434,
|
|
"mmlu_social": 0.7955801104972375,
|
|
"mmlu_stem": 0.5918173168411037,
|
|
"model": "codingmonster1234/chess-sft-modelv2",
|
|
"status": "scored"
|
|
},
|
|
{
|
|
"arc_acc_norm": 0.5366894197952219,
|
|
"avg5": 0.7610974365380638,
|
|
"avg6": null,
|
|
"display_avg": 0.7610974365380638,
|
|
"display_avg_kind": "avg5",
|
|
"downloads": 3126,
|
|
"group": "same-base genuine accessible",
|
|
"gsm8k_flex": 0.8233510235026535,
|
|
"gsm8k_strict": 0.8218347232752085,
|
|
"ifeval_inst": 0.841726618705036,
|
|
"ifeval_prompt": 0.7818853974121996,
|
|
"model": "Applied-Innovation-Center/Karnak-6B-v1.0",
|
|
"status": "scored"
|
|
},
|
|
{
|
|
"arc_acc_norm": 0.42918088737201365,
|
|
"avg5": 0.7317990152643855,
|
|
"avg6": null,
|
|
"display_avg": 0.7317990152643855,
|
|
"display_avg_kind": "avg5",
|
|
"downloads": 28076,
|
|
"group": "previous same-base row",
|
|
"gsm8k_flex": 0.7907505686125853,
|
|
"gsm8k_strict": 0.7278241091736164,
|
|
"ifeval_inst": 0.8812949640287769,
|
|
"ifeval_prompt": 0.8299445471349353,
|
|
"model": "unsloth/Qwen3-4B-Instruct-2507",
|
|
"status": "scored; excluded from genuine verdict"
|
|
},
|
|
{
|
|
"arc_acc_norm": 0.5204778156996587,
|
|
"avg5": 0.7256165583147159,
|
|
"avg6": null,
|
|
"display_avg": 0.7256165583147159,
|
|
"display_avg_kind": "avg5",
|
|
"downloads": 1194286,
|
|
"group": "cross-family",
|
|
"gsm8k_flex": 0.8233510235026535,
|
|
"gsm8k_strict": 0.819560272934041,
|
|
"ifeval_inst": 0.7733812949640287,
|
|
"ifeval_prompt": 0.6913123844731978,
|
|
"model": "microsoft/Phi-4-mini-instruct",
|
|
"status": "scored"
|
|
},
|
|
{
|
|
"arc_acc_norm": 0.4308873720136519,
|
|
"avg5": 0.7349003265000296,
|
|
"avg6": 0.7155717600005587,
|
|
"display_avg": 0.7155717600005587,
|
|
"display_avg_kind": "avg6",
|
|
"downloads": null,
|
|
"group": "base",
|
|
"gsm8k_flex": 0.7915087187263078,
|
|
"gsm8k_strict": 0.7323730098559514,
|
|
"ifeval_inst": 0.8860911270983214,
|
|
"ifeval_prompt": 0.833641404805915,
|
|
"mmlu": 0.6189289275032047,
|
|
"mmlu_humanities": 0.5596174282678003,
|
|
"mmlu_other": 0.6903765690376569,
|
|
"mmlu_social": 0.7565810854728632,
|
|
"mmlu_stem": 0.5026958452267681,
|
|
"model": "Qwen/Qwen3-4B-Instruct-2507",
|
|
"status": "scored"
|
|
},
|
|
{
|
|
"arc_acc_norm": 0.4906143344709898,
|
|
"avg5": 0.6892628966189842,
|
|
"avg6": null,
|
|
"display_avg": 0.6892628966189842,
|
|
"display_avg_kind": "avg5",
|
|
"downloads": 1530,
|
|
"group": "same-base genuine accessible",
|
|
"gsm8k_flex": 0.778620166793025,
|
|
"gsm8k_strict": 0.5981804397270659,
|
|
"ifeval_inst": 0.8321342925659473,
|
|
"ifeval_prompt": 0.7467652495378928,
|
|
"model": "RefalMachine/RuadaptQwen3-4B-Instruct",
|
|
"status": "scored"
|
|
},
|
|
{
|
|
"arc_acc_norm": 0.5452218430034129,
|
|
"avg5": 0.7978064681913222,
|
|
"avg6": 0.775874663292417,
|
|
"display_avg": 0.775874663292417,
|
|
"display_avg_kind": "avg6",
|
|
"downloads": null,
|
|
"group": "merge candidate",
|
|
"gsm8k_flex": 0.8635329795299469,
|
|
"gsm8k_strict": 0.8514025777103866,
|
|
"ifeval_inst": 0.8896882494004796,
|
|
"ifeval_prompt": 0.8391866913123844,
|
|
"mmlu": 0.666215638797892,
|
|
"mmlu_humanities": 0.5957492029755579,
|
|
"mmlu_other": 0.7257804956549726,
|
|
"mmlu_social": 0.7923301917452064,
|
|
"mmlu_stem": 0.5895972090072946,
|
|
"model": "merge:iter1_ta4_cat_t30",
|
|
"status": "scored; full-size merge winner eval; incumbent anchor wins tie"
|
|
}
|
|
]
|
|
}
|