kosa-labs/kosa-4B-it-v1
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: kosa-labs/kosa-4B-it-v1
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-18 16:13:09 +08:00
commit de6644ad83
25 changed files with 10851 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
benchmarks/arc_challenge_base.json Normal file
View File

@@ -0,0 +1,136 @@
{
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
"config": {
"batch_size": "32",
"batch_sizes": [],
"bootstrap_iters": 100000,
"device": "cuda:0",
"fewshot_seed": 1234,
"gen_kwargs": {},
"limit": null,
"model": "hf",
"model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507",
"model_dtype": "torch.bfloat16",
"model_num_parameters": 4022468096,
"model_revision": "main",
"model_sha": "cdbee75f17c01a7cc42f958dc650907174af0554",
"numpy_seed": 1234,
"random_seed": 0,
"torch_seed": 1234,
"use_cache": "[redacted]"
},
"configs": {
"arc_challenge": {
"dataset_name": "ARC-Challenge",
"dataset_path": "allenai/ai2_arc",
"description": "",
"doc_to_choice": "{{choices.text}}",
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": "{{choices.text}}",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "default",
"samples": null,
"split": null,
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
},
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc_norm"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"target_delimiter": " ",
"task": "arc_challenge",
"test_split": "test",
"training_split": "train",
"unsafe_code": false,
"validation_split": "validation"
}
},
"date": 1780955961.7944982,
"eot_token_id": 151645,
"fewshot_as_multiturn": true,
"git_hash": null,
"group_subtasks": {},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"lm_eval_version": "0.4.12",
"max_length": 262144,
"model_name": "Qwen/Qwen3-4B-Instruct-2507",
"model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507",
"model_source": "Qwen/Qwen3-4B-Instruct-2507",
"n-samples": {
"arc_challenge": {
"effective": 1172,
"original": 1172
}
},
"n-shot": {
"arc_challenge": 0
},
"results": {
"arc_challenge": {
"acc,none": 0.4334470989761092,
"acc_norm,none": 0.4308873720136519,
"acc_norm_stderr,none": 0.014471133392642476,
"acc_stderr,none": 0.014481376224558896,
"alias": "arc_challenge",
"name": "arc_challenge",
"sample_len": 1172
}
},
"system_instruction": null,
"system_instruction_sha": null,
"task_hashes": {
"arc_challenge": "9bcbbcee5924b2d8d096f81544b8e32c71e63704d363e292090ad1ba91968f5a"
},
"tokenizer_bos_token": [
null,
"None"
],
"tokenizer_eos_token": [
"<|im_end|>",
"151645"
],
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"total_evaluation_time_seconds": "212.6989970318973",
"transformers_version": "4.57.6",
"upper_git_hash": null,
"versions": {
"arc_challenge": 1.0
}
}

134
benchmarks/arc_challenge_kosa.json Normal file
View File

@@ -0,0 +1,134 @@
{
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
"config": {
"batch_size": "32",
"batch_sizes": [],
"bootstrap_iters": 100000,
"device": "cuda:0",
"fewshot_seed": 1234,
"gen_kwargs": {},
"limit": null,
"model": "vllm",
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
"numpy_seed": 1234,
"random_seed": 0,
"torch_seed": 1234,
"use_cache": "[redacted]"
},
"configs": {
"arc_challenge": {
"dataset_name": "ARC-Challenge",
"dataset_path": "allenai/ai2_arc",
"description": "",
"doc_to_choice": "{{choices.text}}",
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": "{{choices.text}}",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "default",
"samples": null,
"split": null,
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 1.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc"
},
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "acc_norm"
}
],
"num_fewshot": 0,
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"target_delimiter": " ",
"task": "arc_challenge",
"test_split": "test",
"training_split": "train",
"unsafe_code": false,
"validation_split": "validation"
}
},
"date": 1781168624.8804162,
"eot_token_id": 151645,
"fewshot_as_multiturn": true,
"git_hash": null,
"group_subtasks": {},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"lm_eval_version": "0.4.12",
"max_length": 262144,
"model_name": "kosa-labs/kosa-4B-it-v1",
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
"model_source": "kosa-labs/kosa-4B-it-v1",
"n-samples": {
"arc_challenge": {
"effective": 1172,
"original": 1172
}
},
"n-shot": {
"arc_challenge": 0
},
"results": {
"arc_challenge": {
"acc,none": 0.5315699658703071,
"acc_norm,none": 0.5213310580204779,
"acc_norm_stderr,none": 0.014598087973127125,
"acc_stderr,none": 0.014582236460866984,
"alias": "arc_challenge",
"name": "arc_challenge",
"sample_len": 1172
}
},
"system_instruction": null,
"system_instruction_sha": null,
"task_hashes": {
"arc_challenge": "9bcbbcee5924b2d8d096f81544b8e32c71e63704d363e292090ad1ba91968f5a"
},
"tokenizer_bos_token": [
null,
"None"
],
"tokenizer_eos_token": [
"<|im_end|>",
"151645"
],
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"total_evaluation_time_seconds": "334.5368347261101",
"transformers_version": "5.11.0",
"upper_git_hash": null,
"versions": {
"arc_challenge": 1.0
}
}

170
benchmarks/cross_model_comparison_summary.json Normal file
View File

@@ -0,0 +1,170 @@
{
"backend_settings": {
"apply_chat_template": true,
"backend": "vllm",
"batch_size": 32,
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"seed": "0,1234,1234,1234",
"temperature": "task-default 0",
"tensor_parallel_size": 1
},
"benchmark_files": {
"base": [
"gsm8k_base.json",
"ifeval_base.json",
"arc_challenge_base.json",
"mmlu_base.json"
],
"kosa": [
"gsm8k_kosa.json",
"ifeval_kosa.json",
"arc_challenge_kosa.json",
"mmlu_kosa.json"
]
},
"date": "2026-06-11T11:17:13.383746+00:00",
"harness": "lm-evaluation-harness 0.4.12",
"scored_rows": [
{
"arc_acc_norm": 0.5213310580204779,
"avg5": 0.7960271980535851,
"avg6": 0.7729557704902223,
"display_avg": 0.7729557704902223,
"display_avg_kind": "avg6",
"downloads": null,
"group": "kosa",
"gsm8k_flex": 0.8559514783927218,
"gsm8k_strict": 0.8423047763457164,
"ifeval_inst": 0.9028776978417267,
"ifeval_prompt": 0.8576709796672828,
"mmlu": 0.6575986326734083,
"mmlu_humanities": 0.5883103081827843,
"mmlu_other": 0.7219182491149019,
"mmlu_social": 0.7897302567435814,
"mmlu_stem": 0.5686647637170948,
"model": "kosa-labs/kosa-4B-it-v1",
"status": "scored"
},
{
"arc_acc_norm": 0.5170648464163823,
"avg5": 0.7854459095088681,
"avg6": 0.766511860924579,
"display_avg": 0.766511860924579,
"display_avg_kind": "avg6",
"downloads": 798,
"group": "same-base genuine accessible",
"gsm8k_flex": 0.8688400303260045,
"gsm8k_strict": 0.8483699772554966,
"ifeval_inst": 0.8741007194244604,
"ifeval_prompt": 0.8188539741219963,
"mmlu": 0.6718416180031335,
"mmlu_humanities": 0.6063761955366631,
"mmlu_other": 0.7296427421950434,
"mmlu_social": 0.7955801104972375,
"mmlu_stem": 0.5918173168411037,
"model": "codingmonster1234/chess-sft-modelv2",
"status": "scored"
},
{
"arc_acc_norm": 0.5366894197952219,
"avg5": 0.7610974365380638,
"avg6": null,
"display_avg": 0.7610974365380638,
"display_avg_kind": "avg5",
"downloads": 3126,
"group": "same-base genuine accessible",
"gsm8k_flex": 0.8233510235026535,
"gsm8k_strict": 0.8218347232752085,
"ifeval_inst": 0.841726618705036,
"ifeval_prompt": 0.7818853974121996,
"model": "Applied-Innovation-Center/Karnak-6B-v1.0",
"status": "scored"
},
{
"arc_acc_norm": 0.42918088737201365,
"avg5": 0.7317990152643855,
"avg6": null,
"display_avg": 0.7317990152643855,
"display_avg_kind": "avg5",
"downloads": 28076,
"group": "previous same-base row",
"gsm8k_flex": 0.7907505686125853,
"gsm8k_strict": 0.7278241091736164,
"ifeval_inst": 0.8812949640287769,
"ifeval_prompt": 0.8299445471349353,
"model": "unsloth/Qwen3-4B-Instruct-2507",
"status": "scored; excluded from genuine verdict"
},
{
"arc_acc_norm": 0.5204778156996587,
"avg5": 0.7256165583147159,
"avg6": null,
"display_avg": 0.7256165583147159,
"display_avg_kind": "avg5",
"downloads": 1194286,
"group": "cross-family",
"gsm8k_flex": 0.8233510235026535,
"gsm8k_strict": 0.819560272934041,
"ifeval_inst": 0.7733812949640287,
"ifeval_prompt": 0.6913123844731978,
"model": "microsoft/Phi-4-mini-instruct",
"status": "scored"
},
{
"arc_acc_norm": 0.4308873720136519,
"avg5": 0.7349003265000296,
"avg6": 0.7155717600005587,
"display_avg": 0.7155717600005587,
"display_avg_kind": "avg6",
"downloads": null,
"group": "base",
"gsm8k_flex": 0.7915087187263078,
"gsm8k_strict": 0.7323730098559514,
"ifeval_inst": 0.8860911270983214,
"ifeval_prompt": 0.833641404805915,
"mmlu": 0.6189289275032047,
"mmlu_humanities": 0.5596174282678003,
"mmlu_other": 0.6903765690376569,
"mmlu_social": 0.7565810854728632,
"mmlu_stem": 0.5026958452267681,
"model": "Qwen/Qwen3-4B-Instruct-2507",
"status": "scored"
},
{
"arc_acc_norm": 0.4906143344709898,
"avg5": 0.6892628966189842,
"avg6": null,
"display_avg": 0.6892628966189842,
"display_avg_kind": "avg5",
"downloads": 1530,
"group": "same-base genuine accessible",
"gsm8k_flex": 0.778620166793025,
"gsm8k_strict": 0.5981804397270659,
"ifeval_inst": 0.8321342925659473,
"ifeval_prompt": 0.7467652495378928,
"model": "RefalMachine/RuadaptQwen3-4B-Instruct",
"status": "scored"
},
{
"arc_acc_norm": 0.5452218430034129,
"avg5": 0.7978064681913222,
"avg6": 0.775874663292417,
"display_avg": 0.775874663292417,
"display_avg_kind": "avg6",
"downloads": null,
"group": "merge candidate",
"gsm8k_flex": 0.8635329795299469,
"gsm8k_strict": 0.8514025777103866,
"ifeval_inst": 0.8896882494004796,
"ifeval_prompt": 0.8391866913123844,
"mmlu": 0.666215638797892,
"mmlu_humanities": 0.5957492029755579,
"mmlu_other": 0.7257804956549726,
"mmlu_social": 0.7923301917452064,
"mmlu_stem": 0.5895972090072946,
"model": "merge:iter1_ta4_cat_t30",
"status": "scored; full-size merge winner eval; incumbent anchor wins tie"
}
]
}

172
benchmarks/gsm8k_base.json Normal file
View File

@@ -0,0 +1,172 @@
{
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
"config": {
"batch_size": "32",
"batch_sizes": [],
"bootstrap_iters": 100000,
"device": "cuda:0",
"fewshot_seed": 1234,
"gen_kwargs": {},
"limit": null,
"model": "hf",
"model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507",
"model_dtype": "torch.bfloat16",
"model_num_parameters": 4022468096,
"model_revision": "main",
"model_sha": "cdbee75f17c01a7cc42f958dc650907174af0554",
"numpy_seed": 1234,
"random_seed": 0,
"torch_seed": 1234,
"use_cache": "[redacted]"
},
"configs": {
"gsm8k": {
"dataset_name": "main",
"dataset_path": "openai/gsm8k",
"description": "",
"doc_to_target": "{{answer}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": null,
"doc_to_target": "{{answer}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "default",
"samples": null,
"split": "train",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "train",
"filter_list": [
{
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
],
"name": "strict-match"
},
{
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
],
"name": "flexible-extract"
}
],
"generation_kwargs": {
"do_sample": false,
"temperature": 0.0,
"until": [
"Question:",
"</s>",
"<|im_end|>"
]
},
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
"trust_remote_code": true,
"version": 3.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"metric": "exact_match",
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"num_fewshot": 5,
"output_type": "generate_until",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "gsm8k",
"test_split": "test",
"training_split": "train",
"unsafe_code": false
}
},
"date": 1780942857.9108624,
"eot_token_id": 151645,
"fewshot_as_multiturn": true,
"git_hash": null,
"group_subtasks": {},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"lm_eval_version": "0.4.12",
"max_length": 262144,
"model_name": "Qwen/Qwen3-4B-Instruct-2507",
"model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507",
"model_source": "Qwen/Qwen3-4B-Instruct-2507",
"n-samples": {
"gsm8k": {
"effective": 1319,
"original": 1319
}
},
"n-shot": {
"gsm8k": 5
},
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,flexible-extract": 0.7915087187263078,
"exact_match,strict-match": 0.7323730098559514,
"exact_match_stderr,flexible-extract": 0.01118958798579141,
"exact_match_stderr,strict-match": 0.012194764427053348,
"name": "gsm8k",
"sample_len": 1319
}
},
"system_instruction": null,
"system_instruction_sha": null,
"task_hashes": {
"gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206"
},
"tokenizer_bos_token": [
null,
"None"
],
"tokenizer_eos_token": [
"<|im_end|>",
"151645"
],
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"total_evaluation_time_seconds": "1433.1546569541097",
"transformers_version": "4.57.6",
"upper_git_hash": null,
"versions": {
"gsm8k": 3.0
}
}

170
benchmarks/gsm8k_kosa.json Normal file
View File

@@ -0,0 +1,170 @@
{
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
"config": {
"batch_size": "32",
"batch_sizes": [],
"bootstrap_iters": 100000,
"device": "cuda:0",
"fewshot_seed": 1234,
"gen_kwargs": {},
"limit": null,
"model": "vllm",
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
"numpy_seed": 1234,
"random_seed": 0,
"torch_seed": 1234,
"use_cache": "[redacted]"
},
"configs": {
"gsm8k": {
"dataset_name": "main",
"dataset_path": "openai/gsm8k",
"description": "",
"doc_to_target": "{{answer}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_config": {
"doc_to_choice": null,
"doc_to_target": "{{answer}}",
"doc_to_text": "Question: {{question}}\nAnswer:",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "default",
"samples": null,
"split": "train",
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"fewshot_split": "train",
"filter_list": [
{
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
],
"name": "strict-match"
},
{
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
],
"name": "flexible-extract"
}
],
"generation_kwargs": {
"do_sample": false,
"temperature": 0.0,
"until": [
"Question:",
"</s>",
"<|im_end|>"
]
},
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 3.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"metric": "exact_match",
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"num_fewshot": 5,
"output_type": "generate_until",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "gsm8k",
"test_split": "test",
"training_split": "train",
"unsafe_code": false
}
},
"date": 1781168624.8804162,
"eot_token_id": 151645,
"fewshot_as_multiturn": true,
"git_hash": null,
"group_subtasks": {},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"lm_eval_version": "0.4.12",
"max_length": 262144,
"model_name": "kosa-labs/kosa-4B-it-v1",
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
"model_source": "kosa-labs/kosa-4B-it-v1",
"n-samples": {
"gsm8k": {
"effective": 1319,
"original": 1319
}
},
"n-shot": {
"gsm8k": 5
},
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,flexible-extract": 0.8559514783927218,
"exact_match,strict-match": 0.8423047763457164,
"exact_match_stderr,flexible-extract": 0.009672110973065284,
"exact_match_stderr,strict-match": 0.010038901575061383,
"name": "gsm8k",
"sample_len": 1319
}
},
"system_instruction": null,
"system_instruction_sha": null,
"task_hashes": {
"gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206"
},
"tokenizer_bos_token": [
null,
"None"
],
"tokenizer_eos_token": [
"<|im_end|>",
"151645"
],
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"total_evaluation_time_seconds": "334.5368347261101",
"transformers_version": "5.11.0",
"upper_git_hash": null,
"versions": {
"gsm8k": 3.0
}
}

154
benchmarks/ifeval_base.json Normal file
View File

@@ -0,0 +1,154 @@
{
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
"config": {
"batch_size": "16",
"batch_sizes": [],
"bootstrap_iters": 100000,
"device": "cuda:0",
"fewshot_seed": 1234,
"gen_kwargs": {},
"limit": null,
"model": "hf",
"model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507",
"model_dtype": "torch.bfloat16",
"model_num_parameters": 4022468096,
"model_revision": "main",
"model_sha": "cdbee75f17c01a7cc42f958dc650907174af0554",
"numpy_seed": 1234,
"random_seed": 0,
"torch_seed": 1234,
"use_cache": "[redacted]"
},
"configs": {
"ifeval": {
"dataset_path": "google/IFEval",
"description": "",
"doc_to_target": 0,
"doc_to_text": "prompt",
"fewshot_config": {
"doc_to_choice": null,
"doc_to_target": 0,
"doc_to_text": "prompt",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "default",
"samples": null,
"split": null,
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"generation_kwargs": {
"do_sample": false,
"max_gen_toks": 1280,
"temperature": 0.0,
"until": []
},
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"pretrained": "Qwen/Qwen3-4B-Instruct-2507",
"trust_remote_code": true,
"version": 4.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "prompt_level_strict_acc"
},
{
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true,
"metric": "inst_level_strict_acc"
},
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "prompt_level_loose_acc"
},
{
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true,
"metric": "inst_level_loose_acc"
}
],
"num_fewshot": 0,
"output_type": "generate_until",
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "ifeval",
"test_split": "train",
"unsafe_code": false
}
},
"date": 1780940963.7622316,
"eot_token_id": 151645,
"fewshot_as_multiturn": true,
"git_hash": null,
"group_subtasks": {},
"higher_is_better": {
"ifeval": {
"inst_level_loose_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"prompt_level_strict_acc": true
}
},
"lm_eval_version": "0.4.12",
"max_length": 262144,
"model_name": "Qwen/Qwen3-4B-Instruct-2507",
"model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507",
"model_source": "Qwen/Qwen3-4B-Instruct-2507",
"n-samples": {
"ifeval": {
"effective": 541,
"original": 541
}
},
"n-shot": {
"ifeval": 0
},
"results": {
"ifeval": {
"alias": "ifeval",
"inst_level_loose_acc,none": 0.9064748201438849,
"inst_level_loose_acc_stderr,none": "N/A",
"inst_level_strict_acc,none": 0.8860911270983214,
"inst_level_strict_acc_stderr,none": "N/A",
"name": "ifeval",
"prompt_level_loose_acc,none": 0.8650646950092421,
"prompt_level_loose_acc_stderr,none": 0.014702466942017852,
"prompt_level_strict_acc,none": 0.833641404805915,
"prompt_level_strict_acc_stderr,none": 0.016025639930548306,
"sample_len": 541
}
},
"system_instruction": null,
"system_instruction_sha": null,
"task_hashes": {
"ifeval": "e01bec2e716639c6aa89facb027f54c1a8238f533e9488cbce6f5a79ae309c3d"
},
"tokenizer_bos_token": [
null,
"None"
],
"tokenizer_eos_token": [
"<|im_end|>",
"151645"
],
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"total_evaluation_time_seconds": "1874.040236396715",
"transformers_version": "4.57.6",
"upper_git_hash": null,
"versions": {
"ifeval": 4.0
}
}

152
benchmarks/ifeval_kosa.json Normal file
View File

@@ -0,0 +1,152 @@
{
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
"config": {
"batch_size": "32",
"batch_sizes": [],
"bootstrap_iters": 100000,
"device": "cuda:0",
"fewshot_seed": 1234,
"gen_kwargs": {},
"limit": null,
"model": "vllm",
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
"numpy_seed": 1234,
"random_seed": 0,
"torch_seed": 1234,
"use_cache": "[redacted]"
},
"configs": {
"ifeval": {
"dataset_path": "google/IFEval",
"description": "",
"doc_to_target": 0,
"doc_to_text": "prompt",
"fewshot_config": {
"doc_to_choice": null,
"doc_to_target": 0,
"doc_to_text": "prompt",
"fewshot_delimiter": "\n\n",
"fewshot_indices": null,
"gen_prefix": null,
"process_docs": null,
"sampler": "default",
"samples": null,
"split": null,
"target_delimiter": " "
},
"fewshot_delimiter": "\n\n",
"generation_kwargs": {
"do_sample": false,
"max_gen_toks": 1280,
"temperature": 0.0,
"until": []
},
"metadata": {
"config_source": "[redacted]",
"dtype": "bfloat16",
"gpu_memory_utilization": 0.82,
"pretrained": "[redacted]",
"tensor_parallel_size": 1,
"trust_remote_code": true,
"version": 4.0
},
"metric_list": [
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "prompt_level_strict_acc"
},
{
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true,
"metric": "inst_level_strict_acc"
},
{
"aggregation": "mean",
"higher_is_better": true,
"metric": "prompt_level_loose_acc"
},
{
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true,
"metric": "inst_level_loose_acc"
}
],
"num_fewshot": 0,
"output_type": "generate_until",
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"repeats": 1,
"should_decontaminate": false,
"target_delimiter": " ",
"task": "ifeval",
"test_split": "train",
"unsafe_code": false
}
},
"date": 1781168624.8804162,
"eot_token_id": 151645,
"fewshot_as_multiturn": true,
"git_hash": null,
"group_subtasks": {},
"higher_is_better": {
"ifeval": {
"inst_level_loose_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"prompt_level_strict_acc": true
}
},
"lm_eval_version": "0.4.12",
"max_length": 262144,
"model_name": "kosa-labs/kosa-4B-it-v1",
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
"model_source": "kosa-labs/kosa-4B-it-v1",
"n-samples": {
"ifeval": {
"effective": 541,
"original": 541
}
},
"n-shot": {
"ifeval": 0
},
"results": {
"ifeval": {
"alias": "ifeval",
"inst_level_loose_acc,none": 0.9232613908872902,
"inst_level_loose_acc_stderr,none": "N/A",
"inst_level_strict_acc,none": 0.9028776978417267,
"inst_level_strict_acc_stderr,none": "N/A",
"name": "ifeval",
"prompt_level_loose_acc,none": 0.8853974121996303,
"prompt_level_loose_acc_stderr,none": 0.013707853566060982,
"prompt_level_strict_acc,none": 0.8576709796672828,
"prompt_level_strict_acc_stderr,none": 0.015035234562202742,
"sample_len": 541
}
},
"system_instruction": null,
"system_instruction_sha": null,
"task_hashes": {
"ifeval": "e01bec2e716639c6aa89facb027f54c1a8238f533e9488cbce6f5a79ae309c3d"
},
"tokenizer_bos_token": [
null,
"None"
],
"tokenizer_eos_token": [
"<|im_end|>",
"151645"
],
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"total_evaluation_time_seconds": "334.5368347261101",
"transformers_version": "5.11.0",
"upper_git_hash": null,
"versions": {
"ifeval": 4.0
}
}

4530
benchmarks/mmlu_base.json Normal file
View File

File diff suppressed because it is too large Load Diff

4530
benchmarks/mmlu_kosa.json Normal file
View File

File diff suppressed because it is too large Load Diff