commit de6644ad835915884b37fc5fdf5a46d74e3a5e28 Author: ModelHub XC Date: Sat Jul 18 16:13:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: kosa-labs/kosa-4B-it-v1 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7591f5b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +gguf/kosa-4B-it-v1-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +gguf/kosa-4B-it-v1-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +gguf/kosa-4B-it-v1-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..f4f0b91 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +license: apache-2.0 +pipeline_tag: text-generation +language: +- en +tags: +- qwen3 +- instruct +--- +# kosa-4B-it-v1 + +An instruction-tuned model built on Qwen/Qwen3-4B-Instruct-2507 using the Kosa Method. Kosa Labs is a UK-based independent lab. + +## Benchmarks + +| Benchmark | Qwen3-4B-Instruct-2507 | kosa-4B-it-v1 | +|---|---:|---:| +| GSM8K (strict) | 73.24% | 84.23% | +| GSM8K (flexible) | 79.15% | 85.60% | +| IFEval (prompt strict) | 83.36% | 85.77% | +| IFEval (instruction strict) | 88.61% | 90.29% | +| ARC-Challenge (acc_norm) | 43.09% | 52.13% | +| MMLU | 61.89% | 65.76% | +| **Average** | **71.56%** | **77.30%** | + +Both models evaluated in the same session with identical settings: lm-evaluation-harness 0.4.12, vLLM, bfloat16, temperature 0, chat template applied. Raw result files in /benchmarks. Training data verified clean against all four benchmark test sets (13-gram and 8-gram overlap checks with positive-control validation). + +## Usage + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +model_id = "kosa-labs/kosa-4B-it-v1" +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", +) + +messages = [{"role": "user", "content": "Write a concise project update."}] +inputs = tokenizer.apply_chat_template( + messages, + add_generation_prompt=True, + return_tensors="pt", +).to(model.device) +outputs = model.generate(inputs, max_new_tokens=256) +print(tokenizer.decode(outputs[0][inputs.shape[-1]:], skip_special_tokens=True)) +``` + +GGUF quantizations are available in /gguf: +- gguf/kosa-4B-it-v1-Q4_K_M.gguf +- gguf/kosa-4B-it-v1-Q5_K_M.gguf +- gguf/kosa-4B-it-v1-Q8_0.gguf diff --git a/benchmarks/arc_challenge_base.json b/benchmarks/arc_challenge_base.json new file mode 100644 index 0000000..430c23a --- /dev/null +++ b/benchmarks/arc_challenge_base.json @@ -0,0 +1,136 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "32", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "hf", + "model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507", + "model_dtype": "torch.bfloat16", + "model_num_parameters": 4022468096, + "model_revision": "main", + "model_sha": "cdbee75f17c01a7cc42f958dc650907174af0554", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "arc_challenge": { + "dataset_name": "ARC-Challenge", + "dataset_path": "allenai/ai2_arc", + "description": "", + "doc_to_choice": "{{choices.text}}", + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": "{{choices.text}}", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "default", + "samples": null, + "split": null, + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + }, + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc_norm" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "target_delimiter": " ", + "task": "arc_challenge", + "test_split": "test", + "training_split": "train", + "unsafe_code": false, + "validation_split": "validation" + } + }, + "date": 1780955961.7944982, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": {}, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "Qwen/Qwen3-4B-Instruct-2507", + "model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507", + "model_source": "Qwen/Qwen3-4B-Instruct-2507", + "n-samples": { + "arc_challenge": { + "effective": 1172, + "original": 1172 + } + }, + "n-shot": { + "arc_challenge": 0 + }, + "results": { + "arc_challenge": { + "acc,none": 0.4334470989761092, + "acc_norm,none": 0.4308873720136519, + "acc_norm_stderr,none": 0.014471133392642476, + "acc_stderr,none": 0.014481376224558896, + "alias": "arc_challenge", + "name": "arc_challenge", + "sample_len": 1172 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "arc_challenge": "9bcbbcee5924b2d8d096f81544b8e32c71e63704d363e292090ad1ba91968f5a" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "212.6989970318973", + "transformers_version": "4.57.6", + "upper_git_hash": null, + "versions": { + "arc_challenge": 1.0 + } +} diff --git a/benchmarks/arc_challenge_kosa.json b/benchmarks/arc_challenge_kosa.json new file mode 100644 index 0000000..8783f5c --- /dev/null +++ b/benchmarks/arc_challenge_kosa.json @@ -0,0 +1,134 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "32", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "vllm", + "model_args": "pretrained=kosa-labs/kosa-4B-it-v1", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "arc_challenge": { + "dataset_name": "ARC-Challenge", + "dataset_path": "allenai/ai2_arc", + "description": "", + "doc_to_choice": "{{choices.text}}", + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": "{{choices.text}}", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "default", + "samples": null, + "split": null, + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + }, + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc_norm" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "target_delimiter": " ", + "task": "arc_challenge", + "test_split": "test", + "training_split": "train", + "unsafe_code": false, + "validation_split": "validation" + } + }, + "date": 1781168624.8804162, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": {}, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "kosa-labs/kosa-4B-it-v1", + "model_name_sanitized": "kosa-labs/kosa-4B-it-v1", + "model_source": "kosa-labs/kosa-4B-it-v1", + "n-samples": { + "arc_challenge": { + "effective": 1172, + "original": 1172 + } + }, + "n-shot": { + "arc_challenge": 0 + }, + "results": { + "arc_challenge": { + "acc,none": 0.5315699658703071, + "acc_norm,none": 0.5213310580204779, + "acc_norm_stderr,none": 0.014598087973127125, + "acc_stderr,none": 0.014582236460866984, + "alias": "arc_challenge", + "name": "arc_challenge", + "sample_len": 1172 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "arc_challenge": "9bcbbcee5924b2d8d096f81544b8e32c71e63704d363e292090ad1ba91968f5a" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "334.5368347261101", + "transformers_version": "5.11.0", + "upper_git_hash": null, + "versions": { + "arc_challenge": 1.0 + } +} diff --git a/benchmarks/cross_model_comparison_summary.json b/benchmarks/cross_model_comparison_summary.json new file mode 100644 index 0000000..f4dab7d --- /dev/null +++ b/benchmarks/cross_model_comparison_summary.json @@ -0,0 +1,170 @@ +{ + "backend_settings": { + "apply_chat_template": true, + "backend": "vllm", + "batch_size": 32, + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "seed": "0,1234,1234,1234", + "temperature": "task-default 0", + "tensor_parallel_size": 1 + }, + "benchmark_files": { + "base": [ + "gsm8k_base.json", + "ifeval_base.json", + "arc_challenge_base.json", + "mmlu_base.json" + ], + "kosa": [ + "gsm8k_kosa.json", + "ifeval_kosa.json", + "arc_challenge_kosa.json", + "mmlu_kosa.json" + ] + }, + "date": "2026-06-11T11:17:13.383746+00:00", + "harness": "lm-evaluation-harness 0.4.12", + "scored_rows": [ + { + "arc_acc_norm": 0.5213310580204779, + "avg5": 0.7960271980535851, + "avg6": 0.7729557704902223, + "display_avg": 0.7729557704902223, + "display_avg_kind": "avg6", + "downloads": null, + "group": "kosa", + "gsm8k_flex": 0.8559514783927218, + "gsm8k_strict": 0.8423047763457164, + "ifeval_inst": 0.9028776978417267, + "ifeval_prompt": 0.8576709796672828, + "mmlu": 0.6575986326734083, + "mmlu_humanities": 0.5883103081827843, + "mmlu_other": 0.7219182491149019, + "mmlu_social": 0.7897302567435814, + "mmlu_stem": 0.5686647637170948, + "model": "kosa-labs/kosa-4B-it-v1", + "status": "scored" + }, + { + "arc_acc_norm": 0.5170648464163823, + "avg5": 0.7854459095088681, + "avg6": 0.766511860924579, + "display_avg": 0.766511860924579, + "display_avg_kind": "avg6", + "downloads": 798, + "group": "same-base genuine accessible", + "gsm8k_flex": 0.8688400303260045, + "gsm8k_strict": 0.8483699772554966, + "ifeval_inst": 0.8741007194244604, + "ifeval_prompt": 0.8188539741219963, + "mmlu": 0.6718416180031335, + "mmlu_humanities": 0.6063761955366631, + "mmlu_other": 0.7296427421950434, + "mmlu_social": 0.7955801104972375, + "mmlu_stem": 0.5918173168411037, + "model": "codingmonster1234/chess-sft-modelv2", + "status": "scored" + }, + { + "arc_acc_norm": 0.5366894197952219, + "avg5": 0.7610974365380638, + "avg6": null, + "display_avg": 0.7610974365380638, + "display_avg_kind": "avg5", + "downloads": 3126, + "group": "same-base genuine accessible", + "gsm8k_flex": 0.8233510235026535, + "gsm8k_strict": 0.8218347232752085, + "ifeval_inst": 0.841726618705036, + "ifeval_prompt": 0.7818853974121996, + "model": "Applied-Innovation-Center/Karnak-6B-v1.0", + "status": "scored" + }, + { + "arc_acc_norm": 0.42918088737201365, + "avg5": 0.7317990152643855, + "avg6": null, + "display_avg": 0.7317990152643855, + "display_avg_kind": "avg5", + "downloads": 28076, + "group": "previous same-base row", + "gsm8k_flex": 0.7907505686125853, + "gsm8k_strict": 0.7278241091736164, + "ifeval_inst": 0.8812949640287769, + "ifeval_prompt": 0.8299445471349353, + "model": "unsloth/Qwen3-4B-Instruct-2507", + "status": "scored; excluded from genuine verdict" + }, + { + "arc_acc_norm": 0.5204778156996587, + "avg5": 0.7256165583147159, + "avg6": null, + "display_avg": 0.7256165583147159, + "display_avg_kind": "avg5", + "downloads": 1194286, + "group": "cross-family", + "gsm8k_flex": 0.8233510235026535, + "gsm8k_strict": 0.819560272934041, + "ifeval_inst": 0.7733812949640287, + "ifeval_prompt": 0.6913123844731978, + "model": "microsoft/Phi-4-mini-instruct", + "status": "scored" + }, + { + "arc_acc_norm": 0.4308873720136519, + "avg5": 0.7349003265000296, + "avg6": 0.7155717600005587, + "display_avg": 0.7155717600005587, + "display_avg_kind": "avg6", + "downloads": null, + "group": "base", + "gsm8k_flex": 0.7915087187263078, + "gsm8k_strict": 0.7323730098559514, + "ifeval_inst": 0.8860911270983214, + "ifeval_prompt": 0.833641404805915, + "mmlu": 0.6189289275032047, + "mmlu_humanities": 0.5596174282678003, + "mmlu_other": 0.6903765690376569, + "mmlu_social": 0.7565810854728632, + "mmlu_stem": 0.5026958452267681, + "model": "Qwen/Qwen3-4B-Instruct-2507", + "status": "scored" + }, + { + "arc_acc_norm": 0.4906143344709898, + "avg5": 0.6892628966189842, + "avg6": null, + "display_avg": 0.6892628966189842, + "display_avg_kind": "avg5", + "downloads": 1530, + "group": "same-base genuine accessible", + "gsm8k_flex": 0.778620166793025, + "gsm8k_strict": 0.5981804397270659, + "ifeval_inst": 0.8321342925659473, + "ifeval_prompt": 0.7467652495378928, + "model": "RefalMachine/RuadaptQwen3-4B-Instruct", + "status": "scored" + }, + { + "arc_acc_norm": 0.5452218430034129, + "avg5": 0.7978064681913222, + "avg6": 0.775874663292417, + "display_avg": 0.775874663292417, + "display_avg_kind": "avg6", + "downloads": null, + "group": "merge candidate", + "gsm8k_flex": 0.8635329795299469, + "gsm8k_strict": 0.8514025777103866, + "ifeval_inst": 0.8896882494004796, + "ifeval_prompt": 0.8391866913123844, + "mmlu": 0.666215638797892, + "mmlu_humanities": 0.5957492029755579, + "mmlu_other": 0.7257804956549726, + "mmlu_social": 0.7923301917452064, + "mmlu_stem": 0.5895972090072946, + "model": "merge:iter1_ta4_cat_t30", + "status": "scored; full-size merge winner eval; incumbent anchor wins tie" + } + ] +} diff --git a/benchmarks/gsm8k_base.json b/benchmarks/gsm8k_base.json new file mode 100644 index 0000000..3386f3f --- /dev/null +++ b/benchmarks/gsm8k_base.json @@ -0,0 +1,172 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "32", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "hf", + "model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507", + "model_dtype": "torch.bfloat16", + "model_num_parameters": 4022468096, + "model_revision": "main", + "model_sha": "cdbee75f17c01a7cc42f958dc650907174af0554", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "gsm8k": { + "dataset_name": "main", + "dataset_path": "openai/gsm8k", + "description": "", + "doc_to_target": "{{answer}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": null, + "doc_to_target": "{{answer}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "default", + "samples": null, + "split": "train", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "train", + "filter_list": [ + { + "filter": [ + { + "function": "regex", + "regex_pattern": "#### (\\-?[0-9\\.\\,]+)" + }, + { + "function": "take_first" + } + ], + "name": "strict-match" + }, + { + "filter": [ + { + "function": "regex", + "group_select": -1, + "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)" + }, + { + "function": "take_first" + } + ], + "name": "flexible-extract" + } + ], + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "until": [ + "Question:", + "", + "<|im_end|>" + ] + }, + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "trust_remote_code": true, + "version": 3.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "ignore_case": true, + "ignore_punctuation": false, + "metric": "exact_match", + "regexes_to_ignore": [ + ",", + "\\$", + "(?s).*#### ", + "\\.$" + ] + } + ], + "num_fewshot": 5, + "output_type": "generate_until", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "gsm8k", + "test_split": "test", + "training_split": "train", + "unsafe_code": false + } + }, + "date": 1780942857.9108624, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": {}, + "higher_is_better": { + "gsm8k": { + "exact_match": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "Qwen/Qwen3-4B-Instruct-2507", + "model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507", + "model_source": "Qwen/Qwen3-4B-Instruct-2507", + "n-samples": { + "gsm8k": { + "effective": 1319, + "original": 1319 + } + }, + "n-shot": { + "gsm8k": 5 + }, + "results": { + "gsm8k": { + "alias": "gsm8k", + "exact_match,flexible-extract": 0.7915087187263078, + "exact_match,strict-match": 0.7323730098559514, + "exact_match_stderr,flexible-extract": 0.01118958798579141, + "exact_match_stderr,strict-match": 0.012194764427053348, + "name": "gsm8k", + "sample_len": 1319 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "1433.1546569541097", + "transformers_version": "4.57.6", + "upper_git_hash": null, + "versions": { + "gsm8k": 3.0 + } +} diff --git a/benchmarks/gsm8k_kosa.json b/benchmarks/gsm8k_kosa.json new file mode 100644 index 0000000..755bd46 --- /dev/null +++ b/benchmarks/gsm8k_kosa.json @@ -0,0 +1,170 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "32", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "vllm", + "model_args": "pretrained=kosa-labs/kosa-4B-it-v1", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "gsm8k": { + "dataset_name": "main", + "dataset_path": "openai/gsm8k", + "description": "", + "doc_to_target": "{{answer}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": null, + "doc_to_target": "{{answer}}", + "doc_to_text": "Question: {{question}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "default", + "samples": null, + "split": "train", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "train", + "filter_list": [ + { + "filter": [ + { + "function": "regex", + "regex_pattern": "#### (\\-?[0-9\\.\\,]+)" + }, + { + "function": "take_first" + } + ], + "name": "strict-match" + }, + { + "filter": [ + { + "function": "regex", + "group_select": -1, + "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)" + }, + { + "function": "take_first" + } + ], + "name": "flexible-extract" + } + ], + "generation_kwargs": { + "do_sample": false, + "temperature": 0.0, + "until": [ + "Question:", + "", + "<|im_end|>" + ] + }, + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 3.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "ignore_case": true, + "ignore_punctuation": false, + "metric": "exact_match", + "regexes_to_ignore": [ + ",", + "\\$", + "(?s).*#### ", + "\\.$" + ] + } + ], + "num_fewshot": 5, + "output_type": "generate_until", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "gsm8k", + "test_split": "test", + "training_split": "train", + "unsafe_code": false + } + }, + "date": 1781168624.8804162, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": {}, + "higher_is_better": { + "gsm8k": { + "exact_match": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "kosa-labs/kosa-4B-it-v1", + "model_name_sanitized": "kosa-labs/kosa-4B-it-v1", + "model_source": "kosa-labs/kosa-4B-it-v1", + "n-samples": { + "gsm8k": { + "effective": 1319, + "original": 1319 + } + }, + "n-shot": { + "gsm8k": 5 + }, + "results": { + "gsm8k": { + "alias": "gsm8k", + "exact_match,flexible-extract": 0.8559514783927218, + "exact_match,strict-match": 0.8423047763457164, + "exact_match_stderr,flexible-extract": 0.009672110973065284, + "exact_match_stderr,strict-match": 0.010038901575061383, + "name": "gsm8k", + "sample_len": 1319 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "334.5368347261101", + "transformers_version": "5.11.0", + "upper_git_hash": null, + "versions": { + "gsm8k": 3.0 + } +} diff --git a/benchmarks/ifeval_base.json b/benchmarks/ifeval_base.json new file mode 100644 index 0000000..d4fcc55 --- /dev/null +++ b/benchmarks/ifeval_base.json @@ -0,0 +1,154 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "16", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "hf", + "model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507", + "model_dtype": "torch.bfloat16", + "model_num_parameters": 4022468096, + "model_revision": "main", + "model_sha": "cdbee75f17c01a7cc42f958dc650907174af0554", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "ifeval": { + "dataset_path": "google/IFEval", + "description": "", + "doc_to_target": 0, + "doc_to_text": "prompt", + "fewshot_config": { + "doc_to_choice": null, + "doc_to_target": 0, + "doc_to_text": "prompt", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "default", + "samples": null, + "split": null, + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "trust_remote_code": true, + "version": 4.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "prompt_level_strict_acc" + }, + { + "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", + "higher_is_better": true, + "metric": "inst_level_strict_acc" + }, + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "prompt_level_loose_acc" + }, + { + "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", + "higher_is_better": true, + "metric": "inst_level_loose_acc" + } + ], + "num_fewshot": 0, + "output_type": "generate_until", + "process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "ifeval", + "test_split": "train", + "unsafe_code": false + } + }, + "date": 1780940963.7622316, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": {}, + "higher_is_better": { + "ifeval": { + "inst_level_loose_acc": true, + "inst_level_strict_acc": true, + "prompt_level_loose_acc": true, + "prompt_level_strict_acc": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "Qwen/Qwen3-4B-Instruct-2507", + "model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507", + "model_source": "Qwen/Qwen3-4B-Instruct-2507", + "n-samples": { + "ifeval": { + "effective": 541, + "original": 541 + } + }, + "n-shot": { + "ifeval": 0 + }, + "results": { + "ifeval": { + "alias": "ifeval", + "inst_level_loose_acc,none": 0.9064748201438849, + "inst_level_loose_acc_stderr,none": "N/A", + "inst_level_strict_acc,none": 0.8860911270983214, + "inst_level_strict_acc_stderr,none": "N/A", + "name": "ifeval", + "prompt_level_loose_acc,none": 0.8650646950092421, + "prompt_level_loose_acc_stderr,none": 0.014702466942017852, + "prompt_level_strict_acc,none": 0.833641404805915, + "prompt_level_strict_acc_stderr,none": 0.016025639930548306, + "sample_len": 541 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "ifeval": "e01bec2e716639c6aa89facb027f54c1a8238f533e9488cbce6f5a79ae309c3d" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "1874.040236396715", + "transformers_version": "4.57.6", + "upper_git_hash": null, + "versions": { + "ifeval": 4.0 + } +} diff --git a/benchmarks/ifeval_kosa.json b/benchmarks/ifeval_kosa.json new file mode 100644 index 0000000..15e5bb6 --- /dev/null +++ b/benchmarks/ifeval_kosa.json @@ -0,0 +1,152 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "32", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "vllm", + "model_args": "pretrained=kosa-labs/kosa-4B-it-v1", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "ifeval": { + "dataset_path": "google/IFEval", + "description": "", + "doc_to_target": 0, + "doc_to_text": "prompt", + "fewshot_config": { + "doc_to_choice": null, + "doc_to_target": 0, + "doc_to_text": "prompt", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "default", + "samples": null, + "split": null, + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 4.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "prompt_level_strict_acc" + }, + { + "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", + "higher_is_better": true, + "metric": "inst_level_strict_acc" + }, + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "prompt_level_loose_acc" + }, + { + "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", + "higher_is_better": true, + "metric": "inst_level_loose_acc" + } + ], + "num_fewshot": 0, + "output_type": "generate_until", + "process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "ifeval", + "test_split": "train", + "unsafe_code": false + } + }, + "date": 1781168624.8804162, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": {}, + "higher_is_better": { + "ifeval": { + "inst_level_loose_acc": true, + "inst_level_strict_acc": true, + "prompt_level_loose_acc": true, + "prompt_level_strict_acc": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "kosa-labs/kosa-4B-it-v1", + "model_name_sanitized": "kosa-labs/kosa-4B-it-v1", + "model_source": "kosa-labs/kosa-4B-it-v1", + "n-samples": { + "ifeval": { + "effective": 541, + "original": 541 + } + }, + "n-shot": { + "ifeval": 0 + }, + "results": { + "ifeval": { + "alias": "ifeval", + "inst_level_loose_acc,none": 0.9232613908872902, + "inst_level_loose_acc_stderr,none": "N/A", + "inst_level_strict_acc,none": 0.9028776978417267, + "inst_level_strict_acc_stderr,none": "N/A", + "name": "ifeval", + "prompt_level_loose_acc,none": 0.8853974121996303, + "prompt_level_loose_acc_stderr,none": 0.013707853566060982, + "prompt_level_strict_acc,none": 0.8576709796672828, + "prompt_level_strict_acc_stderr,none": 0.015035234562202742, + "sample_len": 541 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "ifeval": "e01bec2e716639c6aa89facb027f54c1a8238f533e9488cbce6f5a79ae309c3d" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "334.5368347261101", + "transformers_version": "5.11.0", + "upper_git_hash": null, + "versions": { + "ifeval": 4.0 + } +} diff --git a/benchmarks/mmlu_base.json b/benchmarks/mmlu_base.json new file mode 100644 index 0000000..d8db049 --- /dev/null +++ b/benchmarks/mmlu_base.json @@ -0,0 +1,4530 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "32", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "vllm", + "model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "mmlu_abstract_algebra": { + "dataset_name": "abstract_algebra", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_abstract_algebra", + "task_alias": "abstract_algebra", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_anatomy": { + "dataset_name": "anatomy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_anatomy", + "task_alias": "anatomy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_astronomy": { + "dataset_name": "astronomy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_astronomy", + "task_alias": "astronomy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_business_ethics": { + "dataset_name": "business_ethics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_business_ethics", + "task_alias": "business_ethics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_clinical_knowledge": { + "dataset_name": "clinical_knowledge", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_clinical_knowledge", + "task_alias": "clinical_knowledge", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_biology": { + "dataset_name": "college_biology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college biology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_biology", + "task_alias": "college_biology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_chemistry": { + "dataset_name": "college_chemistry", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_chemistry", + "task_alias": "college_chemistry", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_computer_science": { + "dataset_name": "college_computer_science", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_computer_science", + "task_alias": "college_computer_science", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_mathematics": { + "dataset_name": "college_mathematics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_mathematics", + "task_alias": "college_mathematics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_medicine": { + "dataset_name": "college_medicine", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_medicine", + "task_alias": "college_medicine", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_physics": { + "dataset_name": "college_physics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college physics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_physics", + "task_alias": "college_physics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_computer_security": { + "dataset_name": "computer_security", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about computer security.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_computer_security", + "task_alias": "computer_security", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_conceptual_physics": { + "dataset_name": "conceptual_physics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_conceptual_physics", + "task_alias": "conceptual_physics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_econometrics": { + "dataset_name": "econometrics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_econometrics", + "task_alias": "econometrics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_electrical_engineering": { + "dataset_name": "electrical_engineering", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_electrical_engineering", + "task_alias": "electrical_engineering", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_elementary_mathematics": { + "dataset_name": "elementary_mathematics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_elementary_mathematics", + "task_alias": "elementary_mathematics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_formal_logic": { + "dataset_name": "formal_logic", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_formal_logic", + "task_alias": "formal_logic", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_global_facts": { + "dataset_name": "global_facts", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about global facts.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_global_facts", + "task_alias": "global_facts", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_biology": { + "dataset_name": "high_school_biology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_biology", + "task_alias": "high_school_biology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_chemistry": { + "dataset_name": "high_school_chemistry", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_chemistry", + "task_alias": "high_school_chemistry", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_computer_science": { + "dataset_name": "high_school_computer_science", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_computer_science", + "task_alias": "high_school_computer_science", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_european_history": { + "dataset_name": "high_school_european_history", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_european_history", + "task_alias": "high_school_european_history", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_geography": { + "dataset_name": "high_school_geography", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_geography", + "task_alias": "high_school_geography", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_government_and_politics": { + "dataset_name": "high_school_government_and_politics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_government_and_politics", + "task_alias": "high_school_government_and_politics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_macroeconomics": { + "dataset_name": "high_school_macroeconomics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_macroeconomics", + "task_alias": "high_school_macroeconomics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_mathematics": { + "dataset_name": "high_school_mathematics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_mathematics", + "task_alias": "high_school_mathematics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_microeconomics": { + "dataset_name": "high_school_microeconomics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_microeconomics", + "task_alias": "high_school_microeconomics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_physics": { + "dataset_name": "high_school_physics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_physics", + "task_alias": "high_school_physics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_psychology": { + "dataset_name": "high_school_psychology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_psychology", + "task_alias": "high_school_psychology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_statistics": { + "dataset_name": "high_school_statistics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_statistics", + "task_alias": "high_school_statistics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_us_history": { + "dataset_name": "high_school_us_history", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_us_history", + "task_alias": "high_school_us_history", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_world_history": { + "dataset_name": "high_school_world_history", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_world_history", + "task_alias": "high_school_world_history", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_human_aging": { + "dataset_name": "human_aging", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about human aging.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_human_aging", + "task_alias": "human_aging", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_human_sexuality": { + "dataset_name": "human_sexuality", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_human_sexuality", + "task_alias": "human_sexuality", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_international_law": { + "dataset_name": "international_law", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about international law.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_international_law", + "task_alias": "international_law", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_jurisprudence": { + "dataset_name": "jurisprudence", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_jurisprudence", + "task_alias": "jurisprudence", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_logical_fallacies": { + "dataset_name": "logical_fallacies", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_logical_fallacies", + "task_alias": "logical_fallacies", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_machine_learning": { + "dataset_name": "machine_learning", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_machine_learning", + "task_alias": "machine_learning", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_management": { + "dataset_name": "management", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about management.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_management", + "task_alias": "management", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_marketing": { + "dataset_name": "marketing", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about marketing.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_marketing", + "task_alias": "marketing", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_medical_genetics": { + "dataset_name": "medical_genetics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_medical_genetics", + "task_alias": "medical_genetics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_miscellaneous": { + "dataset_name": "miscellaneous", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_miscellaneous", + "task_alias": "miscellaneous", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_moral_disputes": { + "dataset_name": "moral_disputes", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_moral_disputes", + "task_alias": "moral_disputes", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_moral_scenarios": { + "dataset_name": "moral_scenarios", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_moral_scenarios", + "task_alias": "moral_scenarios", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_nutrition": { + "dataset_name": "nutrition", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_nutrition", + "task_alias": "nutrition", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_philosophy": { + "dataset_name": "philosophy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_philosophy", + "task_alias": "philosophy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_prehistory": { + "dataset_name": "prehistory", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_prehistory", + "task_alias": "prehistory", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_accounting": { + "dataset_name": "professional_accounting", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_accounting", + "task_alias": "professional_accounting", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_law": { + "dataset_name": "professional_law", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional law.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_law", + "task_alias": "professional_law", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_medicine": { + "dataset_name": "professional_medicine", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_medicine", + "task_alias": "professional_medicine", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_psychology": { + "dataset_name": "professional_psychology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_psychology", + "task_alias": "professional_psychology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_public_relations": { + "dataset_name": "public_relations", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about public relations.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_public_relations", + "task_alias": "public_relations", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_security_studies": { + "dataset_name": "security_studies", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about security studies.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_security_studies", + "task_alias": "security_studies", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_sociology": { + "dataset_name": "sociology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about sociology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_sociology", + "task_alias": "sociology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_us_foreign_policy": { + "dataset_name": "us_foreign_policy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_us_foreign_policy", + "task_alias": "us_foreign_policy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_virology": { + "dataset_name": "virology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about virology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_virology", + "task_alias": "virology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_world_religions": { + "dataset_name": "world_religions", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about world religions.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "Qwen/Qwen3-4B-Instruct-2507", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_world_religions", + "task_alias": "world_religions", + "test_split": "test", + "unsafe_code": false + } + }, + "date": 1781175138.5280895, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": { + "mmlu": [ + "mmlu_stem", + "mmlu_other", + "mmlu_social_sciences", + "mmlu_humanities" + ], + "mmlu_humanities": [ + "mmlu_formal_logic", + "mmlu_high_school_european_history", + "mmlu_high_school_us_history", + "mmlu_high_school_world_history", + "mmlu_international_law", + "mmlu_jurisprudence", + "mmlu_logical_fallacies", + "mmlu_moral_disputes", + "mmlu_moral_scenarios", + "mmlu_philosophy", + "mmlu_prehistory", + "mmlu_professional_law", + "mmlu_world_religions" + ], + "mmlu_other": [ + "mmlu_business_ethics", + "mmlu_clinical_knowledge", + "mmlu_college_medicine", + "mmlu_global_facts", + "mmlu_human_aging", + "mmlu_management", + "mmlu_marketing", + "mmlu_medical_genetics", + "mmlu_miscellaneous", + "mmlu_nutrition", + "mmlu_professional_accounting", + "mmlu_professional_medicine", + "mmlu_virology" + ], + "mmlu_social_sciences": [ + "mmlu_econometrics", + "mmlu_high_school_geography", + "mmlu_high_school_government_and_politics", + "mmlu_high_school_macroeconomics", + "mmlu_high_school_microeconomics", + "mmlu_high_school_psychology", + "mmlu_human_sexuality", + "mmlu_professional_psychology", + "mmlu_public_relations", + "mmlu_security_studies", + "mmlu_sociology", + "mmlu_us_foreign_policy" + ], + "mmlu_stem": [ + "mmlu_abstract_algebra", + "mmlu_anatomy", + "mmlu_astronomy", + "mmlu_college_biology", + "mmlu_college_chemistry", + "mmlu_college_computer_science", + "mmlu_college_mathematics", + "mmlu_college_physics", + "mmlu_computer_security", + "mmlu_conceptual_physics", + "mmlu_electrical_engineering", + "mmlu_elementary_mathematics", + "mmlu_high_school_biology", + "mmlu_high_school_chemistry", + "mmlu_high_school_computer_science", + "mmlu_high_school_mathematics", + "mmlu_high_school_physics", + "mmlu_high_school_statistics", + "mmlu_machine_learning" + ] + }, + "groups": { + "mmlu": { + "acc,none": 0.6189289275032047, + "acc_stderr,none": 0.003746138843326914, + "alias": "mmlu", + "name": "mmlu", + "sample_count": { + "acc,none": 14042 + }, + "sample_len": 14042 + }, + "mmlu_humanities": { + "acc,none": 0.5596174282678003, + "acc_stderr,none": 0.00661316692569494, + "alias": "humanities", + "name": "mmlu_humanities", + "sample_count": { + "acc,none": 4705 + }, + "sample_len": 4705 + }, + "mmlu_other": { + "acc,none": 0.6903765690376569, + "acc_stderr,none": 0.00793504827765504, + "alias": "other", + "name": "mmlu_other", + "sample_count": { + "acc,none": 3107 + }, + "sample_len": 3107 + }, + "mmlu_social_sciences": { + "acc,none": 0.7565810854728632, + "acc_stderr,none": 0.007580062219729006, + "alias": "social sciences", + "name": "mmlu_social_sciences", + "sample_count": { + "acc,none": 3077 + }, + "sample_len": 3077 + }, + "mmlu_stem": { + "acc,none": 0.5026958452267681, + "acc_stderr,none": 0.008068176972734112, + "alias": "stem", + "name": "mmlu_stem", + "sample_count": { + "acc,none": 3153 + }, + "sample_len": 3153 + } + }, + "higher_is_better": { + "mmlu_abstract_algebra": { + "acc": true + }, + "mmlu_anatomy": { + "acc": true + }, + "mmlu_astronomy": { + "acc": true + }, + "mmlu_business_ethics": { + "acc": true + }, + "mmlu_clinical_knowledge": { + "acc": true + }, + "mmlu_college_biology": { + "acc": true + }, + "mmlu_college_chemistry": { + "acc": true + }, + "mmlu_college_computer_science": { + "acc": true + }, + "mmlu_college_mathematics": { + "acc": true + }, + "mmlu_college_medicine": { + "acc": true + }, + "mmlu_college_physics": { + "acc": true + }, + "mmlu_computer_security": { + "acc": true + }, + "mmlu_conceptual_physics": { + "acc": true + }, + "mmlu_econometrics": { + "acc": true + }, + "mmlu_electrical_engineering": { + "acc": true + }, + "mmlu_elementary_mathematics": { + "acc": true + }, + "mmlu_formal_logic": { + "acc": true + }, + "mmlu_global_facts": { + "acc": true + }, + "mmlu_high_school_biology": { + "acc": true + }, + "mmlu_high_school_chemistry": { + "acc": true + }, + "mmlu_high_school_computer_science": { + "acc": true + }, + "mmlu_high_school_european_history": { + "acc": true + }, + "mmlu_high_school_geography": { + "acc": true + }, + "mmlu_high_school_government_and_politics": { + "acc": true + }, + "mmlu_high_school_macroeconomics": { + "acc": true + }, + "mmlu_high_school_mathematics": { + "acc": true + }, + "mmlu_high_school_microeconomics": { + "acc": true + }, + "mmlu_high_school_physics": { + "acc": true + }, + "mmlu_high_school_psychology": { + "acc": true + }, + "mmlu_high_school_statistics": { + "acc": true + }, + "mmlu_high_school_us_history": { + "acc": true + }, + "mmlu_high_school_world_history": { + "acc": true + }, + "mmlu_human_aging": { + "acc": true + }, + "mmlu_human_sexuality": { + "acc": true + }, + "mmlu_humanities": { + "acc": true + }, + "mmlu_international_law": { + "acc": true + }, + "mmlu_jurisprudence": { + "acc": true + }, + "mmlu_logical_fallacies": { + "acc": true + }, + "mmlu_machine_learning": { + "acc": true + }, + "mmlu_management": { + "acc": true + }, + "mmlu_marketing": { + "acc": true + }, + "mmlu_medical_genetics": { + "acc": true + }, + "mmlu_miscellaneous": { + "acc": true + }, + "mmlu_moral_disputes": { + "acc": true + }, + "mmlu_moral_scenarios": { + "acc": true + }, + "mmlu_nutrition": { + "acc": true + }, + "mmlu_other": { + "acc": true + }, + "mmlu_philosophy": { + "acc": true + }, + "mmlu_prehistory": { + "acc": true + }, + "mmlu_professional_accounting": { + "acc": true + }, + "mmlu_professional_law": { + "acc": true + }, + "mmlu_professional_medicine": { + "acc": true + }, + "mmlu_professional_psychology": { + "acc": true + }, + "mmlu_public_relations": { + "acc": true + }, + "mmlu_security_studies": { + "acc": true + }, + "mmlu_social_sciences": { + "acc": true + }, + "mmlu_sociology": { + "acc": true + }, + "mmlu_stem": { + "acc": true + }, + "mmlu_us_foreign_policy": { + "acc": true + }, + "mmlu_virology": { + "acc": true + }, + "mmlu_world_religions": { + "acc": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "Qwen/Qwen3-4B-Instruct-2507", + "model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507", + "model_source": "Qwen/Qwen3-4B-Instruct-2507", + "n-samples": { + "mmlu_abstract_algebra": { + "effective": 100, + "original": 100 + }, + "mmlu_anatomy": { + "effective": 135, + "original": 135 + }, + "mmlu_astronomy": { + "effective": 152, + "original": 152 + }, + "mmlu_business_ethics": { + "effective": 100, + "original": 100 + }, + "mmlu_clinical_knowledge": { + "effective": 265, + "original": 265 + }, + "mmlu_college_biology": { + "effective": 144, + "original": 144 + }, + "mmlu_college_chemistry": { + "effective": 100, + "original": 100 + }, + "mmlu_college_computer_science": { + "effective": 100, + "original": 100 + }, + "mmlu_college_mathematics": { + "effective": 100, + "original": 100 + }, + "mmlu_college_medicine": { + "effective": 173, + "original": 173 + }, + "mmlu_college_physics": { + "effective": 102, + "original": 102 + }, + "mmlu_computer_security": { + "effective": 100, + "original": 100 + }, + "mmlu_conceptual_physics": { + "effective": 235, + "original": 235 + }, + "mmlu_econometrics": { + "effective": 114, + "original": 114 + }, + "mmlu_electrical_engineering": { + "effective": 145, + "original": 145 + }, + "mmlu_elementary_mathematics": { + "effective": 378, + "original": 378 + }, + "mmlu_formal_logic": { + "effective": 126, + "original": 126 + }, + "mmlu_global_facts": { + "effective": 100, + "original": 100 + }, + "mmlu_high_school_biology": { + "effective": 310, + "original": 310 + }, + "mmlu_high_school_chemistry": { + "effective": 203, + "original": 203 + }, + "mmlu_high_school_computer_science": { + "effective": 100, + "original": 100 + }, + "mmlu_high_school_european_history": { + "effective": 165, + "original": 165 + }, + "mmlu_high_school_geography": { + "effective": 198, + "original": 198 + }, + "mmlu_high_school_government_and_politics": { + "effective": 193, + "original": 193 + }, + "mmlu_high_school_macroeconomics": { + "effective": 390, + "original": 390 + }, + "mmlu_high_school_mathematics": { + "effective": 270, + "original": 270 + }, + "mmlu_high_school_microeconomics": { + "effective": 238, + "original": 238 + }, + "mmlu_high_school_physics": { + "effective": 151, + "original": 151 + }, + "mmlu_high_school_psychology": { + "effective": 545, + "original": 545 + }, + "mmlu_high_school_statistics": { + "effective": 216, + "original": 216 + }, + "mmlu_high_school_us_history": { + "effective": 204, + "original": 204 + }, + "mmlu_high_school_world_history": { + "effective": 237, + "original": 237 + }, + "mmlu_human_aging": { + "effective": 223, + "original": 223 + }, + "mmlu_human_sexuality": { + "effective": 131, + "original": 131 + }, + "mmlu_international_law": { + "effective": 121, + "original": 121 + }, + "mmlu_jurisprudence": { + "effective": 108, + "original": 108 + }, + "mmlu_logical_fallacies": { + "effective": 163, + "original": 163 + }, + "mmlu_machine_learning": { + "effective": 112, + "original": 112 + }, + "mmlu_management": { + "effective": 103, + "original": 103 + }, + "mmlu_marketing": { + "effective": 234, + "original": 234 + }, + "mmlu_medical_genetics": { + "effective": 100, + "original": 100 + }, + "mmlu_miscellaneous": { + "effective": 783, + "original": 783 + }, + "mmlu_moral_disputes": { + "effective": 346, + "original": 346 + }, + "mmlu_moral_scenarios": { + "effective": 895, + "original": 895 + }, + "mmlu_nutrition": { + "effective": 306, + "original": 306 + }, + "mmlu_philosophy": { + "effective": 311, + "original": 311 + }, + "mmlu_prehistory": { + "effective": 324, + "original": 324 + }, + "mmlu_professional_accounting": { + "effective": 282, + "original": 282 + }, + "mmlu_professional_law": { + "effective": 1534, + "original": 1534 + }, + "mmlu_professional_medicine": { + "effective": 272, + "original": 272 + }, + "mmlu_professional_psychology": { + "effective": 612, + "original": 612 + }, + "mmlu_public_relations": { + "effective": 110, + "original": 110 + }, + "mmlu_security_studies": { + "effective": 245, + "original": 245 + }, + "mmlu_sociology": { + "effective": 201, + "original": 201 + }, + "mmlu_us_foreign_policy": { + "effective": 100, + "original": 100 + }, + "mmlu_virology": { + "effective": 166, + "original": 166 + }, + "mmlu_world_religions": { + "effective": 171, + "original": 171 + } + }, + "n-shot": { + "mmlu_abstract_algebra": 0, + "mmlu_anatomy": 0, + "mmlu_astronomy": 0, + "mmlu_business_ethics": 0, + "mmlu_clinical_knowledge": 0, + "mmlu_college_biology": 0, + "mmlu_college_chemistry": 0, + "mmlu_college_computer_science": 0, + "mmlu_college_mathematics": 0, + "mmlu_college_medicine": 0, + "mmlu_college_physics": 0, + "mmlu_computer_security": 0, + "mmlu_conceptual_physics": 0, + "mmlu_econometrics": 0, + "mmlu_electrical_engineering": 0, + "mmlu_elementary_mathematics": 0, + "mmlu_formal_logic": 0, + "mmlu_global_facts": 0, + "mmlu_high_school_biology": 0, + "mmlu_high_school_chemistry": 0, + "mmlu_high_school_computer_science": 0, + "mmlu_high_school_european_history": 0, + "mmlu_high_school_geography": 0, + "mmlu_high_school_government_and_politics": 0, + "mmlu_high_school_macroeconomics": 0, + "mmlu_high_school_mathematics": 0, + "mmlu_high_school_microeconomics": 0, + "mmlu_high_school_physics": 0, + "mmlu_high_school_psychology": 0, + "mmlu_high_school_statistics": 0, + "mmlu_high_school_us_history": 0, + "mmlu_high_school_world_history": 0, + "mmlu_human_aging": 0, + "mmlu_human_sexuality": 0, + "mmlu_humanities": 0, + "mmlu_international_law": 0, + "mmlu_jurisprudence": 0, + "mmlu_logical_fallacies": 0, + "mmlu_machine_learning": 0, + "mmlu_management": 0, + "mmlu_marketing": 0, + "mmlu_medical_genetics": 0, + "mmlu_miscellaneous": 0, + "mmlu_moral_disputes": 0, + "mmlu_moral_scenarios": 0, + "mmlu_nutrition": 0, + "mmlu_other": 0, + "mmlu_philosophy": 0, + "mmlu_prehistory": 0, + "mmlu_professional_accounting": 0, + "mmlu_professional_law": 0, + "mmlu_professional_medicine": 0, + "mmlu_professional_psychology": 0, + "mmlu_public_relations": 0, + "mmlu_security_studies": 0, + "mmlu_social_sciences": 0, + "mmlu_sociology": 0, + "mmlu_stem": 0, + "mmlu_us_foreign_policy": 0, + "mmlu_virology": 0, + "mmlu_world_religions": 0 + }, + "results": { + "mmlu": { + "acc,none": 0.6189289275032047, + "acc_stderr,none": 0.003746138843326914, + "alias": "mmlu", + "name": "mmlu", + "sample_count": { + "acc,none": 14042 + }, + "sample_len": 14042 + }, + "mmlu_abstract_algebra": { + "acc,none": 0.23, + "acc_stderr,none": 0.04229525846816507, + "alias": "abstract_algebra", + "name": "mmlu_abstract_algebra", + "sample_len": 100 + }, + "mmlu_anatomy": { + "acc,none": 0.6444444444444445, + "acc_stderr,none": 0.04135176749720385, + "alias": "anatomy", + "name": "mmlu_anatomy", + "sample_len": 135 + }, + "mmlu_astronomy": { + "acc,none": 0.7828947368421053, + "acc_stderr,none": 0.03355045304882924, + "alias": "astronomy", + "name": "mmlu_astronomy", + "sample_len": 152 + }, + "mmlu_business_ethics": { + "acc,none": 0.7, + "acc_stderr,none": 0.04605661864718383, + "alias": "business_ethics", + "name": "mmlu_business_ethics", + "sample_len": 100 + }, + "mmlu_clinical_knowledge": { + "acc,none": 0.7584905660377359, + "acc_stderr,none": 0.02634148037111831, + "alias": "clinical_knowledge", + "name": "mmlu_clinical_knowledge", + "sample_len": 265 + }, + "mmlu_college_biology": { + "acc,none": 0.8194444444444444, + "acc_stderr,none": 0.03216600808802272, + "alias": "college_biology", + "name": "mmlu_college_biology", + "sample_len": 144 + }, + "mmlu_college_chemistry": { + "acc,none": 0.42, + "acc_stderr,none": 0.04960449637488583, + "alias": "college_chemistry", + "name": "mmlu_college_chemistry", + "sample_len": 100 + }, + "mmlu_college_computer_science": { + "acc,none": 0.5, + "acc_stderr,none": 0.050251890762960605, + "alias": "college_computer_science", + "name": "mmlu_college_computer_science", + "sample_len": 100 + }, + "mmlu_college_mathematics": { + "acc,none": 0.22, + "acc_stderr,none": 0.041633319989322654, + "alias": "college_mathematics", + "name": "mmlu_college_mathematics", + "sample_len": 100 + }, + "mmlu_college_medicine": { + "acc,none": 0.6358381502890174, + "acc_stderr,none": 0.03669072477416912, + "alias": "college_medicine", + "name": "mmlu_college_medicine", + "sample_len": 173 + }, + "mmlu_college_physics": { + "acc,none": 0.39215686274509803, + "acc_stderr,none": 0.04858083574266346, + "alias": "college_physics", + "name": "mmlu_college_physics", + "sample_len": 102 + }, + "mmlu_computer_security": { + "acc,none": 0.7, + "acc_stderr,none": 0.04605661864718383, + "alias": "computer_security", + "name": "mmlu_computer_security", + "sample_len": 100 + }, + "mmlu_conceptual_physics": { + "acc,none": 0.6808510638297872, + "acc_stderr,none": 0.03047297336338012, + "alias": "conceptual_physics", + "name": "mmlu_conceptual_physics", + "sample_len": 235 + }, + "mmlu_econometrics": { + "acc,none": 0.5175438596491229, + "acc_stderr,none": 0.04700708033551044, + "alias": "econometrics", + "name": "mmlu_econometrics", + "sample_len": 114 + }, + "mmlu_electrical_engineering": { + "acc,none": 0.593103448275862, + "acc_stderr,none": 0.040937939812662354, + "alias": "electrical_engineering", + "name": "mmlu_electrical_engineering", + "sample_len": 145 + }, + "mmlu_elementary_mathematics": { + "acc,none": 0.24338624338624337, + "acc_stderr,none": 0.022101128787415353, + "alias": "elementary_mathematics", + "name": "mmlu_elementary_mathematics", + "sample_len": 378 + }, + "mmlu_formal_logic": { + "acc,none": 0.3333333333333333, + "acc_stderr,none": 0.04216370213557839, + "alias": "formal_logic", + "name": "mmlu_formal_logic", + "sample_len": 126 + }, + "mmlu_global_facts": { + "acc,none": 0.38, + "acc_stderr,none": 0.04878317312145634, + "alias": "global_facts", + "name": "mmlu_global_facts", + "sample_len": 100 + }, + "mmlu_high_school_biology": { + "acc,none": 0.8161290322580645, + "acc_stderr,none": 0.022037217340267787, + "alias": "high_school_biology", + "name": "mmlu_high_school_biology", + "sample_len": 310 + }, + "mmlu_high_school_chemistry": { + "acc,none": 0.458128078817734, + "acc_stderr,none": 0.035056301407857406, + "alias": "high_school_chemistry", + "name": "mmlu_high_school_chemistry", + "sample_len": 203 + }, + "mmlu_high_school_computer_science": { + "acc,none": 0.69, + "acc_stderr,none": 0.046482319871173176, + "alias": "high_school_computer_science", + "name": "mmlu_high_school_computer_science", + "sample_len": 100 + }, + "mmlu_high_school_european_history": { + "acc,none": 0.8, + "acc_stderr,none": 0.031234752377721213, + "alias": "high_school_european_history", + "name": "mmlu_high_school_european_history", + "sample_len": 165 + }, + "mmlu_high_school_geography": { + "acc,none": 0.7929292929292929, + "acc_stderr,none": 0.02886977846026699, + "alias": "high_school_geography", + "name": "mmlu_high_school_geography", + "sample_len": 198 + }, + "mmlu_high_school_government_and_politics": { + "acc,none": 0.8756476683937824, + "acc_stderr,none": 0.02381447708659357, + "alias": "high_school_government_and_politics", + "name": "mmlu_high_school_government_and_politics", + "sample_len": 193 + }, + "mmlu_high_school_macroeconomics": { + "acc,none": 0.6615384615384615, + "acc_stderr,none": 0.023991500500313064, + "alias": "high_school_macroeconomics", + "name": "mmlu_high_school_macroeconomics", + "sample_len": 390 + }, + "mmlu_high_school_mathematics": { + "acc,none": 0.2111111111111111, + "acc_stderr,none": 0.02488211685765511, + "alias": "high_school_mathematics", + "name": "mmlu_high_school_mathematics", + "sample_len": 270 + }, + "mmlu_high_school_microeconomics": { + "acc,none": 0.7857142857142857, + "acc_stderr,none": 0.026653531596715505, + "alias": "high_school_microeconomics", + "name": "mmlu_high_school_microeconomics", + "sample_len": 238 + }, + "mmlu_high_school_physics": { + "acc,none": 0.3841059602649007, + "acc_stderr,none": 0.03971301814719192, + "alias": "high_school_physics", + "name": "mmlu_high_school_physics", + "sample_len": 151 + }, + "mmlu_high_school_psychology": { + "acc,none": 0.8660550458715597, + "acc_stderr,none": 0.014602811435592633, + "alias": "high_school_psychology", + "name": "mmlu_high_school_psychology", + "sample_len": 545 + }, + "mmlu_high_school_statistics": { + "acc,none": 0.4537037037037037, + "acc_stderr,none": 0.033953227263757976, + "alias": "high_school_statistics", + "name": "mmlu_high_school_statistics", + "sample_len": 216 + }, + "mmlu_high_school_us_history": { + "acc,none": 0.8284313725490197, + "acc_stderr,none": 0.026460569561240675, + "alias": "high_school_us_history", + "name": "mmlu_high_school_us_history", + "sample_len": 204 + }, + "mmlu_high_school_world_history": { + "acc,none": 0.8270042194092827, + "acc_stderr,none": 0.024621562866768382, + "alias": "high_school_world_history", + "name": "mmlu_high_school_world_history", + "sample_len": 237 + }, + "mmlu_human_aging": { + "acc,none": 0.6367713004484304, + "acc_stderr,none": 0.03227790442850494, + "alias": "human_aging", + "name": "mmlu_human_aging", + "sample_len": 223 + }, + "mmlu_human_sexuality": { + "acc,none": 0.732824427480916, + "acc_stderr,none": 0.03880848301082397, + "alias": "human_sexuality", + "name": "mmlu_human_sexuality", + "sample_len": 131 + }, + "mmlu_humanities": { + "acc,none": 0.5596174282678003, + "acc_stderr,none": 0.00661316692569494, + "alias": "humanities", + "name": "mmlu_humanities", + "sample_count": { + "acc,none": 4705 + }, + "sample_len": 4705 + }, + "mmlu_international_law": { + "acc,none": 0.8347107438016529, + "acc_stderr,none": 0.03390780612972774, + "alias": "international_law", + "name": "mmlu_international_law", + "sample_len": 121 + }, + "mmlu_jurisprudence": { + "acc,none": 0.7407407407407407, + "acc_stderr,none": 0.04236511258094632, + "alias": "jurisprudence", + "name": "mmlu_jurisprudence", + "sample_len": 108 + }, + "mmlu_logical_fallacies": { + "acc,none": 0.7116564417177914, + "acc_stderr,none": 0.03559039531617345, + "alias": "logical_fallacies", + "name": "mmlu_logical_fallacies", + "sample_len": 163 + }, + "mmlu_machine_learning": { + "acc,none": 0.42857142857142855, + "acc_stderr,none": 0.04697113923010208, + "alias": "machine_learning", + "name": "mmlu_machine_learning", + "sample_len": 112 + }, + "mmlu_management": { + "acc,none": 0.8543689320388349, + "acc_stderr,none": 0.034926064766237934, + "alias": "management", + "name": "mmlu_management", + "sample_len": 103 + }, + "mmlu_marketing": { + "acc,none": 0.8461538461538461, + "acc_stderr,none": 0.023636873317489267, + "alias": "marketing", + "name": "mmlu_marketing", + "sample_len": 234 + }, + "mmlu_medical_genetics": { + "acc,none": 0.73, + "acc_stderr,none": 0.04461960433384737, + "alias": "medical_genetics", + "name": "mmlu_medical_genetics", + "sample_len": 100 + }, + "mmlu_miscellaneous": { + "acc,none": 0.8033205619412516, + "acc_stderr,none": 0.014214138556913851, + "alias": "miscellaneous", + "name": "mmlu_miscellaneous", + "sample_len": 783 + }, + "mmlu_moral_disputes": { + "acc,none": 0.7138728323699421, + "acc_stderr,none": 0.02433214677913417, + "alias": "moral_disputes", + "name": "mmlu_moral_disputes", + "sample_len": 346 + }, + "mmlu_moral_scenarios": { + "acc,none": 0.26927374301675977, + "acc_stderr,none": 0.014835616582882544, + "alias": "moral_scenarios", + "name": "mmlu_moral_scenarios", + "sample_len": 895 + }, + "mmlu_nutrition": { + "acc,none": 0.7647058823529411, + "acc_stderr,none": 0.02428861946604616, + "alias": "nutrition", + "name": "mmlu_nutrition", + "sample_len": 306 + }, + "mmlu_other": { + "acc,none": 0.6903765690376569, + "acc_stderr,none": 0.00793504827765504, + "alias": "other", + "name": "mmlu_other", + "sample_count": { + "acc,none": 3107 + }, + "sample_len": 3107 + }, + "mmlu_philosophy": { + "acc,none": 0.7041800643086816, + "acc_stderr,none": 0.025922371788818833, + "alias": "philosophy", + "name": "mmlu_philosophy", + "sample_len": 311 + }, + "mmlu_prehistory": { + "acc,none": 0.7654320987654321, + "acc_stderr,none": 0.0235768817440057, + "alias": "prehistory", + "name": "mmlu_prehistory", + "sample_len": 324 + }, + "mmlu_professional_accounting": { + "acc,none": 0.46808510638297873, + "acc_stderr,none": 0.02976667507587383, + "alias": "professional_accounting", + "name": "mmlu_professional_accounting", + "sample_len": 282 + }, + "mmlu_professional_law": { + "acc,none": 0.4589308996088657, + "acc_stderr,none": 0.0127270848267999, + "alias": "professional_law", + "name": "mmlu_professional_law", + "sample_len": 1534 + }, + "mmlu_professional_medicine": { + "acc,none": 0.5551470588235294, + "acc_stderr,none": 0.030187532060329314, + "alias": "professional_medicine", + "name": "mmlu_professional_medicine", + "sample_len": 272 + }, + "mmlu_professional_psychology": { + "acc,none": 0.6993464052287581, + "acc_stderr,none": 0.01855063450295291, + "alias": "professional_psychology", + "name": "mmlu_professional_psychology", + "sample_len": 612 + }, + "mmlu_public_relations": { + "acc,none": 0.6454545454545455, + "acc_stderr,none": 0.04582004841505413, + "alias": "public_relations", + "name": "mmlu_public_relations", + "sample_len": 110 + }, + "mmlu_security_studies": { + "acc,none": 0.7428571428571429, + "acc_stderr,none": 0.027979823538744557, + "alias": "security_studies", + "name": "mmlu_security_studies", + "sample_len": 245 + }, + "mmlu_social_sciences": { + "acc,none": 0.7565810854728632, + "acc_stderr,none": 0.007580062219729006, + "alias": "social sciences", + "name": "mmlu_social_sciences", + "sample_count": { + "acc,none": 3077 + }, + "sample_len": 3077 + }, + "mmlu_sociology": { + "acc,none": 0.835820895522388, + "acc_stderr,none": 0.026193923544454094, + "alias": "sociology", + "name": "mmlu_sociology", + "sample_len": 201 + }, + "mmlu_stem": { + "acc,none": 0.5026958452267681, + "acc_stderr,none": 0.008068176972734112, + "alias": "stem", + "name": "mmlu_stem", + "sample_count": { + "acc,none": 3153 + }, + "sample_len": 3153 + }, + "mmlu_us_foreign_policy": { + "acc,none": 0.81, + "acc_stderr,none": 0.039427724440366255, + "alias": "us_foreign_policy", + "name": "mmlu_us_foreign_policy", + "sample_len": 100 + }, + "mmlu_virology": { + "acc,none": 0.4759036144578313, + "acc_stderr,none": 0.03887971849597268, + "alias": "virology", + "name": "mmlu_virology", + "sample_len": 166 + }, + "mmlu_world_religions": { + "acc,none": 0.8070175438596491, + "acc_stderr,none": 0.030267457554898448, + "alias": "world_religions", + "name": "mmlu_world_religions", + "sample_len": 171 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "mmlu_abstract_algebra": "1ecc29cafeff483bf80347adc2878dae00d01596988032f438afce6291c8cfe7", + "mmlu_anatomy": "51232e78f32967e0dd431bab9309f69be490f9998b4ddb93b641b09ba1f613cf", + "mmlu_astronomy": "3e8d7c33bd0431518a3d5d4b1666bdabccf46c4cc4cfe95a3886cb2266743f86", + "mmlu_business_ethics": "6ec93578c18c7bba58a384ecaa7e6e84c5e681262b34b1caa76abbbb0f9ebb02", + "mmlu_clinical_knowledge": "cf39fec90aa76a379e440c1f0d5f5b04d9f32edc4004ffcfb8eadcc25c18b118", + "mmlu_college_biology": "5378a297efa1cd71344228c95dcb29ba54757c7e527261400a4d65fd8ceed92b", + "mmlu_college_chemistry": "714e60ea7e340f0344ffde41b247a19b68e6384152a1bdc9acc0ecf24cd0e7cd", + "mmlu_college_computer_science": "2b66d244f98e61e6a78e5ed48b728e0ca2ca81c7757ccdd086fe6b0c1cf02da1", + "mmlu_college_mathematics": "90b1ce25d7d38abd925181ae3aad095cfd783375fe8de0607a57420f47454f0f", + "mmlu_college_medicine": "00cc41d26670913bdee6181e5b602726dde9f03e6a43236c2bf57486149845be", + "mmlu_college_physics": "2abddb12a247904f9ba60b39766b19a8967c24fe8eee021022a353b93d73742c", + "mmlu_computer_security": "8ffba21cebb9e39d5794c848a22e7e37cddff5c59f7b47a1d89872d9e8704b0c", + "mmlu_conceptual_physics": "c0da3219fe7e5292dad2aa7e72790e3eba951711007cc57618b0850d941b85ec", + "mmlu_econometrics": "bc667156b739500e02f7c8cb8f9621b6d62ea2011e1eaa3404cb45f6d16778ec", + "mmlu_electrical_engineering": "0a02cb16a31ec7debba4190ab1cc31afadb9416764e0b074b9b96553f346b892", + "mmlu_elementary_mathematics": "6048f1506468e757010d537a0ceedaadc587c18159a03e7664b5994843afedc2", + "mmlu_formal_logic": "5fef452fe37d3c5aefce6276a26a839e5ee5d655d72adf50bbf0a8ae0e8d74e4", + "mmlu_global_facts": "6163d6678b902c8cb5cd6c50f59bc0f31a9a356e977fcd6184614f48e3320e7a", + "mmlu_high_school_biology": "2aabe83bb100a352d24a0136c24e612b68490bbe818c33f06cc6cee5379ecad8", + "mmlu_high_school_chemistry": "dd3321fd42c8717094f57dd0eaeab0002d9ed192518a7de7aaef4913796c30f0", + "mmlu_high_school_computer_science": "888aabb9a7ac77299e8b2b8429fa44c1fbbb5ac8165f556a0b00b347e51de3b5", + "mmlu_high_school_european_history": "0b7be1963c7e224bffc18d72fe277408c4980f854a5fc94805fc008133640f44", + "mmlu_high_school_geography": "915db5f9f9c584c6886ae92cfc28ad9c364bf685f22c15009ad8bf064b4a8027", + "mmlu_high_school_government_and_politics": "411c6c6c179d1e2a252fab88f33123721540fc70f8223310e1f1349ec0b67753", + "mmlu_high_school_macroeconomics": "291b29471838255d1a2dd704960610e98b7e3f2fd001e3aabc6b25be0c236e29", + "mmlu_high_school_mathematics": "9b9567fc5c7875065ac91eac69621904f201cab822621440a2442a390f62b497", + "mmlu_high_school_microeconomics": "6f4a887cb73beee5414248002f75216c16d7c8ff739b37a2607c6bcfad16712b", + "mmlu_high_school_physics": "53edb93aefb6fc54fba283278bae10a58a7902902ff55edc30cb9c81745dbaa8", + "mmlu_high_school_psychology": "ce01f01ff9a3bdc3e956ada7ec7c02f13771c386d076323056ac00eec0095785", + "mmlu_high_school_statistics": "e0d652bf0cb6f927b0fc738e58a141f0ba8f9073615f4ddd70bf7a3b97998f21", + "mmlu_high_school_us_history": "1b31fdc7e0d51803fa6107a0c7aa3b6968031bc449ed3513a7349956fa0b53d3", + "mmlu_high_school_world_history": "9e352460adb245947aaf9671f135ae6484e44a115312955c45c7caaa0688f160", + "mmlu_human_aging": "4b4276095e96dc6e2872517b2becc2ebe9fad0f42b321486f6f2ceaef81f385e", + "mmlu_human_sexuality": "9395014a475c63632383587439df5c48c5dd3333eaa8d10ef9e0eaabd0997c3a", + "mmlu_international_law": "3d2036ffe61014fa8a2de679c6bee2c0c3ade112f03cd55e4a6fba5bdc56bf7b", + "mmlu_jurisprudence": "1548c06ef17823048b473d028f4889b3cf014beac4ac808521898ee141a34d23", + "mmlu_logical_fallacies": "66f9691122f3fa813b86fda1131037c03cf42babe650c7374924f41882716332", + "mmlu_machine_learning": "13ad1d3650a26d655a2a2eaa3be1f106d990e5b693b0e9006f8ebef221d2ed82", + "mmlu_management": "84a18a4a3039f23b7dd18838c89a80171025a1a45eda3305464a3cfcd8257b69", + "mmlu_marketing": "d6382d405e202221a78087356861723acb445fbd6f1db94152201f788b5c0606", + "mmlu_medical_genetics": "b4546cc241c5811dddfec47d0bba37122fbd65545c93d60f50b993e810d37e2d", + "mmlu_miscellaneous": "ecf4c3e0dc895ca5b7803d1b3f01fe3027b461f900ac001d36f5dd750c8eca1d", + "mmlu_moral_disputes": "84f580166c00063790619588b4f9fcd2f800a6a04778f08c1bada043bfdc6ec0", + "mmlu_moral_scenarios": "0dda4f43fd25e66311e64f4590698bfcc4c00c24ac58dbfec74b613512280ffc", + "mmlu_nutrition": "732959f39ac0c6c0e9c3823082d3669467531cebfb5f7550d79bcc31e5cfdbce", + "mmlu_philosophy": "ff6284cd5c6bfdb17898ef97dabc97916b70fc7485c57543edabdd44b1d8b543", + "mmlu_prehistory": "efb58ea420f5b7975ff34766ddc7470676e1ababa0b4705bfdf150f6871fdd83", + "mmlu_professional_accounting": "3c555bc35f86d097f509755f487781d2bd288de5dfe9aa59d8009a1dd0583c45", + "mmlu_professional_law": "b108c04f61cbbb4b9351de3730d7e3396710c7702c6cf75703155dbfdbc7819d", + "mmlu_professional_medicine": "0821fa48ca3715604bbef122215aea4e55f218dad6ba5191da2eb831971ac6f9", + "mmlu_professional_psychology": "ab273adb58e2b365b3811e4ed4c6485c9630f78058580273703eb2573dde9010", + "mmlu_public_relations": "da7dfa166f545d33a61c930908eee4f8d8544cad6328569b1c4b341becaf68f1", + "mmlu_security_studies": "85ea8f6c3b791998f3c1d57164b6e28bcf4498f65fc1988dfa4a6a65d31fb505", + "mmlu_sociology": "055d024e9587edc811c54e6cdd8fd74d535c6f5b1aad17cfd5f1598a1371606a", + "mmlu_us_foreign_policy": "a99e6492b6831a5df8c7f4c76edf15b500e59ab4c87dc73fdbd61d43bc704682", + "mmlu_virology": "ba4fc492ed4e9af97e30850257d2fdc5e7b53b37e7b4d68d443f7bd157aac21f", + "mmlu_world_religions": "9634e441f5f53329b4b6c760b08b1a130f8f03b3cc1902b23d07f57db76b7263" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "617.2125840899535", + "transformers_version": "5.11.0", + "upper_git_hash": null, + "versions": { + "mmlu": "2", + "mmlu_abstract_algebra": 1.0, + "mmlu_anatomy": 1.0, + "mmlu_astronomy": 1.0, + "mmlu_business_ethics": 1.0, + "mmlu_clinical_knowledge": 1.0, + "mmlu_college_biology": 1.0, + "mmlu_college_chemistry": 1.0, + "mmlu_college_computer_science": 1.0, + "mmlu_college_mathematics": 1.0, + "mmlu_college_medicine": 1.0, + "mmlu_college_physics": 1.0, + "mmlu_computer_security": 1.0, + "mmlu_conceptual_physics": 1.0, + "mmlu_econometrics": 1.0, + "mmlu_electrical_engineering": 1.0, + "mmlu_elementary_mathematics": 1.0, + "mmlu_formal_logic": 1.0, + "mmlu_global_facts": 1.0, + "mmlu_high_school_biology": 1.0, + "mmlu_high_school_chemistry": 1.0, + "mmlu_high_school_computer_science": 1.0, + "mmlu_high_school_european_history": 1.0, + "mmlu_high_school_geography": 1.0, + "mmlu_high_school_government_and_politics": 1.0, + "mmlu_high_school_macroeconomics": 1.0, + "mmlu_high_school_mathematics": 1.0, + "mmlu_high_school_microeconomics": 1.0, + "mmlu_high_school_physics": 1.0, + "mmlu_high_school_psychology": 1.0, + "mmlu_high_school_statistics": 1.0, + "mmlu_high_school_us_history": 1.0, + "mmlu_high_school_world_history": 1.0, + "mmlu_human_aging": 1.0, + "mmlu_human_sexuality": 1.0, + "mmlu_humanities": "2", + "mmlu_international_law": 1.0, + "mmlu_jurisprudence": 1.0, + "mmlu_logical_fallacies": 1.0, + "mmlu_machine_learning": 1.0, + "mmlu_management": 1.0, + "mmlu_marketing": 1.0, + "mmlu_medical_genetics": 1.0, + "mmlu_miscellaneous": 1.0, + "mmlu_moral_disputes": 1.0, + "mmlu_moral_scenarios": 1.0, + "mmlu_nutrition": 1.0, + "mmlu_other": "2", + "mmlu_philosophy": 1.0, + "mmlu_prehistory": 1.0, + "mmlu_professional_accounting": 1.0, + "mmlu_professional_law": 1.0, + "mmlu_professional_medicine": 1.0, + "mmlu_professional_psychology": 1.0, + "mmlu_public_relations": 1.0, + "mmlu_security_studies": 1.0, + "mmlu_social_sciences": "2", + "mmlu_sociology": 1.0, + "mmlu_stem": "2", + "mmlu_us_foreign_policy": 1.0, + "mmlu_virology": 1.0, + "mmlu_world_religions": 1.0 + } +} diff --git a/benchmarks/mmlu_kosa.json b/benchmarks/mmlu_kosa.json new file mode 100644 index 0000000..8ea1f9b --- /dev/null +++ b/benchmarks/mmlu_kosa.json @@ -0,0 +1,4530 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", + "config": { + "batch_size": "32", + "batch_sizes": [], + "bootstrap_iters": 100000, + "device": "cuda:0", + "fewshot_seed": 1234, + "gen_kwargs": {}, + "limit": null, + "model": "vllm", + "model_args": "pretrained=kosa-labs/kosa-4B-it-v1", + "numpy_seed": 1234, + "random_seed": 0, + "torch_seed": 1234, + "use_cache": "[redacted]" + }, + "configs": { + "mmlu_abstract_algebra": { + "dataset_name": "abstract_algebra", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_abstract_algebra", + "task_alias": "abstract_algebra", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_anatomy": { + "dataset_name": "anatomy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_anatomy", + "task_alias": "anatomy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_astronomy": { + "dataset_name": "astronomy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_astronomy", + "task_alias": "astronomy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_business_ethics": { + "dataset_name": "business_ethics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_business_ethics", + "task_alias": "business_ethics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_clinical_knowledge": { + "dataset_name": "clinical_knowledge", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_clinical_knowledge", + "task_alias": "clinical_knowledge", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_biology": { + "dataset_name": "college_biology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college biology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_biology", + "task_alias": "college_biology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_chemistry": { + "dataset_name": "college_chemistry", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_chemistry", + "task_alias": "college_chemistry", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_computer_science": { + "dataset_name": "college_computer_science", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_computer_science", + "task_alias": "college_computer_science", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_mathematics": { + "dataset_name": "college_mathematics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_mathematics", + "task_alias": "college_mathematics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_medicine": { + "dataset_name": "college_medicine", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_medicine", + "task_alias": "college_medicine", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_college_physics": { + "dataset_name": "college_physics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about college physics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_college_physics", + "task_alias": "college_physics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_computer_security": { + "dataset_name": "computer_security", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about computer security.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_computer_security", + "task_alias": "computer_security", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_conceptual_physics": { + "dataset_name": "conceptual_physics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_conceptual_physics", + "task_alias": "conceptual_physics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_econometrics": { + "dataset_name": "econometrics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_econometrics", + "task_alias": "econometrics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_electrical_engineering": { + "dataset_name": "electrical_engineering", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_electrical_engineering", + "task_alias": "electrical_engineering", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_elementary_mathematics": { + "dataset_name": "elementary_mathematics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_elementary_mathematics", + "task_alias": "elementary_mathematics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_formal_logic": { + "dataset_name": "formal_logic", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_formal_logic", + "task_alias": "formal_logic", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_global_facts": { + "dataset_name": "global_facts", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about global facts.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_global_facts", + "task_alias": "global_facts", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_biology": { + "dataset_name": "high_school_biology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_biology", + "task_alias": "high_school_biology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_chemistry": { + "dataset_name": "high_school_chemistry", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_chemistry", + "task_alias": "high_school_chemistry", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_computer_science": { + "dataset_name": "high_school_computer_science", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_computer_science", + "task_alias": "high_school_computer_science", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_european_history": { + "dataset_name": "high_school_european_history", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_european_history", + "task_alias": "high_school_european_history", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_geography": { + "dataset_name": "high_school_geography", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_geography", + "task_alias": "high_school_geography", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_government_and_politics": { + "dataset_name": "high_school_government_and_politics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_government_and_politics", + "task_alias": "high_school_government_and_politics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_macroeconomics": { + "dataset_name": "high_school_macroeconomics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_macroeconomics", + "task_alias": "high_school_macroeconomics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_mathematics": { + "dataset_name": "high_school_mathematics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_mathematics", + "task_alias": "high_school_mathematics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_microeconomics": { + "dataset_name": "high_school_microeconomics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_microeconomics", + "task_alias": "high_school_microeconomics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_physics": { + "dataset_name": "high_school_physics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_physics", + "task_alias": "high_school_physics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_psychology": { + "dataset_name": "high_school_psychology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_psychology", + "task_alias": "high_school_psychology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_statistics": { + "dataset_name": "high_school_statistics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_statistics", + "task_alias": "high_school_statistics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_us_history": { + "dataset_name": "high_school_us_history", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_us_history", + "task_alias": "high_school_us_history", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_high_school_world_history": { + "dataset_name": "high_school_world_history", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_high_school_world_history", + "task_alias": "high_school_world_history", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_human_aging": { + "dataset_name": "human_aging", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about human aging.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_human_aging", + "task_alias": "human_aging", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_human_sexuality": { + "dataset_name": "human_sexuality", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_human_sexuality", + "task_alias": "human_sexuality", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_international_law": { + "dataset_name": "international_law", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about international law.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_international_law", + "task_alias": "international_law", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_jurisprudence": { + "dataset_name": "jurisprudence", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_jurisprudence", + "task_alias": "jurisprudence", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_logical_fallacies": { + "dataset_name": "logical_fallacies", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_logical_fallacies", + "task_alias": "logical_fallacies", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_machine_learning": { + "dataset_name": "machine_learning", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_machine_learning", + "task_alias": "machine_learning", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_management": { + "dataset_name": "management", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about management.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_management", + "task_alias": "management", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_marketing": { + "dataset_name": "marketing", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about marketing.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_marketing", + "task_alias": "marketing", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_medical_genetics": { + "dataset_name": "medical_genetics", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_medical_genetics", + "task_alias": "medical_genetics", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_miscellaneous": { + "dataset_name": "miscellaneous", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_miscellaneous", + "task_alias": "miscellaneous", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_moral_disputes": { + "dataset_name": "moral_disputes", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_moral_disputes", + "task_alias": "moral_disputes", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_moral_scenarios": { + "dataset_name": "moral_scenarios", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_moral_scenarios", + "task_alias": "moral_scenarios", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_nutrition": { + "dataset_name": "nutrition", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_nutrition", + "task_alias": "nutrition", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_philosophy": { + "dataset_name": "philosophy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_philosophy", + "task_alias": "philosophy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_prehistory": { + "dataset_name": "prehistory", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_prehistory", + "task_alias": "prehistory", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_accounting": { + "dataset_name": "professional_accounting", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_accounting", + "task_alias": "professional_accounting", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_law": { + "dataset_name": "professional_law", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional law.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_law", + "task_alias": "professional_law", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_medicine": { + "dataset_name": "professional_medicine", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_medicine", + "task_alias": "professional_medicine", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_professional_psychology": { + "dataset_name": "professional_psychology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_professional_psychology", + "task_alias": "professional_psychology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_public_relations": { + "dataset_name": "public_relations", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about public relations.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_public_relations", + "task_alias": "public_relations", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_security_studies": { + "dataset_name": "security_studies", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about security studies.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_security_studies", + "task_alias": "security_studies", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_sociology": { + "dataset_name": "sociology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about sociology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_sociology", + "task_alias": "sociology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_us_foreign_policy": { + "dataset_name": "us_foreign_policy", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_us_foreign_policy", + "task_alias": "us_foreign_policy", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_virology": { + "dataset_name": "virology", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about virology.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_virology", + "task_alias": "virology", + "test_split": "test", + "unsafe_code": false + }, + "mmlu_world_religions": { + "dataset_name": "world_religions", + "dataset_path": "cais/mmlu", + "description": "The following are multiple choice questions (with answers) about world religions.\n\n", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_config": { + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", + "fewshot_delimiter": "\n\n", + "fewshot_indices": null, + "gen_prefix": null, + "process_docs": null, + "sampler": "first_n", + "samples": null, + "split": "dev", + "target_delimiter": " " + }, + "fewshot_delimiter": "\n\n", + "fewshot_split": "dev", + "metadata": { + "config_source": "[redacted]", + "dtype": "bfloat16", + "gpu_memory_utilization": 0.82, + "pretrained": "[redacted]", + "tensor_parallel_size": 1, + "trust_remote_code": true, + "version": 1.0 + }, + "metric_list": [ + { + "aggregation": "mean", + "higher_is_better": true, + "metric": "acc" + } + ], + "num_fewshot": 0, + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "target_delimiter": " ", + "task": "mmlu_world_religions", + "task_alias": "world_religions", + "test_split": "test", + "unsafe_code": false + } + }, + "date": 1781174539.5106196, + "eot_token_id": 151645, + "fewshot_as_multiturn": true, + "git_hash": null, + "group_subtasks": { + "mmlu": [ + "mmlu_stem", + "mmlu_other", + "mmlu_social_sciences", + "mmlu_humanities" + ], + "mmlu_humanities": [ + "mmlu_formal_logic", + "mmlu_high_school_european_history", + "mmlu_high_school_us_history", + "mmlu_high_school_world_history", + "mmlu_international_law", + "mmlu_jurisprudence", + "mmlu_logical_fallacies", + "mmlu_moral_disputes", + "mmlu_moral_scenarios", + "mmlu_philosophy", + "mmlu_prehistory", + "mmlu_professional_law", + "mmlu_world_religions" + ], + "mmlu_other": [ + "mmlu_business_ethics", + "mmlu_clinical_knowledge", + "mmlu_college_medicine", + "mmlu_global_facts", + "mmlu_human_aging", + "mmlu_management", + "mmlu_marketing", + "mmlu_medical_genetics", + "mmlu_miscellaneous", + "mmlu_nutrition", + "mmlu_professional_accounting", + "mmlu_professional_medicine", + "mmlu_virology" + ], + "mmlu_social_sciences": [ + "mmlu_econometrics", + "mmlu_high_school_geography", + "mmlu_high_school_government_and_politics", + "mmlu_high_school_macroeconomics", + "mmlu_high_school_microeconomics", + "mmlu_high_school_psychology", + "mmlu_human_sexuality", + "mmlu_professional_psychology", + "mmlu_public_relations", + "mmlu_security_studies", + "mmlu_sociology", + "mmlu_us_foreign_policy" + ], + "mmlu_stem": [ + "mmlu_abstract_algebra", + "mmlu_anatomy", + "mmlu_astronomy", + "mmlu_college_biology", + "mmlu_college_chemistry", + "mmlu_college_computer_science", + "mmlu_college_mathematics", + "mmlu_college_physics", + "mmlu_computer_security", + "mmlu_conceptual_physics", + "mmlu_electrical_engineering", + "mmlu_elementary_mathematics", + "mmlu_high_school_biology", + "mmlu_high_school_chemistry", + "mmlu_high_school_computer_science", + "mmlu_high_school_mathematics", + "mmlu_high_school_physics", + "mmlu_high_school_statistics", + "mmlu_machine_learning" + ] + }, + "groups": { + "mmlu": { + "acc,none": 0.6575986326734083, + "acc_stderr,none": 0.0036785245967130517, + "alias": "mmlu", + "name": "mmlu", + "sample_count": { + "acc,none": 14042 + }, + "sample_len": 14042 + }, + "mmlu_humanities": { + "acc,none": 0.5883103081827843, + "acc_stderr,none": 0.006616678303159137, + "alias": "humanities", + "name": "mmlu_humanities", + "sample_count": { + "acc,none": 4705 + }, + "sample_len": 4705 + }, + "mmlu_other": { + "acc,none": 0.7219182491149019, + "acc_stderr,none": 0.007740746200087848, + "alias": "other", + "name": "mmlu_other", + "sample_count": { + "acc,none": 3107 + }, + "sample_len": 3107 + }, + "mmlu_social_sciences": { + "acc,none": 0.7897302567435814, + "acc_stderr,none": 0.007206549343813523, + "alias": "social sciences", + "name": "mmlu_social_sciences", + "sample_count": { + "acc,none": 3077 + }, + "sample_len": 3077 + }, + "mmlu_stem": { + "acc,none": 0.5686647637170948, + "acc_stderr,none": 0.007952947130384178, + "alias": "stem", + "name": "mmlu_stem", + "sample_count": { + "acc,none": 3153 + }, + "sample_len": 3153 + } + }, + "higher_is_better": { + "mmlu_abstract_algebra": { + "acc": true + }, + "mmlu_anatomy": { + "acc": true + }, + "mmlu_astronomy": { + "acc": true + }, + "mmlu_business_ethics": { + "acc": true + }, + "mmlu_clinical_knowledge": { + "acc": true + }, + "mmlu_college_biology": { + "acc": true + }, + "mmlu_college_chemistry": { + "acc": true + }, + "mmlu_college_computer_science": { + "acc": true + }, + "mmlu_college_mathematics": { + "acc": true + }, + "mmlu_college_medicine": { + "acc": true + }, + "mmlu_college_physics": { + "acc": true + }, + "mmlu_computer_security": { + "acc": true + }, + "mmlu_conceptual_physics": { + "acc": true + }, + "mmlu_econometrics": { + "acc": true + }, + "mmlu_electrical_engineering": { + "acc": true + }, + "mmlu_elementary_mathematics": { + "acc": true + }, + "mmlu_formal_logic": { + "acc": true + }, + "mmlu_global_facts": { + "acc": true + }, + "mmlu_high_school_biology": { + "acc": true + }, + "mmlu_high_school_chemistry": { + "acc": true + }, + "mmlu_high_school_computer_science": { + "acc": true + }, + "mmlu_high_school_european_history": { + "acc": true + }, + "mmlu_high_school_geography": { + "acc": true + }, + "mmlu_high_school_government_and_politics": { + "acc": true + }, + "mmlu_high_school_macroeconomics": { + "acc": true + }, + "mmlu_high_school_mathematics": { + "acc": true + }, + "mmlu_high_school_microeconomics": { + "acc": true + }, + "mmlu_high_school_physics": { + "acc": true + }, + "mmlu_high_school_psychology": { + "acc": true + }, + "mmlu_high_school_statistics": { + "acc": true + }, + "mmlu_high_school_us_history": { + "acc": true + }, + "mmlu_high_school_world_history": { + "acc": true + }, + "mmlu_human_aging": { + "acc": true + }, + "mmlu_human_sexuality": { + "acc": true + }, + "mmlu_humanities": { + "acc": true + }, + "mmlu_international_law": { + "acc": true + }, + "mmlu_jurisprudence": { + "acc": true + }, + "mmlu_logical_fallacies": { + "acc": true + }, + "mmlu_machine_learning": { + "acc": true + }, + "mmlu_management": { + "acc": true + }, + "mmlu_marketing": { + "acc": true + }, + "mmlu_medical_genetics": { + "acc": true + }, + "mmlu_miscellaneous": { + "acc": true + }, + "mmlu_moral_disputes": { + "acc": true + }, + "mmlu_moral_scenarios": { + "acc": true + }, + "mmlu_nutrition": { + "acc": true + }, + "mmlu_other": { + "acc": true + }, + "mmlu_philosophy": { + "acc": true + }, + "mmlu_prehistory": { + "acc": true + }, + "mmlu_professional_accounting": { + "acc": true + }, + "mmlu_professional_law": { + "acc": true + }, + "mmlu_professional_medicine": { + "acc": true + }, + "mmlu_professional_psychology": { + "acc": true + }, + "mmlu_public_relations": { + "acc": true + }, + "mmlu_security_studies": { + "acc": true + }, + "mmlu_social_sciences": { + "acc": true + }, + "mmlu_sociology": { + "acc": true + }, + "mmlu_stem": { + "acc": true + }, + "mmlu_us_foreign_policy": { + "acc": true + }, + "mmlu_virology": { + "acc": true + }, + "mmlu_world_religions": { + "acc": true + } + }, + "lm_eval_version": "0.4.12", + "max_length": 262144, + "model_name": "kosa-labs/kosa-4B-it-v1", + "model_name_sanitized": "kosa-labs/kosa-4B-it-v1", + "model_source": "kosa-labs/kosa-4B-it-v1", + "n-samples": { + "mmlu_abstract_algebra": { + "effective": 100, + "original": 100 + }, + "mmlu_anatomy": { + "effective": 135, + "original": 135 + }, + "mmlu_astronomy": { + "effective": 152, + "original": 152 + }, + "mmlu_business_ethics": { + "effective": 100, + "original": 100 + }, + "mmlu_clinical_knowledge": { + "effective": 265, + "original": 265 + }, + "mmlu_college_biology": { + "effective": 144, + "original": 144 + }, + "mmlu_college_chemistry": { + "effective": 100, + "original": 100 + }, + "mmlu_college_computer_science": { + "effective": 100, + "original": 100 + }, + "mmlu_college_mathematics": { + "effective": 100, + "original": 100 + }, + "mmlu_college_medicine": { + "effective": 173, + "original": 173 + }, + "mmlu_college_physics": { + "effective": 102, + "original": 102 + }, + "mmlu_computer_security": { + "effective": 100, + "original": 100 + }, + "mmlu_conceptual_physics": { + "effective": 235, + "original": 235 + }, + "mmlu_econometrics": { + "effective": 114, + "original": 114 + }, + "mmlu_electrical_engineering": { + "effective": 145, + "original": 145 + }, + "mmlu_elementary_mathematics": { + "effective": 378, + "original": 378 + }, + "mmlu_formal_logic": { + "effective": 126, + "original": 126 + }, + "mmlu_global_facts": { + "effective": 100, + "original": 100 + }, + "mmlu_high_school_biology": { + "effective": 310, + "original": 310 + }, + "mmlu_high_school_chemistry": { + "effective": 203, + "original": 203 + }, + "mmlu_high_school_computer_science": { + "effective": 100, + "original": 100 + }, + "mmlu_high_school_european_history": { + "effective": 165, + "original": 165 + }, + "mmlu_high_school_geography": { + "effective": 198, + "original": 198 + }, + "mmlu_high_school_government_and_politics": { + "effective": 193, + "original": 193 + }, + "mmlu_high_school_macroeconomics": { + "effective": 390, + "original": 390 + }, + "mmlu_high_school_mathematics": { + "effective": 270, + "original": 270 + }, + "mmlu_high_school_microeconomics": { + "effective": 238, + "original": 238 + }, + "mmlu_high_school_physics": { + "effective": 151, + "original": 151 + }, + "mmlu_high_school_psychology": { + "effective": 545, + "original": 545 + }, + "mmlu_high_school_statistics": { + "effective": 216, + "original": 216 + }, + "mmlu_high_school_us_history": { + "effective": 204, + "original": 204 + }, + "mmlu_high_school_world_history": { + "effective": 237, + "original": 237 + }, + "mmlu_human_aging": { + "effective": 223, + "original": 223 + }, + "mmlu_human_sexuality": { + "effective": 131, + "original": 131 + }, + "mmlu_international_law": { + "effective": 121, + "original": 121 + }, + "mmlu_jurisprudence": { + "effective": 108, + "original": 108 + }, + "mmlu_logical_fallacies": { + "effective": 163, + "original": 163 + }, + "mmlu_machine_learning": { + "effective": 112, + "original": 112 + }, + "mmlu_management": { + "effective": 103, + "original": 103 + }, + "mmlu_marketing": { + "effective": 234, + "original": 234 + }, + "mmlu_medical_genetics": { + "effective": 100, + "original": 100 + }, + "mmlu_miscellaneous": { + "effective": 783, + "original": 783 + }, + "mmlu_moral_disputes": { + "effective": 346, + "original": 346 + }, + "mmlu_moral_scenarios": { + "effective": 895, + "original": 895 + }, + "mmlu_nutrition": { + "effective": 306, + "original": 306 + }, + "mmlu_philosophy": { + "effective": 311, + "original": 311 + }, + "mmlu_prehistory": { + "effective": 324, + "original": 324 + }, + "mmlu_professional_accounting": { + "effective": 282, + "original": 282 + }, + "mmlu_professional_law": { + "effective": 1534, + "original": 1534 + }, + "mmlu_professional_medicine": { + "effective": 272, + "original": 272 + }, + "mmlu_professional_psychology": { + "effective": 612, + "original": 612 + }, + "mmlu_public_relations": { + "effective": 110, + "original": 110 + }, + "mmlu_security_studies": { + "effective": 245, + "original": 245 + }, + "mmlu_sociology": { + "effective": 201, + "original": 201 + }, + "mmlu_us_foreign_policy": { + "effective": 100, + "original": 100 + }, + "mmlu_virology": { + "effective": 166, + "original": 166 + }, + "mmlu_world_religions": { + "effective": 171, + "original": 171 + } + }, + "n-shot": { + "mmlu_abstract_algebra": 0, + "mmlu_anatomy": 0, + "mmlu_astronomy": 0, + "mmlu_business_ethics": 0, + "mmlu_clinical_knowledge": 0, + "mmlu_college_biology": 0, + "mmlu_college_chemistry": 0, + "mmlu_college_computer_science": 0, + "mmlu_college_mathematics": 0, + "mmlu_college_medicine": 0, + "mmlu_college_physics": 0, + "mmlu_computer_security": 0, + "mmlu_conceptual_physics": 0, + "mmlu_econometrics": 0, + "mmlu_electrical_engineering": 0, + "mmlu_elementary_mathematics": 0, + "mmlu_formal_logic": 0, + "mmlu_global_facts": 0, + "mmlu_high_school_biology": 0, + "mmlu_high_school_chemistry": 0, + "mmlu_high_school_computer_science": 0, + "mmlu_high_school_european_history": 0, + "mmlu_high_school_geography": 0, + "mmlu_high_school_government_and_politics": 0, + "mmlu_high_school_macroeconomics": 0, + "mmlu_high_school_mathematics": 0, + "mmlu_high_school_microeconomics": 0, + "mmlu_high_school_physics": 0, + "mmlu_high_school_psychology": 0, + "mmlu_high_school_statistics": 0, + "mmlu_high_school_us_history": 0, + "mmlu_high_school_world_history": 0, + "mmlu_human_aging": 0, + "mmlu_human_sexuality": 0, + "mmlu_humanities": 0, + "mmlu_international_law": 0, + "mmlu_jurisprudence": 0, + "mmlu_logical_fallacies": 0, + "mmlu_machine_learning": 0, + "mmlu_management": 0, + "mmlu_marketing": 0, + "mmlu_medical_genetics": 0, + "mmlu_miscellaneous": 0, + "mmlu_moral_disputes": 0, + "mmlu_moral_scenarios": 0, + "mmlu_nutrition": 0, + "mmlu_other": 0, + "mmlu_philosophy": 0, + "mmlu_prehistory": 0, + "mmlu_professional_accounting": 0, + "mmlu_professional_law": 0, + "mmlu_professional_medicine": 0, + "mmlu_professional_psychology": 0, + "mmlu_public_relations": 0, + "mmlu_security_studies": 0, + "mmlu_social_sciences": 0, + "mmlu_sociology": 0, + "mmlu_stem": 0, + "mmlu_us_foreign_policy": 0, + "mmlu_virology": 0, + "mmlu_world_religions": 0 + }, + "results": { + "mmlu": { + "acc,none": 0.6575986326734083, + "acc_stderr,none": 0.0036785245967130517, + "alias": "mmlu", + "name": "mmlu", + "sample_count": { + "acc,none": 14042 + }, + "sample_len": 14042 + }, + "mmlu_abstract_algebra": { + "acc,none": 0.25, + "acc_stderr,none": 0.04351941398892446, + "alias": "abstract_algebra", + "name": "mmlu_abstract_algebra", + "sample_len": 100 + }, + "mmlu_anatomy": { + "acc,none": 0.674074074074074, + "acc_stderr,none": 0.040491220417025006, + "alias": "anatomy", + "name": "mmlu_anatomy", + "sample_len": 135 + }, + "mmlu_astronomy": { + "acc,none": 0.8355263157894737, + "acc_stderr,none": 0.030167533468632726, + "alias": "astronomy", + "name": "mmlu_astronomy", + "sample_len": 152 + }, + "mmlu_business_ethics": { + "acc,none": 0.74, + "acc_stderr,none": 0.0440844002276808, + "alias": "business_ethics", + "name": "mmlu_business_ethics", + "sample_len": 100 + }, + "mmlu_clinical_knowledge": { + "acc,none": 0.7509433962264151, + "acc_stderr,none": 0.02661648298050167, + "alias": "clinical_knowledge", + "name": "mmlu_clinical_knowledge", + "sample_len": 265 + }, + "mmlu_college_biology": { + "acc,none": 0.8680555555555556, + "acc_stderr,none": 0.02830096838204442, + "alias": "college_biology", + "name": "mmlu_college_biology", + "sample_len": 144 + }, + "mmlu_college_chemistry": { + "acc,none": 0.47, + "acc_stderr,none": 0.05016135580465919, + "alias": "college_chemistry", + "name": "mmlu_college_chemistry", + "sample_len": 100 + }, + "mmlu_college_computer_science": { + "acc,none": 0.51, + "acc_stderr,none": 0.05024183937956913, + "alias": "college_computer_science", + "name": "mmlu_college_computer_science", + "sample_len": 100 + }, + "mmlu_college_mathematics": { + "acc,none": 0.24, + "acc_stderr,none": 0.04292346959909278, + "alias": "college_mathematics", + "name": "mmlu_college_mathematics", + "sample_len": 100 + }, + "mmlu_college_medicine": { + "acc,none": 0.6705202312138728, + "acc_stderr,none": 0.03583901754736415, + "alias": "college_medicine", + "name": "mmlu_college_medicine", + "sample_len": 173 + }, + "mmlu_college_physics": { + "acc,none": 0.39215686274509803, + "acc_stderr,none": 0.04858083574266346, + "alias": "college_physics", + "name": "mmlu_college_physics", + "sample_len": 102 + }, + "mmlu_computer_security": { + "acc,none": 0.79, + "acc_stderr,none": 0.040936018074033236, + "alias": "computer_security", + "name": "mmlu_computer_security", + "sample_len": 100 + }, + "mmlu_conceptual_physics": { + "acc,none": 0.774468085106383, + "acc_stderr,none": 0.02732107841738755, + "alias": "conceptual_physics", + "name": "mmlu_conceptual_physics", + "sample_len": 235 + }, + "mmlu_econometrics": { + "acc,none": 0.5614035087719298, + "acc_stderr,none": 0.04668000738510451, + "alias": "econometrics", + "name": "mmlu_econometrics", + "sample_len": 114 + }, + "mmlu_electrical_engineering": { + "acc,none": 0.6896551724137931, + "acc_stderr,none": 0.03855289616378948, + "alias": "electrical_engineering", + "name": "mmlu_electrical_engineering", + "sample_len": 145 + }, + "mmlu_elementary_mathematics": { + "acc,none": 0.36507936507936506, + "acc_stderr,none": 0.02479606060269989, + "alias": "elementary_mathematics", + "name": "mmlu_elementary_mathematics", + "sample_len": 378 + }, + "mmlu_formal_logic": { + "acc,none": 0.5079365079365079, + "acc_stderr,none": 0.04471572536294351, + "alias": "formal_logic", + "name": "mmlu_formal_logic", + "sample_len": 126 + }, + "mmlu_global_facts": { + "acc,none": 0.43, + "acc_stderr,none": 0.049756985195624305, + "alias": "global_facts", + "name": "mmlu_global_facts", + "sample_len": 100 + }, + "mmlu_high_school_biology": { + "acc,none": 0.8516129032258064, + "acc_stderr,none": 0.02022273755433042, + "alias": "high_school_biology", + "name": "mmlu_high_school_biology", + "sample_len": 310 + }, + "mmlu_high_school_chemistry": { + "acc,none": 0.6009852216748769, + "acc_stderr,none": 0.03445487686264716, + "alias": "high_school_chemistry", + "name": "mmlu_high_school_chemistry", + "sample_len": 203 + }, + "mmlu_high_school_computer_science": { + "acc,none": 0.78, + "acc_stderr,none": 0.041633319989322654, + "alias": "high_school_computer_science", + "name": "mmlu_high_school_computer_science", + "sample_len": 100 + }, + "mmlu_high_school_european_history": { + "acc,none": 0.8363636363636363, + "acc_stderr,none": 0.028887872395487978, + "alias": "high_school_european_history", + "name": "mmlu_high_school_european_history", + "sample_len": 165 + }, + "mmlu_high_school_geography": { + "acc,none": 0.8434343434343434, + "acc_stderr,none": 0.025890520358141458, + "alias": "high_school_geography", + "name": "mmlu_high_school_geography", + "sample_len": 198 + }, + "mmlu_high_school_government_and_politics": { + "acc,none": 0.8911917098445595, + "acc_stderr,none": 0.02247325333276875, + "alias": "high_school_government_and_politics", + "name": "mmlu_high_school_government_and_politics", + "sample_len": 193 + }, + "mmlu_high_school_macroeconomics": { + "acc,none": 0.7256410256410256, + "acc_stderr,none": 0.022622765767493235, + "alias": "high_school_macroeconomics", + "name": "mmlu_high_school_macroeconomics", + "sample_len": 390 + }, + "mmlu_high_school_mathematics": { + "acc,none": 0.21851851851851853, + "acc_stderr,none": 0.025195752251823727, + "alias": "high_school_mathematics", + "name": "mmlu_high_school_mathematics", + "sample_len": 270 + }, + "mmlu_high_school_microeconomics": { + "acc,none": 0.8445378151260504, + "acc_stderr,none": 0.023536818625398935, + "alias": "high_school_microeconomics", + "name": "mmlu_high_school_microeconomics", + "sample_len": 238 + }, + "mmlu_high_school_physics": { + "acc,none": 0.4768211920529801, + "acc_stderr,none": 0.04078093859163086, + "alias": "high_school_physics", + "name": "mmlu_high_school_physics", + "sample_len": 151 + }, + "mmlu_high_school_psychology": { + "acc,none": 0.8880733944954129, + "acc_stderr,none": 0.013517352714958736, + "alias": "high_school_psychology", + "name": "mmlu_high_school_psychology", + "sample_len": 545 + }, + "mmlu_high_school_statistics": { + "acc,none": 0.49537037037037035, + "acc_stderr,none": 0.034098255191635805, + "alias": "high_school_statistics", + "name": "mmlu_high_school_statistics", + "sample_len": 216 + }, + "mmlu_high_school_us_history": { + "acc,none": 0.8529411764705882, + "acc_stderr,none": 0.024857478080250486, + "alias": "high_school_us_history", + "name": "mmlu_high_school_us_history", + "sample_len": 204 + }, + "mmlu_high_school_world_history": { + "acc,none": 0.8481012658227848, + "acc_stderr,none": 0.023363878096632425, + "alias": "high_school_world_history", + "name": "mmlu_high_school_world_history", + "sample_len": 237 + }, + "mmlu_human_aging": { + "acc,none": 0.6860986547085202, + "acc_stderr,none": 0.031146796482972486, + "alias": "human_aging", + "name": "mmlu_human_aging", + "sample_len": 223 + }, + "mmlu_human_sexuality": { + "acc,none": 0.7480916030534351, + "acc_stderr,none": 0.03807387116306089, + "alias": "human_sexuality", + "name": "mmlu_human_sexuality", + "sample_len": 131 + }, + "mmlu_humanities": { + "acc,none": 0.5883103081827843, + "acc_stderr,none": 0.006616678303159137, + "alias": "humanities", + "name": "mmlu_humanities", + "sample_count": { + "acc,none": 4705 + }, + "sample_len": 4705 + }, + "mmlu_international_law": { + "acc,none": 0.7933884297520661, + "acc_stderr,none": 0.03695980128098826, + "alias": "international_law", + "name": "mmlu_international_law", + "sample_len": 121 + }, + "mmlu_jurisprudence": { + "acc,none": 0.7777777777777778, + "acc_stderr,none": 0.04019107472557348, + "alias": "jurisprudence", + "name": "mmlu_jurisprudence", + "sample_len": 108 + }, + "mmlu_logical_fallacies": { + "acc,none": 0.7975460122699386, + "acc_stderr,none": 0.031570650789119054, + "alias": "logical_fallacies", + "name": "mmlu_logical_fallacies", + "sample_len": 163 + }, + "mmlu_machine_learning": { + "acc,none": 0.5535714285714286, + "acc_stderr,none": 0.04718471485219583, + "alias": "machine_learning", + "name": "mmlu_machine_learning", + "sample_len": 112 + }, + "mmlu_management": { + "acc,none": 0.8446601941747572, + "acc_stderr,none": 0.03586594738573976, + "alias": "management", + "name": "mmlu_management", + "sample_len": 103 + }, + "mmlu_marketing": { + "acc,none": 0.8888888888888888, + "acc_stderr,none": 0.02058849131609242, + "alias": "marketing", + "name": "mmlu_marketing", + "sample_len": 234 + }, + "mmlu_medical_genetics": { + "acc,none": 0.83, + "acc_stderr,none": 0.03775251680686369, + "alias": "medical_genetics", + "name": "mmlu_medical_genetics", + "sample_len": 100 + }, + "mmlu_miscellaneous": { + "acc,none": 0.8071519795657727, + "acc_stderr,none": 0.014108533515757481, + "alias": "miscellaneous", + "name": "mmlu_miscellaneous", + "sample_len": 783 + }, + "mmlu_moral_disputes": { + "acc,none": 0.7167630057803468, + "acc_stderr,none": 0.024257901705323326, + "alias": "moral_disputes", + "name": "mmlu_moral_disputes", + "sample_len": 346 + }, + "mmlu_moral_scenarios": { + "acc,none": 0.31508379888268156, + "acc_stderr,none": 0.015536850852473702, + "alias": "moral_scenarios", + "name": "mmlu_moral_scenarios", + "sample_len": 895 + }, + "mmlu_nutrition": { + "acc,none": 0.7810457516339869, + "acc_stderr,none": 0.02367908986180772, + "alias": "nutrition", + "name": "mmlu_nutrition", + "sample_len": 306 + }, + "mmlu_other": { + "acc,none": 0.7219182491149019, + "acc_stderr,none": 0.007740746200087848, + "alias": "other", + "name": "mmlu_other", + "sample_count": { + "acc,none": 3107 + }, + "sample_len": 3107 + }, + "mmlu_philosophy": { + "acc,none": 0.7106109324758842, + "acc_stderr,none": 0.02575586592263286, + "alias": "philosophy", + "name": "mmlu_philosophy", + "sample_len": 311 + }, + "mmlu_prehistory": { + "acc,none": 0.7716049382716049, + "acc_stderr,none": 0.02335821184062624, + "alias": "prehistory", + "name": "mmlu_prehistory", + "sample_len": 324 + }, + "mmlu_professional_accounting": { + "acc,none": 0.49645390070921985, + "acc_stderr,none": 0.029826749153280853, + "alias": "professional_accounting", + "name": "mmlu_professional_accounting", + "sample_len": 282 + }, + "mmlu_professional_law": { + "acc,none": 0.4810951760104302, + "acc_stderr,none": 0.012761104871472521, + "alias": "professional_law", + "name": "mmlu_professional_law", + "sample_len": 1534 + }, + "mmlu_professional_medicine": { + "acc,none": 0.6764705882352942, + "acc_stderr,none": 0.028418208619406728, + "alias": "professional_medicine", + "name": "mmlu_professional_medicine", + "sample_len": 272 + }, + "mmlu_professional_psychology": { + "acc,none": 0.7336601307189542, + "acc_stderr,none": 0.017883188134667258, + "alias": "professional_psychology", + "name": "mmlu_professional_psychology", + "sample_len": 612 + }, + "mmlu_public_relations": { + "acc,none": 0.6545454545454545, + "acc_stderr,none": 0.045546196175410524, + "alias": "public_relations", + "name": "mmlu_public_relations", + "sample_len": 110 + }, + "mmlu_security_studies": { + "acc,none": 0.7673469387755102, + "acc_stderr,none": 0.02704925791589615, + "alias": "security_studies", + "name": "mmlu_security_studies", + "sample_len": 245 + }, + "mmlu_social_sciences": { + "acc,none": 0.7897302567435814, + "acc_stderr,none": 0.007206549343813523, + "alias": "social sciences", + "name": "mmlu_social_sciences", + "sample_count": { + "acc,none": 3077 + }, + "sample_len": 3077 + }, + "mmlu_sociology": { + "acc,none": 0.8507462686567164, + "acc_stderr,none": 0.02519692987482706, + "alias": "sociology", + "name": "mmlu_sociology", + "sample_len": 201 + }, + "mmlu_stem": { + "acc,none": 0.5686647637170948, + "acc_stderr,none": 0.007952947130384178, + "alias": "stem", + "name": "mmlu_stem", + "sample_count": { + "acc,none": 3153 + }, + "sample_len": 3153 + }, + "mmlu_us_foreign_policy": { + "acc,none": 0.81, + "acc_stderr,none": 0.039427724440366255, + "alias": "us_foreign_policy", + "name": "mmlu_us_foreign_policy", + "sample_len": 100 + }, + "mmlu_virology": { + "acc,none": 0.5120481927710844, + "acc_stderr,none": 0.038913644958358196, + "alias": "virology", + "name": "mmlu_virology", + "sample_len": 166 + }, + "mmlu_world_religions": { + "acc,none": 0.8304093567251462, + "acc_stderr,none": 0.02878210810540169, + "alias": "world_religions", + "name": "mmlu_world_religions", + "sample_len": 171 + } + }, + "system_instruction": null, + "system_instruction_sha": null, + "task_hashes": { + "mmlu_abstract_algebra": "1ecc29cafeff483bf80347adc2878dae00d01596988032f438afce6291c8cfe7", + "mmlu_anatomy": "51232e78f32967e0dd431bab9309f69be490f9998b4ddb93b641b09ba1f613cf", + "mmlu_astronomy": "3e8d7c33bd0431518a3d5d4b1666bdabccf46c4cc4cfe95a3886cb2266743f86", + "mmlu_business_ethics": "6ec93578c18c7bba58a384ecaa7e6e84c5e681262b34b1caa76abbbb0f9ebb02", + "mmlu_clinical_knowledge": "cf39fec90aa76a379e440c1f0d5f5b04d9f32edc4004ffcfb8eadcc25c18b118", + "mmlu_college_biology": "5378a297efa1cd71344228c95dcb29ba54757c7e527261400a4d65fd8ceed92b", + "mmlu_college_chemistry": "714e60ea7e340f0344ffde41b247a19b68e6384152a1bdc9acc0ecf24cd0e7cd", + "mmlu_college_computer_science": "2b66d244f98e61e6a78e5ed48b728e0ca2ca81c7757ccdd086fe6b0c1cf02da1", + "mmlu_college_mathematics": "90b1ce25d7d38abd925181ae3aad095cfd783375fe8de0607a57420f47454f0f", + "mmlu_college_medicine": "00cc41d26670913bdee6181e5b602726dde9f03e6a43236c2bf57486149845be", + "mmlu_college_physics": "2abddb12a247904f9ba60b39766b19a8967c24fe8eee021022a353b93d73742c", + "mmlu_computer_security": "8ffba21cebb9e39d5794c848a22e7e37cddff5c59f7b47a1d89872d9e8704b0c", + "mmlu_conceptual_physics": "c0da3219fe7e5292dad2aa7e72790e3eba951711007cc57618b0850d941b85ec", + "mmlu_econometrics": "bc667156b739500e02f7c8cb8f9621b6d62ea2011e1eaa3404cb45f6d16778ec", + "mmlu_electrical_engineering": "0a02cb16a31ec7debba4190ab1cc31afadb9416764e0b074b9b96553f346b892", + "mmlu_elementary_mathematics": "6048f1506468e757010d537a0ceedaadc587c18159a03e7664b5994843afedc2", + "mmlu_formal_logic": "5fef452fe37d3c5aefce6276a26a839e5ee5d655d72adf50bbf0a8ae0e8d74e4", + "mmlu_global_facts": "6163d6678b902c8cb5cd6c50f59bc0f31a9a356e977fcd6184614f48e3320e7a", + "mmlu_high_school_biology": "2aabe83bb100a352d24a0136c24e612b68490bbe818c33f06cc6cee5379ecad8", + "mmlu_high_school_chemistry": "dd3321fd42c8717094f57dd0eaeab0002d9ed192518a7de7aaef4913796c30f0", + "mmlu_high_school_computer_science": "888aabb9a7ac77299e8b2b8429fa44c1fbbb5ac8165f556a0b00b347e51de3b5", + "mmlu_high_school_european_history": "0b7be1963c7e224bffc18d72fe277408c4980f854a5fc94805fc008133640f44", + "mmlu_high_school_geography": "915db5f9f9c584c6886ae92cfc28ad9c364bf685f22c15009ad8bf064b4a8027", + "mmlu_high_school_government_and_politics": "411c6c6c179d1e2a252fab88f33123721540fc70f8223310e1f1349ec0b67753", + "mmlu_high_school_macroeconomics": "291b29471838255d1a2dd704960610e98b7e3f2fd001e3aabc6b25be0c236e29", + "mmlu_high_school_mathematics": "9b9567fc5c7875065ac91eac69621904f201cab822621440a2442a390f62b497", + "mmlu_high_school_microeconomics": "6f4a887cb73beee5414248002f75216c16d7c8ff739b37a2607c6bcfad16712b", + "mmlu_high_school_physics": "53edb93aefb6fc54fba283278bae10a58a7902902ff55edc30cb9c81745dbaa8", + "mmlu_high_school_psychology": "ce01f01ff9a3bdc3e956ada7ec7c02f13771c386d076323056ac00eec0095785", + "mmlu_high_school_statistics": "e0d652bf0cb6f927b0fc738e58a141f0ba8f9073615f4ddd70bf7a3b97998f21", + "mmlu_high_school_us_history": "1b31fdc7e0d51803fa6107a0c7aa3b6968031bc449ed3513a7349956fa0b53d3", + "mmlu_high_school_world_history": "9e352460adb245947aaf9671f135ae6484e44a115312955c45c7caaa0688f160", + "mmlu_human_aging": "4b4276095e96dc6e2872517b2becc2ebe9fad0f42b321486f6f2ceaef81f385e", + "mmlu_human_sexuality": "9395014a475c63632383587439df5c48c5dd3333eaa8d10ef9e0eaabd0997c3a", + "mmlu_international_law": "3d2036ffe61014fa8a2de679c6bee2c0c3ade112f03cd55e4a6fba5bdc56bf7b", + "mmlu_jurisprudence": "1548c06ef17823048b473d028f4889b3cf014beac4ac808521898ee141a34d23", + "mmlu_logical_fallacies": "66f9691122f3fa813b86fda1131037c03cf42babe650c7374924f41882716332", + "mmlu_machine_learning": "13ad1d3650a26d655a2a2eaa3be1f106d990e5b693b0e9006f8ebef221d2ed82", + "mmlu_management": "84a18a4a3039f23b7dd18838c89a80171025a1a45eda3305464a3cfcd8257b69", + "mmlu_marketing": "d6382d405e202221a78087356861723acb445fbd6f1db94152201f788b5c0606", + "mmlu_medical_genetics": "b4546cc241c5811dddfec47d0bba37122fbd65545c93d60f50b993e810d37e2d", + "mmlu_miscellaneous": "ecf4c3e0dc895ca5b7803d1b3f01fe3027b461f900ac001d36f5dd750c8eca1d", + "mmlu_moral_disputes": "84f580166c00063790619588b4f9fcd2f800a6a04778f08c1bada043bfdc6ec0", + "mmlu_moral_scenarios": "0dda4f43fd25e66311e64f4590698bfcc4c00c24ac58dbfec74b613512280ffc", + "mmlu_nutrition": "732959f39ac0c6c0e9c3823082d3669467531cebfb5f7550d79bcc31e5cfdbce", + "mmlu_philosophy": "ff6284cd5c6bfdb17898ef97dabc97916b70fc7485c57543edabdd44b1d8b543", + "mmlu_prehistory": "efb58ea420f5b7975ff34766ddc7470676e1ababa0b4705bfdf150f6871fdd83", + "mmlu_professional_accounting": "3c555bc35f86d097f509755f487781d2bd288de5dfe9aa59d8009a1dd0583c45", + "mmlu_professional_law": "b108c04f61cbbb4b9351de3730d7e3396710c7702c6cf75703155dbfdbc7819d", + "mmlu_professional_medicine": "0821fa48ca3715604bbef122215aea4e55f218dad6ba5191da2eb831971ac6f9", + "mmlu_professional_psychology": "ab273adb58e2b365b3811e4ed4c6485c9630f78058580273703eb2573dde9010", + "mmlu_public_relations": "da7dfa166f545d33a61c930908eee4f8d8544cad6328569b1c4b341becaf68f1", + "mmlu_security_studies": "85ea8f6c3b791998f3c1d57164b6e28bcf4498f65fc1988dfa4a6a65d31fb505", + "mmlu_sociology": "055d024e9587edc811c54e6cdd8fd74d535c6f5b1aad17cfd5f1598a1371606a", + "mmlu_us_foreign_policy": "a99e6492b6831a5df8c7f4c76edf15b500e59ab4c87dc73fdbd61d43bc704682", + "mmlu_virology": "ba4fc492ed4e9af97e30850257d2fdc5e7b53b37e7b4d68d443f7bd157aac21f", + "mmlu_world_religions": "9634e441f5f53329b4b6c760b08b1a130f8f03b3cc1902b23d07f57db76b7263" + }, + "tokenizer_bos_token": [ + null, + "None" + ], + "tokenizer_eos_token": [ + "<|im_end|>", + "151645" + ], + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "total_evaluation_time_seconds": "587.0024431729689", + "transformers_version": "5.11.0", + "upper_git_hash": null, + "versions": { + "mmlu": "2", + "mmlu_abstract_algebra": 1.0, + "mmlu_anatomy": 1.0, + "mmlu_astronomy": 1.0, + "mmlu_business_ethics": 1.0, + "mmlu_clinical_knowledge": 1.0, + "mmlu_college_biology": 1.0, + "mmlu_college_chemistry": 1.0, + "mmlu_college_computer_science": 1.0, + "mmlu_college_mathematics": 1.0, + "mmlu_college_medicine": 1.0, + "mmlu_college_physics": 1.0, + "mmlu_computer_security": 1.0, + "mmlu_conceptual_physics": 1.0, + "mmlu_econometrics": 1.0, + "mmlu_electrical_engineering": 1.0, + "mmlu_elementary_mathematics": 1.0, + "mmlu_formal_logic": 1.0, + "mmlu_global_facts": 1.0, + "mmlu_high_school_biology": 1.0, + "mmlu_high_school_chemistry": 1.0, + "mmlu_high_school_computer_science": 1.0, + "mmlu_high_school_european_history": 1.0, + "mmlu_high_school_geography": 1.0, + "mmlu_high_school_government_and_politics": 1.0, + "mmlu_high_school_macroeconomics": 1.0, + "mmlu_high_school_mathematics": 1.0, + "mmlu_high_school_microeconomics": 1.0, + "mmlu_high_school_physics": 1.0, + "mmlu_high_school_psychology": 1.0, + "mmlu_high_school_statistics": 1.0, + "mmlu_high_school_us_history": 1.0, + "mmlu_high_school_world_history": 1.0, + "mmlu_human_aging": 1.0, + "mmlu_human_sexuality": 1.0, + "mmlu_humanities": "2", + "mmlu_international_law": 1.0, + "mmlu_jurisprudence": 1.0, + "mmlu_logical_fallacies": 1.0, + "mmlu_machine_learning": 1.0, + "mmlu_management": 1.0, + "mmlu_marketing": 1.0, + "mmlu_medical_genetics": 1.0, + "mmlu_miscellaneous": 1.0, + "mmlu_moral_disputes": 1.0, + "mmlu_moral_scenarios": 1.0, + "mmlu_nutrition": 1.0, + "mmlu_other": "2", + "mmlu_philosophy": 1.0, + "mmlu_prehistory": 1.0, + "mmlu_professional_accounting": 1.0, + "mmlu_professional_law": 1.0, + "mmlu_professional_medicine": 1.0, + "mmlu_professional_psychology": 1.0, + "mmlu_public_relations": 1.0, + "mmlu_security_studies": 1.0, + "mmlu_social_sciences": "2", + "mmlu_sociology": 1.0, + "mmlu_stem": "2", + "mmlu_us_foreign_policy": 1.0, + "mmlu_virology": 1.0, + "mmlu_world_religions": 1.0 + } +} diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..70adff8 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..7ce6bb6 --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 5000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.11.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..eb58087 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.11.0" +} diff --git a/gguf/kosa-4B-it-v1-Q4_K_M.gguf b/gguf/kosa-4B-it-v1-Q4_K_M.gguf new file mode 100644 index 0000000..f86dc16 --- /dev/null +++ b/gguf/kosa-4B-it-v1-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5a57a5b3ad55fc83b4bed4e97a12871fa1d5f917e48348482699e7bdb8fb7cf +size 2497278752 diff --git a/gguf/kosa-4B-it-v1-Q5_K_M.gguf b/gguf/kosa-4B-it-v1-Q5_K_M.gguf new file mode 100644 index 0000000..22c6ea8 --- /dev/null +++ b/gguf/kosa-4B-it-v1-Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1ab2269c16ad4177bb81cecdcee8e599e3c5c70cda9fceee409e499e688f77 +size 2889511712 diff --git a/gguf/kosa-4B-it-v1-Q8_0.gguf b/gguf/kosa-4B-it-v1-Q8_0.gguf new file mode 100644 index 0000000..82744dd --- /dev/null +++ b/gguf/kosa-4B-it-v1-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:890a1a9f41604fb80c4e2ce6096f652a442970d5b9e02a0af2eed2cabf835679 +size 4280403232 diff --git a/model-00001-of-00005.safetensors b/model-00001-of-00005.safetensors new file mode 100644 index 0000000..25ada7d --- /dev/null +++ b/model-00001-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7314aaedbb6a166e626646219fa9ebe90ccb4ea7d050b250c50e12ba1e36e09 +size 1989095024 diff --git a/model-00002-of-00005.safetensors b/model-00002-of-00005.safetensors new file mode 100644 index 0000000..83ef4b0 --- /dev/null +++ b/model-00002-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f1cbd7cde765328ed06f8b3e3429d27e0935eb8cdfb468255b4d3f1b742f2dd +size 1992409040 diff --git a/model-00003-of-00005.safetensors b/model-00003-of-00005.safetensors new file mode 100644 index 0000000..a2f8e89 --- /dev/null +++ b/model-00003-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24681841d86a64ace15bee114c53683ad8e4267e6ffffc32b4a704a706c0ecb6 +size 1997656952 diff --git a/model-00004-of-00005.safetensors b/model-00004-of-00005.safetensors new file mode 100644 index 0000000..a8ffb62 --- /dev/null +++ b/model-00004-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4daf76144a2e97be5c49ff3b89bb571aa94934f793d500146e9086d06554fc5d +size 1963572880 diff --git a/model-00005-of-00005.safetensors b/model-00005-of-00005.safetensors new file mode 100644 index 0000000..af96553 --- /dev/null +++ b/model-00005-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:960ed978de289e643b62e113c42e6e2e394edc56c8a5e21d1c61ef2fe3aa7244 +size 102247936 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..bd5adac --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,406 @@ +{ + "metadata": { + "total_parameters": 4022468096, + "total_size": 8044936192 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00005.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.26.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.27.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.28.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.29.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.30.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.31.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00005-of-00005.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00005-of-00005.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.norm.weight": "model-00005-of-00005.safetensors" + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..cca9831 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}