kosa-4B-it-v1/benchmarks/arc_challenge_base.json

{
  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = '' %}\n    {%- endif %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
  "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
  "config": {
    "batch_size": "32",
    "batch_sizes": [],
    "bootstrap_iters": 100000,
    "device": "cuda:0",
    "fewshot_seed": 1234,
    "gen_kwargs": {},
    "limit": null,
    "model": "hf",
    "model_args": "pretrained=Qwen/Qwen3-4B-Instruct-2507",
    "model_dtype": "torch.bfloat16",
    "model_num_parameters": 4022468096,
    "model_revision": "main",
    "model_sha": "cdbee75f17c01a7cc42f958dc650907174af0554",
    "numpy_seed": 1234,
    "random_seed": 0,
    "torch_seed": 1234,
    "use_cache": "[redacted]"
  },
  "configs": {
    "arc_challenge": {
      "dataset_name": "ARC-Challenge",
      "dataset_path": "allenai/ai2_arc",
      "description": "",
      "doc_to_choice": "{{choices.text}}",
      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
      "doc_to_target": "{{choices.label.index(answerKey)}}",
      "doc_to_text": "Question: {{question}}\nAnswer:",
      "fewshot_config": {
        "doc_to_choice": "{{choices.text}}",
        "doc_to_target": "{{choices.label.index(answerKey)}}",
        "doc_to_text": "Question: {{question}}\nAnswer:",
        "fewshot_delimiter": "\n\n",
        "fewshot_indices": null,
        "gen_prefix": null,
        "process_docs": null,
        "sampler": "default",
        "samples": null,
        "split": null,
        "target_delimiter": " "
      },
      "fewshot_delimiter": "\n\n",
      "metadata": {
        "config_source": "[redacted]",
        "dtype": "bfloat16",
        "pretrained": "Qwen/Qwen3-4B-Instruct-2507",
        "trust_remote_code": true,
        "version": 1.0
      },
      "metric_list": [
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc"
        },
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc_norm"
        }
      ],
      "num_fewshot": 0,
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": true,
      "target_delimiter": " ",
      "task": "arc_challenge",
      "test_split": "test",
      "training_split": "train",
      "unsafe_code": false,
      "validation_split": "validation"
    }
  },
  "date": 1780955961.7944982,
  "eot_token_id": 151645,
  "fewshot_as_multiturn": true,
  "git_hash": null,
  "group_subtasks": {},
  "higher_is_better": {
    "arc_challenge": {
      "acc": true,
      "acc_norm": true
    }
  },
  "lm_eval_version": "0.4.12",
  "max_length": 262144,
  "model_name": "Qwen/Qwen3-4B-Instruct-2507",
  "model_name_sanitized": "Qwen/Qwen3-4B-Instruct-2507",
  "model_source": "Qwen/Qwen3-4B-Instruct-2507",
  "n-samples": {
    "arc_challenge": {
      "effective": 1172,
      "original": 1172
    }
  },
  "n-shot": {
    "arc_challenge": 0
  },
  "results": {
    "arc_challenge": {
      "acc,none": 0.4334470989761092,
      "acc_norm,none": 0.4308873720136519,
      "acc_norm_stderr,none": 0.014471133392642476,
      "acc_stderr,none": 0.014481376224558896,
      "alias": "arc_challenge",
      "name": "arc_challenge",
      "sample_len": 1172
    }
  },
  "system_instruction": null,
  "system_instruction_sha": null,
  "task_hashes": {
    "arc_challenge": "9bcbbcee5924b2d8d096f81544b8e32c71e63704d363e292090ad1ba91968f5a"
  },
  "tokenizer_bos_token": [
    null,
    "None"
  ],
  "tokenizer_eos_token": [
    "<|im_end|>",
    "151645"
  ],
  "tokenizer_pad_token": [
    "<|endoftext|>",
    "151643"
  ],
  "total_evaluation_time_seconds": "212.6989970318973",
  "transformers_version": "4.57.6",
  "upper_git_hash": null,
  "versions": {
    "arc_challenge": 1.0
  }
}