{
  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = '' %}\n    {%- endif %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
  "chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
  "config": {
    "batch_size": "32",
    "batch_sizes": [],
    "bootstrap_iters": 100000,
    "device": "cuda:0",
    "fewshot_seed": 1234,
    "gen_kwargs": {},
    "limit": null,
    "model": "vllm",
    "model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
    "numpy_seed": 1234,
    "random_seed": 0,
    "torch_seed": 1234,
    "use_cache": "[redacted]"
  },
  "configs": {
    "gsm8k": {
      "dataset_name": "main",
      "dataset_path": "openai/gsm8k",
      "description": "",
      "doc_to_target": "{{answer}}",
      "doc_to_text": "Question: {{question}}\nAnswer:",
      "fewshot_config": {
        "doc_to_choice": null,
        "doc_to_target": "{{answer}}",
        "doc_to_text": "Question: {{question}}\nAnswer:",
        "fewshot_delimiter": "\n\n",
        "fewshot_indices": null,
        "gen_prefix": null,
        "process_docs": null,
        "sampler": "default",
        "samples": null,
        "split": "train",
        "target_delimiter": " "
      },
      "fewshot_delimiter": "\n\n",
      "fewshot_split": "train",
      "filter_list": [
        {
          "filter": [
            {
              "function": "regex",
              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
            },
            {
              "function": "take_first"
            }
          ],
          "name": "strict-match"
        },
        {
          "filter": [
            {
              "function": "regex",
              "group_select": -1,
              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
            },
            {
              "function": "take_first"
            }
          ],
          "name": "flexible-extract"
        }
      ],
      "generation_kwargs": {
        "do_sample": false,
        "temperature": 0.0,
        "until": [
          "Question:",
          "</s>",
          "<|im_end|>"
        ]
      },
      "metadata": {
        "config_source": "[redacted]",
        "dtype": "bfloat16",
        "gpu_memory_utilization": 0.82,
        "pretrained": "[redacted]",
        "tensor_parallel_size": 1,
        "trust_remote_code": true,
        "version": 3.0
      },
      "metric_list": [
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "ignore_case": true,
          "ignore_punctuation": false,
          "metric": "exact_match",
          "regexes_to_ignore": [
            ",",
            "\\$",
            "(?s).*#### ",
            "\\.$"
          ]
        }
      ],
      "num_fewshot": 5,
      "output_type": "generate_until",
      "repeats": 1,
      "should_decontaminate": false,
      "target_delimiter": " ",
      "task": "gsm8k",
      "test_split": "test",
      "training_split": "train",
      "unsafe_code": false
    }
  },
  "date": 1781168624.8804162,
  "eot_token_id": 151645,
  "fewshot_as_multiturn": true,
  "git_hash": null,
  "group_subtasks": {},
  "higher_is_better": {
    "gsm8k": {
      "exact_match": true
    }
  },
  "lm_eval_version": "0.4.12",
  "max_length": 262144,
  "model_name": "kosa-labs/kosa-4B-it-v1",
  "model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
  "model_source": "kosa-labs/kosa-4B-it-v1",
  "n-samples": {
    "gsm8k": {
      "effective": 1319,
      "original": 1319
    }
  },
  "n-shot": {
    "gsm8k": 5
  },
  "results": {
    "gsm8k": {
      "alias": "gsm8k",
      "exact_match,flexible-extract": 0.8559514783927218,
      "exact_match,strict-match": 0.8423047763457164,
      "exact_match_stderr,flexible-extract": 0.009672110973065284,
      "exact_match_stderr,strict-match": 0.010038901575061383,
      "name": "gsm8k",
      "sample_len": 1319
    }
  },
  "system_instruction": null,
  "system_instruction_sha": null,
  "task_hashes": {
    "gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206"
  },
  "tokenizer_bos_token": [
    null,
    "None"
  ],
  "tokenizer_eos_token": [
    "<|im_end|>",
    "151645"
  ],
  "tokenizer_pad_token": [
    "<|endoftext|>",
    "151643"
  ],
  "total_evaluation_time_seconds": "334.5368347261101",
  "transformers_version": "5.11.0",
  "upper_git_hash": null,
  "versions": {
    "gsm8k": 3.0
  }
}