153 lines
7.6 KiB
JSON
153 lines
7.6 KiB
JSON
|
{
|
||
|
|
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
|
||
|
|
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
|
||
|
|
"config": {
|
||
|
|
"batch_size": "32",
|
||
|
|
"batch_sizes": [],
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"device": "cuda:0",
|
||
|
|
"fewshot_seed": 1234,
|
||
|
|
"gen_kwargs": {},
|
||
|
|
"limit": null,
|
||
|
|
"model": "vllm",
|
||
|
|
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
|
||
|
|
"numpy_seed": 1234,
|
||
|
|
"random_seed": 0,
|
||
|
|
"torch_seed": 1234,
|
||
|
|
"use_cache": "[redacted]"
|
||
|
|
},
|
||
|
|
"configs": {
|
||
|
|
"ifeval": {
|
||
|
|
"dataset_path": "google/IFEval",
|
||
|
|
"description": "",
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"doc_to_text": "prompt",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": null,
|
||
|
|
"doc_to_target": 0,
|
||
|
|
"doc_to_text": "prompt",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "default",
|
||
|
|
"samples": null,
|
||
|
|
"split": null,
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"generation_kwargs": {
|
||
|
|
"do_sample": false,
|
||
|
|
"max_gen_toks": 1280,
|
||
|
|
"temperature": 0.0,
|
||
|
|
"until": []
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "[redacted]",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 4.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "prompt_level_strict_acc"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "inst_level_strict_acc"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "prompt_level_loose_acc"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "inst_level_loose_acc"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "generate_until",
|
||
|
|
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "ifeval",
|
||
|
|
"test_split": "train",
|
||
|
|
"unsafe_code": false
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"date": 1781168624.8804162,
|
||
|
|
"eot_token_id": 151645,
|
||
|
|
"fewshot_as_multiturn": true,
|
||
|
|
"git_hash": null,
|
||
|
|
"group_subtasks": {},
|
||
|
|
"higher_is_better": {
|
||
|
|
"ifeval": {
|
||
|
|
"inst_level_loose_acc": true,
|
||
|
|
"inst_level_strict_acc": true,
|
||
|
|
"prompt_level_loose_acc": true,
|
||
|
|
"prompt_level_strict_acc": true
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"lm_eval_version": "0.4.12",
|
||
|
|
"max_length": 262144,
|
||
|
|
"model_name": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"model_source": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"n-samples": {
|
||
|
|
"ifeval": {
|
||
|
|
"effective": 541,
|
||
|
|
"original": 541
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"n-shot": {
|
||
|
|
"ifeval": 0
|
||
|
|
},
|
||
|
|
"results": {
|
||
|
|
"ifeval": {
|
||
|
|
"alias": "ifeval",
|
||
|
|
"inst_level_loose_acc,none": 0.9232613908872902,
|
||
|
|
"inst_level_loose_acc_stderr,none": "N/A",
|
||
|
|
"inst_level_strict_acc,none": 0.9028776978417267,
|
||
|
|
"inst_level_strict_acc_stderr,none": "N/A",
|
||
|
|
"name": "ifeval",
|
||
|
|
"prompt_level_loose_acc,none": 0.8853974121996303,
|
||
|
|
"prompt_level_loose_acc_stderr,none": 0.013707853566060982,
|
||
|
|
"prompt_level_strict_acc,none": 0.8576709796672828,
|
||
|
|
"prompt_level_strict_acc_stderr,none": 0.015035234562202742,
|
||
|
|
"sample_len": 541
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"system_instruction": null,
|
||
|
|
"system_instruction_sha": null,
|
||
|
|
"task_hashes": {
|
||
|
|
"ifeval": "e01bec2e716639c6aa89facb027f54c1a8238f533e9488cbce6f5a79ae309c3d"
|
||
|
|
},
|
||
|
|
"tokenizer_bos_token": [
|
||
|
|
null,
|
||
|
|
"None"
|
||
|
|
],
|
||
|
|
"tokenizer_eos_token": [
|
||
|
|
"<|im_end|>",
|
||
|
|
"151645"
|
||
|
|
],
|
||
|
|
"tokenizer_pad_token": [
|
||
|
|
"<|endoftext|>",
|
||
|
|
"151643"
|
||
|
|
],
|
||
|
|
"total_evaluation_time_seconds": "334.5368347261101",
|
||
|
|
"transformers_version": "5.11.0",
|
||
|
|
"upper_git_hash": null,
|
||
|
|
"versions": {
|
||
|
|
"ifeval": 4.0
|
||
|
|
}
|
||
|
|
}
|