135 lines
6.2 KiB
JSON
135 lines
6.2 KiB
JSON
|
{
|
||
|
|
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
|
||
|
|
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
|
||
|
|
"config": {
|
||
|
|
"batch_size": "32",
|
||
|
|
"batch_sizes": [],
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"device": "cuda:0",
|
||
|
|
"fewshot_seed": 1234,
|
||
|
|
"gen_kwargs": {},
|
||
|
|
"limit": null,
|
||
|
|
"model": "vllm",
|
||
|
|
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
|
||
|
|
"numpy_seed": 1234,
|
||
|
|
"random_seed": 0,
|
||
|
|
"torch_seed": 1234,
|
||
|
|
"use_cache": "[redacted]"
|
||
|
|
},
|
||
|
|
"configs": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"dataset_name": "ARC-Challenge",
|
||
|
|
"dataset_path": "allenai/ai2_arc",
|
||
|
|
"description": "",
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "default",
|
||
|
|
"samples": null,
|
||
|
|
"split": null,
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "[redacted]",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 1.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"metric": "acc_norm"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "arc_challenge",
|
||
|
|
"test_split": "test",
|
||
|
|
"training_split": "train",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"validation_split": "validation"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"date": 1781168624.8804162,
|
||
|
|
"eot_token_id": 151645,
|
||
|
|
"fewshot_as_multiturn": true,
|
||
|
|
"git_hash": null,
|
||
|
|
"group_subtasks": {},
|
||
|
|
"higher_is_better": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"acc": true,
|
||
|
|
"acc_norm": true
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"lm_eval_version": "0.4.12",
|
||
|
|
"max_length": 262144,
|
||
|
|
"model_name": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"model_source": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"n-samples": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"effective": 1172,
|
||
|
|
"original": 1172
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"n-shot": {
|
||
|
|
"arc_challenge": 0
|
||
|
|
},
|
||
|
|
"results": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"acc,none": 0.5315699658703071,
|
||
|
|
"acc_norm,none": 0.5213310580204779,
|
||
|
|
"acc_norm_stderr,none": 0.014598087973127125,
|
||
|
|
"acc_stderr,none": 0.014582236460866984,
|
||
|
|
"alias": "arc_challenge",
|
||
|
|
"name": "arc_challenge",
|
||
|
|
"sample_len": 1172
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"system_instruction": null,
|
||
|
|
"system_instruction_sha": null,
|
||
|
|
"task_hashes": {
|
||
|
|
"arc_challenge": "9bcbbcee5924b2d8d096f81544b8e32c71e63704d363e292090ad1ba91968f5a"
|
||
|
|
},
|
||
|
|
"tokenizer_bos_token": [
|
||
|
|
null,
|
||
|
|
"None"
|
||
|
|
],
|
||
|
|
"tokenizer_eos_token": [
|
||
|
|
"<|im_end|>",
|
||
|
|
"151645"
|
||
|
|
],
|
||
|
|
"tokenizer_pad_token": [
|
||
|
|
"<|endoftext|>",
|
||
|
|
"151643"
|
||
|
|
],
|
||
|
|
"total_evaluation_time_seconds": "334.5368347261101",
|
||
|
|
"transformers_version": "5.11.0",
|
||
|
|
"upper_git_hash": null,
|
||
|
|
"versions": {
|
||
|
|
"arc_challenge": 1.0
|
||
|
|
}
|
||
|
|
}
|