171 lines
6.9 KiB
JSON
171 lines
6.9 KiB
JSON
|
{
|
||
|
|
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
|
||
|
|
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
|
||
|
|
"config": {
|
||
|
|
"batch_size": "32",
|
||
|
|
"batch_sizes": [],
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"device": "cuda:0",
|
||
|
|
"fewshot_seed": 1234,
|
||
|
|
"gen_kwargs": {},
|
||
|
|
"limit": null,
|
||
|
|
"model": "vllm",
|
||
|
|
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
|
||
|
|
"numpy_seed": 1234,
|
||
|
|
"random_seed": 0,
|
||
|
|
"torch_seed": 1234,
|
||
|
|
"use_cache": "[redacted]"
|
||
|
|
},
|
||
|
|
"configs": {
|
||
|
|
"gsm8k": {
|
||
|
|
"dataset_name": "main",
|
||
|
|
"dataset_path": "openai/gsm8k",
|
||
|
|
"description": "",
|
||
|
|
"doc_to_target": "{{answer}}",
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"fewshot_config": {
|
||
|
|
"doc_to_choice": null,
|
||
|
|
"doc_to_target": "{{answer}}",
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"gen_prefix": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"sampler": "default",
|
||
|
|
"samples": null,
|
||
|
|
"split": "train",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_split": "train",
|
||
|
|
"filter_list": [
|
||
|
|
{
|
||
|
|
"filter": [
|
||
|
|
{
|
||
|
|
"function": "regex",
|
||
|
|
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"function": "take_first"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"name": "strict-match"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"filter": [
|
||
|
|
{
|
||
|
|
"function": "regex",
|
||
|
|
"group_select": -1,
|
||
|
|
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"function": "take_first"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"name": "flexible-extract"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"generation_kwargs": {
|
||
|
|
"do_sample": false,
|
||
|
|
"temperature": 0.0,
|
||
|
|
"until": [
|
||
|
|
"Question:",
|
||
|
|
"</s>",
|
||
|
|
"<|im_end|>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"config_source": "[redacted]",
|
||
|
|
"dtype": "bfloat16",
|
||
|
|
"gpu_memory_utilization": 0.82,
|
||
|
|
"pretrained": "[redacted]",
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"version": 3.0
|
||
|
|
},
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true,
|
||
|
|
"ignore_case": true,
|
||
|
|
"ignore_punctuation": false,
|
||
|
|
"metric": "exact_match",
|
||
|
|
"regexes_to_ignore": [
|
||
|
|
",",
|
||
|
|
"\\$",
|
||
|
|
"(?s).*#### ",
|
||
|
|
"\\.$"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"num_fewshot": 5,
|
||
|
|
"output_type": "generate_until",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"task": "gsm8k",
|
||
|
|
"test_split": "test",
|
||
|
|
"training_split": "train",
|
||
|
|
"unsafe_code": false
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"date": 1781168624.8804162,
|
||
|
|
"eot_token_id": 151645,
|
||
|
|
"fewshot_as_multiturn": true,
|
||
|
|
"git_hash": null,
|
||
|
|
"group_subtasks": {},
|
||
|
|
"higher_is_better": {
|
||
|
|
"gsm8k": {
|
||
|
|
"exact_match": true
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"lm_eval_version": "0.4.12",
|
||
|
|
"max_length": 262144,
|
||
|
|
"model_name": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"model_source": "kosa-labs/kosa-4B-it-v1",
|
||
|
|
"n-samples": {
|
||
|
|
"gsm8k": {
|
||
|
|
"effective": 1319,
|
||
|
|
"original": 1319
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"n-shot": {
|
||
|
|
"gsm8k": 5
|
||
|
|
},
|
||
|
|
"results": {
|
||
|
|
"gsm8k": {
|
||
|
|
"alias": "gsm8k",
|
||
|
|
"exact_match,flexible-extract": 0.8559514783927218,
|
||
|
|
"exact_match,strict-match": 0.8423047763457164,
|
||
|
|
"exact_match_stderr,flexible-extract": 0.009672110973065284,
|
||
|
|
"exact_match_stderr,strict-match": 0.010038901575061383,
|
||
|
|
"name": "gsm8k",
|
||
|
|
"sample_len": 1319
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"system_instruction": null,
|
||
|
|
"system_instruction_sha": null,
|
||
|
|
"task_hashes": {
|
||
|
|
"gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206"
|
||
|
|
},
|
||
|
|
"tokenizer_bos_token": [
|
||
|
|
null,
|
||
|
|
"None"
|
||
|
|
],
|
||
|
|
"tokenizer_eos_token": [
|
||
|
|
"<|im_end|>",
|
||
|
|
"151645"
|
||
|
|
],
|
||
|
|
"tokenizer_pad_token": [
|
||
|
|
"<|endoftext|>",
|
||
|
|
"151643"
|
||
|
|
],
|
||
|
|
"total_evaluation_time_seconds": "334.5368347261101",
|
||
|
|
"transformers_version": "5.11.0",
|
||
|
|
"upper_git_hash": null,
|
||
|
|
"versions": {
|
||
|
|
"gsm8k": 3.0
|
||
|
|
}
|
||
|
|
}
|