初始化项目，由ModelHub XC社区提供模型
Model: kosa-labs/kosa-4B-it-v1 Source: Original Platform
This commit is contained in:
ModelHub XC
170
benchmarks/gsm8k_kosa.json
Normal file
170
benchmarks/gsm8k_kosa.json
Normal file
@@ -0,0 +1,170 @@
|
||||
{
|
||||
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
|
||||
"chat_template_sha": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326",
|
||||
"config": {
|
||||
"batch_size": "32",
|
||||
"batch_sizes": [],
|
||||
"bootstrap_iters": 100000,
|
||||
"device": "cuda:0",
|
||||
"fewshot_seed": 1234,
|
||||
"gen_kwargs": {},
|
||||
"limit": null,
|
||||
"model": "vllm",
|
||||
"model_args": "pretrained=kosa-labs/kosa-4B-it-v1",
|
||||
"numpy_seed": 1234,
|
||||
"random_seed": 0,
|
||||
"torch_seed": 1234,
|
||||
"use_cache": "[redacted]"
|
||||
},
|
||||
"configs": {
|
||||
"gsm8k": {
|
||||
"dataset_name": "main",
|
||||
"dataset_path": "openai/gsm8k",
|
||||
"description": "",
|
||||
"doc_to_target": "{{answer}}",
|
||||
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||
"fewshot_config": {
|
||||
"doc_to_choice": null,
|
||||
"doc_to_target": "{{answer}}",
|
||||
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||
"fewshot_delimiter": "\n\n",
|
||||
"fewshot_indices": null,
|
||||
"gen_prefix": null,
|
||||
"process_docs": null,
|
||||
"sampler": "default",
|
||||
"samples": null,
|
||||
"split": "train",
|
||||
"target_delimiter": " "
|
||||
},
|
||||
"fewshot_delimiter": "\n\n",
|
||||
"fewshot_split": "train",
|
||||
"filter_list": [
|
||||
{
|
||||
"filter": [
|
||||
{
|
||||
"function": "regex",
|
||||
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
|
||||
},
|
||||
{
|
||||
"function": "take_first"
|
||||
}
|
||||
],
|
||||
"name": "strict-match"
|
||||
},
|
||||
{
|
||||
"filter": [
|
||||
{
|
||||
"function": "regex",
|
||||
"group_select": -1,
|
||||
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
|
||||
},
|
||||
{
|
||||
"function": "take_first"
|
||||
}
|
||||
],
|
||||
"name": "flexible-extract"
|
||||
}
|
||||
],
|
||||
"generation_kwargs": {
|
||||
"do_sample": false,
|
||||
"temperature": 0.0,
|
||||
"until": [
|
||||
"Question:",
|
||||
"</s>",
|
||||
"<|im_end|>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"config_source": "[redacted]",
|
||||
"dtype": "bfloat16",
|
||||
"gpu_memory_utilization": 0.82,
|
||||
"pretrained": "[redacted]",
|
||||
"tensor_parallel_size": 1,
|
||||
"trust_remote_code": true,
|
||||
"version": 3.0
|
||||
},
|
||||
"metric_list": [
|
||||
{
|
||||
"aggregation": "mean",
|
||||
"higher_is_better": true,
|
||||
"ignore_case": true,
|
||||
"ignore_punctuation": false,
|
||||
"metric": "exact_match",
|
||||
"regexes_to_ignore": [
|
||||
",",
|
||||
"\\$",
|
||||
"(?s).*#### ",
|
||||
"\\.$"
|
||||
]
|
||||
}
|
||||
],
|
||||
"num_fewshot": 5,
|
||||
"output_type": "generate_until",
|
||||
"repeats": 1,
|
||||
"should_decontaminate": false,
|
||||
"target_delimiter": " ",
|
||||
"task": "gsm8k",
|
||||
"test_split": "test",
|
||||
"training_split": "train",
|
||||
"unsafe_code": false
|
||||
}
|
||||
},
|
||||
"date": 1781168624.8804162,
|
||||
"eot_token_id": 151645,
|
||||
"fewshot_as_multiturn": true,
|
||||
"git_hash": null,
|
||||
"group_subtasks": {},
|
||||
"higher_is_better": {
|
||||
"gsm8k": {
|
||||
"exact_match": true
|
||||
}
|
||||
},
|
||||
"lm_eval_version": "0.4.12",
|
||||
"max_length": 262144,
|
||||
"model_name": "kosa-labs/kosa-4B-it-v1",
|
||||
"model_name_sanitized": "kosa-labs/kosa-4B-it-v1",
|
||||
"model_source": "kosa-labs/kosa-4B-it-v1",
|
||||
"n-samples": {
|
||||
"gsm8k": {
|
||||
"effective": 1319,
|
||||
"original": 1319
|
||||
}
|
||||
},
|
||||
"n-shot": {
|
||||
"gsm8k": 5
|
||||
},
|
||||
"results": {
|
||||
"gsm8k": {
|
||||
"alias": "gsm8k",
|
||||
"exact_match,flexible-extract": 0.8559514783927218,
|
||||
"exact_match,strict-match": 0.8423047763457164,
|
||||
"exact_match_stderr,flexible-extract": 0.009672110973065284,
|
||||
"exact_match_stderr,strict-match": 0.010038901575061383,
|
||||
"name": "gsm8k",
|
||||
"sample_len": 1319
|
||||
}
|
||||
},
|
||||
"system_instruction": null,
|
||||
"system_instruction_sha": null,
|
||||
"task_hashes": {
|
||||
"gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206"
|
||||
},
|
||||
"tokenizer_bos_token": [
|
||||
null,
|
||||
"None"
|
||||
],
|
||||
"tokenizer_eos_token": [
|
||||
"<|im_end|>",
|
||||
"151645"
|
||||
],
|
||||
"tokenizer_pad_token": [
|
||||
"<|endoftext|>",
|
||||
"151643"
|
||||
],
|
||||
"total_evaluation_time_seconds": "334.5368347261101",
|
||||
"transformers_version": "5.11.0",
|
||||
"upper_git_hash": null,
|
||||
"versions": {
|
||||
"gsm8k": 3.0
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user