202 lines
7.4 KiB
JSON
202 lines
7.4 KiB
JSON
|
{
|
||
|
|
"results": {
|
||
|
|
"hellaswag": {
|
||
|
|
"name": "hellaswag",
|
||
|
|
"alias": "hellaswag",
|
||
|
|
"sample_len": 10042,
|
||
|
|
"acc,none": 0.26926906990639315,
|
||
|
|
"acc_stderr,none": 0.00442673471880892,
|
||
|
|
"acc_norm,none": 0.2680740888269269,
|
||
|
|
"acc_norm_stderr,none": 0.004420511215131016
|
||
|
|
},
|
||
|
|
"arc_challenge": {
|
||
|
|
"name": "arc_challenge",
|
||
|
|
"alias": "arc_challenge",
|
||
|
|
"sample_len": 1172,
|
||
|
|
"acc,none": 0.1945392491467577,
|
||
|
|
"acc_stderr,none": 0.011567709174648728,
|
||
|
|
"acc_norm,none": 0.2593856655290102,
|
||
|
|
"acc_norm_stderr,none": 0.012808273573927099
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"group_subtasks": {},
|
||
|
|
"configs": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"task": "arc_challenge",
|
||
|
|
"dataset_path": "allenai/ai2_arc",
|
||
|
|
"dataset_name": "ARC-Challenge",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"test_split": "test",
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": null,
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||
|
|
"doc_to_choice": "{{choices.text}}",
|
||
|
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "acc_norm",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": true,
|
||
|
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "jerteh/gpt2-orao",
|
||
|
|
"config_source": "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\arc\\arc_challenge.yaml"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"hellaswag": {
|
||
|
|
"task": "hellaswag",
|
||
|
|
"dataset_path": "Rowan/hellaswag",
|
||
|
|
"training_split": "train",
|
||
|
|
"validation_split": "validation",
|
||
|
|
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
|
||
|
|
"doc_to_text": "{{query}}",
|
||
|
|
"doc_to_target": "{{label}}",
|
||
|
|
"unsafe_code": false,
|
||
|
|
"doc_to_choice": "choices",
|
||
|
|
"description": "",
|
||
|
|
"target_delimiter": " ",
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"fewshot_config": {
|
||
|
|
"sampler": "default",
|
||
|
|
"split": null,
|
||
|
|
"process_docs": "<function process_docs at 0x000001BBC6DBF760>",
|
||
|
|
"fewshot_indices": null,
|
||
|
|
"samples": null,
|
||
|
|
"doc_to_text": "{{query}}",
|
||
|
|
"doc_to_choice": "choices",
|
||
|
|
"doc_to_target": "{{label}}",
|
||
|
|
"gen_prefix": null,
|
||
|
|
"fewshot_delimiter": "\n\n",
|
||
|
|
"target_delimiter": " "
|
||
|
|
},
|
||
|
|
"num_fewshot": 0,
|
||
|
|
"metric_list": [
|
||
|
|
{
|
||
|
|
"metric": "acc",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"metric": "acc_norm",
|
||
|
|
"aggregation": "mean",
|
||
|
|
"higher_is_better": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"output_type": "multiple_choice",
|
||
|
|
"repeats": 1,
|
||
|
|
"should_decontaminate": false,
|
||
|
|
"metadata": {
|
||
|
|
"version": 1.0,
|
||
|
|
"pretrained": "jerteh/gpt2-orao",
|
||
|
|
"config_source": "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\hellaswag\\hellaswag.yaml"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"versions": {
|
||
|
|
"arc_challenge": 1.0,
|
||
|
|
"hellaswag": 1.0
|
||
|
|
},
|
||
|
|
"n-shot": {
|
||
|
|
"arc_challenge": 0,
|
||
|
|
"hellaswag": 0
|
||
|
|
},
|
||
|
|
"higher_is_better": {
|
||
|
|
"arc_challenge": {
|
||
|
|
"acc": true,
|
||
|
|
"acc_norm": true
|
||
|
|
},
|
||
|
|
"hellaswag": {
|
||
|
|
"acc": true,
|
||
|
|
"acc_norm": true
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"n-samples": {
|
||
|
|
"hellaswag": {
|
||
|
|
"original": 10042,
|
||
|
|
"effective": 10042
|
||
|
|
},
|
||
|
|
"arc_challenge": {
|
||
|
|
"original": 1172,
|
||
|
|
"effective": 1172
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"config": {
|
||
|
|
"model": "hf",
|
||
|
|
"model_args": {
|
||
|
|
"pretrained": "jerteh/gpt2-orao"
|
||
|
|
},
|
||
|
|
"model_num_parameters": 772615680,
|
||
|
|
"model_dtype": "torch.float32",
|
||
|
|
"model_revision": "main",
|
||
|
|
"model_sha": "a7dfec6b4290bcb48e579e13e39d20d0ea574449",
|
||
|
|
"batch_size": "4",
|
||
|
|
"batch_sizes": [],
|
||
|
|
"device": "cuda:0",
|
||
|
|
"use_cache": null,
|
||
|
|
"limit": null,
|
||
|
|
"bootstrap_iters": 100000,
|
||
|
|
"gen_kwargs": {},
|
||
|
|
"random_seed": 0,
|
||
|
|
"numpy_seed": 1234,
|
||
|
|
"torch_seed": 1234,
|
||
|
|
"fewshot_seed": 1234
|
||
|
|
},
|
||
|
|
"git_hash": null,
|
||
|
|
"date": 1782051514.3003025,
|
||
|
|
"pretty_env_info": "PyTorch version: 2.11.0+cu130\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Microsoft Windows 11 Pro (10.0.26200 64-bit)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: N/A\n\nPython version: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 64 bit (AMD64)] (64-bit runtime)\nPython platform: Windows-10-10.0.26200-SP0\nIs CUDA available: True\nCUDA runtime version: 13.0.48\r\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 2050\nNvidia driver version: 591.86\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nName: 12th Gen Intel(R) Core(TM) i5-12450H\nManufacturer: GenuineIntel\nFamily: 205\nArchitecture: 9\nProcessorType: 3\nDeviceID: CPU0\nCurrentClockSpeed: 2000\nMaxClockSpeed: 2000\nL2CacheSize: 7168\nL2CacheSpeed: None\nRevision: None\n\nVersions of relevant libraries:\n[pip3] flash_attn==2.8.3+cu130torch2.11\n[pip3] numpy==2.2.6\n[pip3] optree==0.18.0\n[pip3] rotary-embedding-torch==0.6.4\n[pip3] torch==2.11.0+cu130\n[pip3] torchaudio==2.11.0+cu130\n[pip3] torchcrepe==0.0.20\n[pip3] torchgen==0.0.1\n[pip3] torchvision==0.26.0+cu130\n[conda] Could not collect",
|
||
|
|
"transformers_version": "5.12.0",
|
||
|
|
"lm_eval_version": "0.4.12",
|
||
|
|
"upper_git_hash": null,
|
||
|
|
"tokenizer_pad_token": [
|
||
|
|
"<|endoftext|>",
|
||
|
|
"49152"
|
||
|
|
],
|
||
|
|
"tokenizer_eos_token": [
|
||
|
|
"<|endoftext|>",
|
||
|
|
"49152"
|
||
|
|
],
|
||
|
|
"tokenizer_bos_token": [
|
||
|
|
"<|endoftext|>",
|
||
|
|
"49152"
|
||
|
|
],
|
||
|
|
"eot_token_id": 49152,
|
||
|
|
"max_length": 1024,
|
||
|
|
"task_hashes": {},
|
||
|
|
"model_source": "hf",
|
||
|
|
"model_name": "jerteh/gpt2-orao",
|
||
|
|
"model_name_sanitized": "jerteh__gpt2-orao",
|
||
|
|
"system_instruction": null,
|
||
|
|
"system_instruction_sha": null,
|
||
|
|
"fewshot_as_multiturn": null,
|
||
|
|
"chat_template": null,
|
||
|
|
"chat_template_sha": null,
|
||
|
|
"total_evaluation_time_seconds": "4028.2001800000007"
|
||
|
|
}
|