lanawwas/ALLaM-7B-Instruct-preview
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: lanawwas/ALLaM-7B-Instruct-preview
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-22 10:54:04 +08:00
commit 3e4c694337
466 changed files with 259661 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1136
evaluations/en/AceGPT-v2-32B-Chat/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

125
evaluations/en/AceGPT-v2-32B-Chat/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,125 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.5179180887372014,
"acc_stderr,none": 0.014602005585490971,
"acc_norm,none": 0.5392491467576792,
"acc_norm_stderr,none": 0.014566303676636586
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "hf",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
"model_num_parameters": 32512545792,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
"batch_size": "auto",
"batch_sizes": [
64
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737972876.8138564,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {
"arc_challenge": "09f9ae87a0905d63512cffc4aa91a55e44258fc35160e40fa1eb66fb75473e34"
},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 1683130.71663661,
"end_time": 1683230.116914329,
"total_evaluation_time_seconds": "99.40027771890163"
}

125
evaluations/en/AceGPT-v2-32B-Chat/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,125 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.328125,
"acc_stderr,none": 0.0222080353262888,
"acc_norm,none": 0.328125,
"acc_norm_stderr,none": 0.0222080353262888
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "hf",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 32512545792,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "b955b2950",
"date": 1739796947.9720185,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {
"gpqa_main_n_shot": "a3483bbbe2e4b606b3eccce05ccdbeeebe27c393296c82d64bf645fff6aed3ff"
},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 413228.20145324,
"end_time": 415139.438325981,
"total_evaluation_time_seconds": "1911.2368727410212"
}

153
evaluations/en/AceGPT-v2-32B-Chat/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,153 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.7869598180439727,
"exact_match_stderr,strict-match": 0.011278447856900771,
"exact_match,flexible-extract": 0.7952994692949203,
"exact_match_stderr,flexible-extract": 0.011113916396062962
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,tensor_parallel_size=2,data_parallel_size=4,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737583211.3834355,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {},
"model_source": "vllm",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 111293.791044811,
"end_time": 111435.003001496,
"total_evaluation_time_seconds": "141.2119566850015"
}

126
evaluations/en/AceGPT-v2-32B-Chat/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,126 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.645488946425015,
"acc_stderr,none": 0.004773872456201065,
"acc_norm,none": 0.8329018123879706,
"acc_norm_stderr,none": 0.0037230107458785114
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "hf",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 32512545792,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
"batch_size": "auto",
"batch_sizes": [
64
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737896278.0364246,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {
"hellaswag": "f3c11b39766a06b6c303d8176d8f35fc9c3026e524aee7b9aaa946c35951cde8"
},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 6712.201821225,
"end_time": 7280.43429144,
"total_evaluation_time_seconds": "568.2324702150008"
}

319
evaluations/en/AceGPT-v2-32B-Chat/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,319 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.6149292149292149,
"acc_stderr,none": 0.00780806172478048
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.5433815350389322,
"acc_stderr,none": 0.00830767934735274
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.6368343195266272,
"acc_stderr,none": 0.009250018627925967
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.6283277870216306,
"acc_stderr,none": 0.006970053615681693
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.8878391959798995,
"acc_stderr,none": 0.004474400177505811
}
},
"group_subtasks": {
"ethics_deontology": [],
"ethics_virtue": [],
"ethics_cm": [],
"ethics_utilitarianism": [],
"ethics_justice": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
},
"ethics_cm": {
"original": 3885,
"effective": 3885
},
"ethics_virtue": {
"original": 4975,
"effective": 4975
},
"ethics_deontology": {
"original": 3596,
"effective": 3596
}
},
"config": {
"model": "hf",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
"model_num_parameters": 32512545792,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
"batch_size": "auto",
"batch_sizes": [
8
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737973124.5927782,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {
"ethics_justice": "29e70305fd625a6fa42aa154ef0c4fcd7ffbfce91483485d61ef01ebaab02235",
"ethics_utilitarianism": "50e3b75384c265c6c5fb9691f46a46b22a44ffb07d131e285b5f0a84b1025bc8",
"ethics_cm": "088ead6c08bb523b9de2bf5098b07ad2d484b8d19d068937634e20e4a776db84",
"ethics_virtue": "b3e6efc9b8e5a591f9e9bd96c14a97d118c29455f4441e52d97b10b404513a55",
"ethics_deontology": "5311ba877c2291b107da9263731e4895484636a7fdce77b31855eb34cc6c2a37"
},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 1683378.388609929,
"end_time": 1683984.191104153,
"total_evaluation_time_seconds": "605.8024942239281"
}

132
evaluations/en/AceGPT-v2-32B-Chat/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,132 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.2754158964879852,
"prompt_level_strict_acc_stderr,none": 0.019223923196242006,
"inst_level_strict_acc,none": 0.4088729016786571,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.3364140480591497,
"prompt_level_loose_acc_stderr,none": 0.020332406004701264,
"inst_level_loose_acc,none": 0.46882494004796166,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,tensor_parallel_size=2,data_parallel_size=4,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737582090.0582705,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {},
"model_source": "vllm",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 110172.444165653,
"end_time": 110319.072051442,
"total_evaluation_time_seconds": "146.62788578899927"
}

521
evaluations/en/AceGPT-v2-32B-Chat/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,521 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.328,
"exact_match_stderr,none": 0.006239030429451531,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.4818871103622578,
"exact_match_stderr,none": 0.014509167981143361
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.2911392405063291,
"exact_match_stderr,none": 0.020888164059267196
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.2651356993736952,
"exact_match_stderr,none": 0.02018941478172901
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.14396456256921372,
"exact_match_stderr,none": 0.011688812818875677
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.2111111111111111,
"exact_match_stderr,none": 0.017577984727516007
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.5510907003444316,
"exact_match_stderr,none": 0.01686285928831101
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.1446886446886447,
"exact_match_stderr,none": 0.015068884082729252
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.328,
"exact_match_stderr,none": 0.006239030429451531,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a7faea9750>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a7faea76d0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a7faea4790>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a7fbbb1b40>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a7fbbb09d0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a7fbb1ea70>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a7fc249240>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,tensor_parallel_size=2,data_parallel_size=4,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737581383.6780143,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {},
"model_source": "vllm",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 109466.080707565,
"end_time": 109890.218887646,
"total_evaluation_time_seconds": "424.138180081005"
}

3289
evaluations/en/AceGPT-v2-32B-Chat/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

1103
evaluations/en/AceGPT-v2-32B-Chat/mmlu_pro_5_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

128
evaluations/en/AceGPT-v2-32B-Chat/triviaqa_5_shot.json Normal file
View File

@@ -0,0 +1,128 @@
{
"results": {
"triviaqa": {
"alias": "triviaqa",
"exact_match,remove_whitespace": 0.6945497102095408,
"exact_match_stderr,remove_whitespace": 0.0034385426018490157
}
},
"group_subtasks": {
"triviaqa": []
},
"configs": {
"triviaqa": {
"task": "triviaqa",
"dataset_path": "trivia_qa",
"dataset_name": "rc.nocontext",
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "Question: {{question}}?\nAnswer:",
"doc_to_target": "{{answer.aliases}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"\n",
".",
","
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "remove_whitespace",
"filter": [
{
"function": "remove_whitespace"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 3.0
}
}
},
"versions": {
"triviaqa": 3.0
},
"n-shot": {
"triviaqa": 5
},
"higher_is_better": {
"triviaqa": {
"exact_match": true
}
},
"n-samples": {
"triviaqa": {
"original": 17944,
"effective": 17944
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,tensor_parallel_size=2,data_parallel_size=4,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737580930.105174,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {},
"model_source": "vllm",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 109012.375283453,
"end_time": 109308.798750485,
"total_evaluation_time_seconds": "296.4234670320002"
}

116
evaluations/en/AceGPT-v2-32B-Chat/truthfulqa_mc2_0_shot.json Normal file
View File

@@ -0,0 +1,116 @@
{
"results": {
"truthfulqa_mc2": {
"alias": "truthfulqa_mc2",
"acc,none": 0.5917866931851031,
"acc_stderr,none": 0.015068975512501583
}
},
"group_subtasks": {
"truthfulqa_mc2": []
},
"configs": {
"truthfulqa_mc2": {
"task": "truthfulqa_mc2",
"tag": [
"truthfulqa"
],
"dataset_path": "truthful_qa",
"dataset_name": "multiple_choice",
"validation_split": "validation",
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
"doc_to_target": 0,
"doc_to_choice": "{{mc2_targets.choices}}",
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 2.0
}
}
},
"versions": {
"truthfulqa_mc2": 2.0
},
"n-shot": {
"truthfulqa_mc2": 0
},
"higher_is_better": {
"truthfulqa_mc2": {
"acc": true
}
},
"n-samples": {
"truthfulqa_mc2": {
"original": 817,
"effective": 817
}
},
"config": {
"model": "hf",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
"model_num_parameters": 32512545792,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
"batch_size": "auto",
"batch_sizes": [
64
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737973862.8433588,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {
"truthfulqa_mc2": "a84d12f632c7780645b884ce110adebc1f8277817f5cf11484c396efe340e882"
},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 1684116.84150855,
"end_time": 1684487.429520878,
"total_evaluation_time_seconds": "370.58801232790574"
}

116
evaluations/en/AceGPT-v2-32B-Chat/winogrande_0_shot.json Normal file
View File

@@ -0,0 +1,116 @@
{
"results": {
"winogrande": {
"alias": "winogrande",
"acc,none": 0.7916337805840569,
"acc_stderr,none": 0.011414554399987741
}
},
"group_subtasks": {
"winogrande": []
},
"configs": {
"winogrande": {
"task": "winogrande",
"dataset_path": "winogrande",
"dataset_name": "winogrande_xl",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "sentence",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"winogrande": 1.0
},
"n-shot": {
"winogrande": 0
},
"higher_is_better": {
"winogrande": {
"acc": true
}
},
"n-samples": {
"winogrande": {
"original": 1267,
"effective": 1267
}
},
"config": {
"model": "hf",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 32512545792,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
"batch_size": "auto",
"batch_sizes": [
64
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737893686.1748393,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"151643"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 151643,
"max_length": 32768,
"task_hashes": {
"winogrande": "2ad49ed9c32e5a093513b5bf67c7da0e586ad24e6c1a2839c2a00bb5bbd55c85"
},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 4120.397054559,
"end_time": 6650.279180562,
"total_evaluation_time_seconds": "2529.882126003"
}

1108
evaluations/en/AceGPT-v2-8B-Chat/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

121
evaluations/en/AceGPT-v2-8B-Chat/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,121 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.5264505119453925,
"acc_stderr,none": 0.014590931358120172,
"acc_norm,none": 0.5349829351535836,
"acc_norm_stderr,none": 0.014575583922019667
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457305.6782017,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 934793.053771435,
"end_time": 935373.4405872,
"total_evaluation_time_seconds": "580.3868157649413"
}

123
evaluations/en/AceGPT-v2-8B-Chat/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,123 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.25669642857142855,
"acc_stderr,none": 0.020660425491724695,
"acc_norm,none": 0.25669642857142855,
"acc_norm_stderr,none": 0.020660425491724695
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732096631.7343132,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 8414.073662303,
"end_time": 8890.174062302,
"total_evaluation_time_seconds": "476.1003999989989"
}

157
evaluations/en/AceGPT-v2-8B-Chat/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,157 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.5686125852918877,
"exact_match_stderr,strict-match": 0.013642195352511571,
"exact_match,flexible-extract": 0.5708870356330553,
"exact_match_stderr,flexible-extract": 0.01363336942564724
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457285.5259154,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 934772.957176889,
"end_time": 941452.488443649,
"total_evaluation_time_seconds": "6679.531266760081"
}

122
evaluations/en/AceGPT-v2-8B-Chat/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,122 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.6086436964748058,
"acc_stderr,none": 0.004870563921220627,
"acc_norm,none": 0.7920732921728739,
"acc_norm_stderr,none": 0.004049947000889764
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457282.163765,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 937618.837620989,
"end_time": 939731.337945906,
"total_evaluation_time_seconds": "2112.500324917026"
}

307
evaluations/en/AceGPT-v2-8B-Chat/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,307 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.6244530244530244,
"acc_stderr,none": 0.007770382729389901
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.5984427141268076,
"acc_stderr,none": 0.008175900541354739
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.6856508875739645,
"acc_stderr,none": 0.008929653715581846
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.6191763727121464,
"acc_stderr,none": 0.007003773124794958
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.8793969849246231,
"acc_stderr,none": 0.0046176251872955725
}
},
"group_subtasks": {
"ethics_cm": [],
"ethics_deontology": [],
"ethics_utilitarianism": [],
"ethics_justice": [],
"ethics_virtue": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_virtue": {
"original": 4975,
"effective": 4975
},
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
},
"ethics_deontology": {
"original": 3596,
"effective": 3596
},
"ethics_cm": {
"original": 3885,
"effective": 3885
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.4,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735751872.733654,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "vllm",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 12157.959493773,
"end_time": 12394.614153199,
"total_evaluation_time_seconds": "236.65465942599985"
}

132
evaluations/en/AceGPT-v2-8B-Chat/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,132 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.23475046210720887,
"prompt_level_strict_acc_stderr,none": 0.018239288213433787,
"inst_level_strict_acc,none": 0.32973621103117506,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.27171903881700554,
"prompt_level_loose_acc_stderr,none": 0.01914311609959402,
"inst_level_loose_acc,none": 0.3669064748201439,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.4,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735753816.3503323,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "vllm",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 14101.634559681,
"end_time": 14173.619575398,
"total_evaluation_time_seconds": "71.98501571699853"
}

525
evaluations/en/AceGPT-v2-8B-Chat/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,525 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.1758,
"exact_match_stderr,none": 0.005170915337066609,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.2670598146588037,
"exact_match_stderr,none": 0.012846836411288906
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.15611814345991562,
"exact_match_stderr,none": 0.01668925473342588
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.1315240083507307,
"exact_match_stderr,none": 0.015458504556847509
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.04983388704318937,
"exact_match_stderr,none": 0.007245341858973181
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.0962962962962963,
"exact_match_stderr,none": 0.012706426844176376
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.3340987370838117,
"exact_match_stderr,none": 0.015991260938213656
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.06776556776556776,
"exact_match_stderr,none": 0.010766359056008468
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.1758,
"exact_match_stderr,none": 0.005170915337066609,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148f3175b7f0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148f31759870>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148f30f825f0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148f30f81fc0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148f31792c20>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148f31c3ff40>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148f32ade440>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457279.5400486,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 934767.019303019,
"end_time": 971111.469964088,
"total_evaluation_time_seconds": "36344.450661069015"
}

3283
evaluations/en/AceGPT-v2-8B-Chat/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

1092
evaluations/en/AceGPT-v2-8B-Chat/mmlu_pro_5_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

132
evaluations/en/AceGPT-v2-8B-Chat/triviaqa_5_shot.json Normal file
View File

@@ -0,0 +1,132 @@
{
"results": {
"triviaqa": {
"alias": "triviaqa",
"exact_match,remove_whitespace": 0.6764935354436024,
"exact_match_stderr,remove_whitespace": 0.003492414467248401
}
},
"group_subtasks": {
"triviaqa": []
},
"configs": {
"triviaqa": {
"task": "triviaqa",
"dataset_path": "trivia_qa",
"dataset_name": "rc.nocontext",
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "Question: {{question}}?\nAnswer:",
"doc_to_target": "{{answer.aliases}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"\n",
".",
","
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "remove_whitespace",
"filter": [
{
"function": "remove_whitespace"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 3.0
}
}
},
"versions": {
"triviaqa": 3.0
},
"n-shot": {
"triviaqa": 5
},
"higher_is_better": {
"triviaqa": {
"exact_match": true
}
},
"n-samples": {
"triviaqa": {
"original": 17944,
"effective": 17944
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732530416.4028962,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 876731.027243315,
"end_time": 880169.77139674,
"total_evaluation_time_seconds": "3438.744153424981"
}

112
evaluations/en/AceGPT-v2-8B-Chat/truthfulqa_mc2_0_shot.json Normal file
View File

@@ -0,0 +1,112 @@
{
"results": {
"truthfulqa_mc2": {
"alias": "truthfulqa_mc2",
"acc,none": 0.5520106526990918,
"acc_stderr,none": 0.015258721249238388
}
},
"group_subtasks": {
"truthfulqa_mc2": []
},
"configs": {
"truthfulqa_mc2": {
"task": "truthfulqa_mc2",
"tag": [
"truthfulqa"
],
"dataset_path": "truthful_qa",
"dataset_name": "multiple_choice",
"validation_split": "validation",
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
"doc_to_target": 0,
"doc_to_choice": "{{mc2_targets.choices}}",
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 2.0
}
}
},
"versions": {
"truthfulqa_mc2": 2.0
},
"n-shot": {
"truthfulqa_mc2": 0
},
"higher_is_better": {
"truthfulqa_mc2": {
"acc": true
}
},
"n-samples": {
"truthfulqa_mc2": {
"original": 817,
"effective": 817
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457284.7916152,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 937621.506371343,
"end_time": 938295.585706235,
"total_evaluation_time_seconds": "674.0793348919833"
}

112
evaluations/en/AceGPT-v2-8B-Chat/winogrande_0_shot.json Normal file
View File

@@ -0,0 +1,112 @@
{
"results": {
"winogrande": {
"alias": "winogrande",
"acc,none": 0.7371744277821626,
"acc_stderr,none": 0.012370922527262008
}
},
"group_subtasks": {
"winogrande": []
},
"configs": {
"winogrande": {
"task": "winogrande",
"dataset_path": "winogrande",
"dataset_name": "winogrande_xl",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "sentence",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"winogrande": 1.0
},
"n-shot": {
"winogrande": 0
},
"higher_is_better": {
"winogrande": {
"acc": true
}
},
"n-samples": {
"winogrande": {
"original": 1267,
"effective": 1267
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 8030261248,
"model_dtype": "torch.float16",
"model_revision": "main",
"model_sha": "562d0998c03c02d315e346f81650a43955711901",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457295.7930105,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_eos_token": [
"<|end_of_text|>",
"128001"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128001,
"max_length": 8192,
"task_hashes": {},
"model_source": "hf",
"model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
"model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 934783.15582321,
"end_time": 935295.980413407,
"total_evaluation_time_seconds": "512.8245901969494"
}

1108
evaluations/en/Allam-7b-instruct-preview/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

117
evaluations/en/Allam-7b-instruct-preview/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,117 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.5127986348122867,
"acc_stderr,none": 0.014606603181012541,
"acc_norm,none": 0.5127986348122867,
"acc_norm_stderr,none": 0.014606603181012538
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735958479.5122433,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.90\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 25148.877885035,
"end_time": 25235.270896756,
"total_evaluation_time_seconds": "86.39301172100022"
}

121
evaluations/en/Allam-7b-instruct-preview/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,121 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.22767857142857142,
"acc_stderr,none": 0.0198338196436619,
"acc_norm,none": 0.22767857142857142,
"acc_norm_stderr,none": 0.0198338196436619
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737961176.7588274,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {
"gpqa_main_n_shot": "4a64f5415ed03d5c5fec2b22dd8bfd718011928a30847c5b126c837aaf0c0619"
},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 330039.670361117,
"end_time": 330095.888966536,
"total_evaluation_time_seconds": "56.21860541898059"
}

153
evaluations/en/Allam-7b-instruct-preview/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,153 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.6178923426838514,
"exact_match_stderr,strict-match": 0.013384173935648495,
"exact_match,flexible-extract": 0.6224412433661866,
"exact_match_stderr,flexible-extract": 0.013353150666358532
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737546137.8667536,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 23682.650060164,
"end_time": 23828.827645231,
"total_evaluation_time_seconds": "146.1775850669983"
}

118
evaluations/en/Allam-7b-instruct-preview/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,118 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.5771758613821948,
"acc_stderr,none": 0.00492998369279507,
"acc_norm,none": 0.7625970922127067,
"acc_norm_stderr,none": 0.0042462162299898715
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735957117.4813576,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.90\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 23786.943776673,
"end_time": 23998.958401018,
"total_evaluation_time_seconds": "212.0146243449999"
}

307
evaluations/en/Allam-7b-instruct-preview/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,307 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.7392535392535392,
"acc_stderr,none": 0.007044761695158352
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.5786985539488321,
"acc_stderr,none": 0.00823518246369769
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.771819526627219,
"acc_stderr,none": 0.00807186884011459
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.6541181364392679,
"acc_stderr,none": 0.006860486742815242
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.9147738693467337,
"acc_stderr,none": 0.003959044383441912
}
},
"group_subtasks": {
"ethics_deontology": [],
"ethics_virtue": [],
"ethics_cm": [],
"ethics_utilitarianism": [],
"ethics_justice": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
},
"ethics_cm": {
"original": 3885,
"effective": 3885
},
"ethics_virtue": {
"original": 4975,
"effective": 4975
},
"ethics_deontology": {
"original": 3596,
"effective": 3596
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735957382.509422,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.90\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 24051.95882374,
"end_time": 24251.353762318,
"total_evaluation_time_seconds": "199.3949385779997"
}

132
evaluations/en/Allam-7b-instruct-preview/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,132 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.3807763401109057,
"prompt_level_strict_acc_stderr,none": 0.020895937888190833,
"inst_level_strict_acc,none": 0.5,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.4214417744916821,
"prompt_level_loose_acc_stderr,none": 0.021249340085831084,
"inst_level_loose_acc,none": 0.5407673860911271,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737545156.5536008,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 22701.50615791,
"end_time": 22785.243168339,
"total_evaluation_time_seconds": "83.73701042899847"
}

521
evaluations/en/Allam-7b-instruct-preview/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,521 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.173,
"exact_match_stderr,none": 0.005146622162421542,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.2409435551811289,
"exact_match_stderr,none": 0.012418019817467794
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.17088607594936708,
"exact_match_stderr,none": 0.01730732195419626
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.12108559498956159,
"exact_match_stderr,none": 0.014921262921998898
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.053156146179401995,
"exact_match_stderr,none": 0.00746986334739643
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.11296296296296296,
"exact_match_stderr,none": 0.013634666880074295
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.34328358208955223,
"exact_match_stderr,none": 0.01609740338728602
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.05860805860805861,
"exact_match_stderr,none": 0.010061567725278785
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.173,
"exact_match_stderr,none": 0.005146622162421542,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148513aa0f70>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148510eeaef0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148510ee8ca0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148510ee17e0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148510ee15a0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148510e02b90>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x148516fe49d0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737544396.9634442,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 21941.885116993,
"end_time": 22486.922181144,
"total_evaluation_time_seconds": "545.0370641510017"
}

3289
evaluations/en/Allam-7b-instruct-preview/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

1103
evaluations/en/Allam-7b-instruct-preview/mmlu_pro_5_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

128
evaluations/en/Allam-7b-instruct-preview/triviaqa_5_shot.json Normal file
View File

@@ -0,0 +1,128 @@
{
"results": {
"triviaqa": {
"alias": "triviaqa",
"exact_match,remove_whitespace": 0.16066651805617477,
"exact_match_stderr,remove_whitespace": 0.002741463299754975
}
},
"group_subtasks": {
"triviaqa": []
},
"configs": {
"triviaqa": {
"task": "triviaqa",
"dataset_path": "trivia_qa",
"dataset_name": "rc.nocontext",
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "Question: {{question}}?\nAnswer:",
"doc_to_target": "{{answer.aliases}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"\n",
".",
","
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "remove_whitespace",
"filter": [
{
"function": "remove_whitespace"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 3.0
}
}
},
"versions": {
"triviaqa": 3.0
},
"n-shot": {
"triviaqa": 5
},
"higher_is_better": {
"triviaqa": {
"exact_match": true
}
},
"n-samples": {
"triviaqa": {
"original": 17944,
"effective": 17944
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737544037.6055677,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 21582.583321473,
"end_time": 21855.449312492,
"total_evaluation_time_seconds": "272.8659910189999"
}

108
evaluations/en/Allam-7b-instruct-preview/truthfulqa_mc2_0_shot.json Normal file
View File

@@ -0,0 +1,108 @@
{
"results": {
"truthfulqa_mc2": {
"alias": "truthfulqa_mc2",
"acc,none": 0.4667466051524712,
"acc_stderr,none": 0.015605585169281691
}
},
"group_subtasks": {
"truthfulqa_mc2": []
},
"configs": {
"truthfulqa_mc2": {
"task": "truthfulqa_mc2",
"tag": [
"truthfulqa"
],
"dataset_path": "truthful_qa",
"dataset_name": "multiple_choice",
"validation_split": "validation",
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
"doc_to_target": 0,
"doc_to_choice": "{{mc2_targets.choices}}",
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 2.0
}
}
},
"versions": {
"truthfulqa_mc2": 2.0
},
"n-shot": {
"truthfulqa_mc2": 0
},
"higher_is_better": {
"truthfulqa_mc2": {
"acc": true
}
},
"n-samples": {
"truthfulqa_mc2": {
"original": 817,
"effective": 817
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735957764.7570622,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.90\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 24434.078025398,
"end_time": 24545.624577618,
"total_evaluation_time_seconds": "111.54655221999928"
}

108
evaluations/en/Allam-7b-instruct-preview/winogrande_0_shot.json Normal file
View File

@@ -0,0 +1,108 @@
{
"results": {
"winogrande": {
"alias": "winogrande",
"acc,none": 0.7048145224940805,
"acc_stderr,none": 0.012819410741754765
}
},
"group_subtasks": {
"winogrande": []
},
"configs": {
"winogrande": {
"task": "winogrande",
"dataset_path": "winogrande",
"dataset_name": "winogrande_xl",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "sentence",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"winogrande": 1.0
},
"n-shot": {
"winogrande": 0
},
"higher_is_better": {
"winogrande": {
"acc": true
}
},
"n-samples": {
"winogrande": {
"original": 1267,
"effective": 1267
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735957928.9213855,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.90\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 4096,
"task_hashes": {},
"model_source": "vllm",
"model_name": "/tmp/7b-alpha-v1.27.2.25",
"model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 24598.479043164,
"end_time": 24674.97354231,
"total_evaluation_time_seconds": "76.49449914599973"
}

1134
evaluations/en/Falcon3-7B-Instruct/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

123
evaluations/en/Falcon3-7B-Instruct/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,123 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.5571672354948806,
"acc_stderr,none": 0.014515573873348892,
"acc_norm,none": 0.5947098976109215,
"acc_norm_stderr,none": 0.01434686906022932
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736910183.5373647,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"arc_challenge": "a6a6d87aa680bdfdb3d3f0c716078b0dc58062b476f9c2d71adccaae38cf3e10"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 620433.885763592,
"end_time": 620496.540439545,
"total_evaluation_time_seconds": "62.654675952973776"
}

127
evaluations/en/Falcon3-7B-Instruct/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,127 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.33705357142857145,
"acc_stderr,none": 0.02235810146577642,
"acc_norm,none": 0.33705357142857145,
"acc_norm_stderr,none": 0.02235810146577642
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": "auto",
"batch_sizes": [
16
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737963526.1678772,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"gpqa_main_n_shot": "baab13c53a170f647515cafd634518b1d56d1b633ce63ab63ea081a49cbeed1a"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 47062.544835171,
"end_time": 47158.146115345,
"total_evaluation_time_seconds": "95.60128017399984"
}

159
evaluations/en/Falcon3-7B-Instruct/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,159 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.7892342683851402,
"exact_match_stderr,strict-match": 0.011234280469030463,
"exact_match,flexible-extract": 0.7930250189537529,
"exact_match_stderr,flexible-extract": 0.011159498164891776
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736905859.2699218,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"gsm8k": "6dc2d8763af1e4661e72a6cdacb6cca4979ac315556ee509687d296da8051cc2"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 616109.524047477,
"end_time": 616801.085240661,
"total_evaluation_time_seconds": "691.5611931839958"
}

124
evaluations/en/Falcon3-7B-Instruct/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,124 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.6032662816172077,
"acc_stderr,none": 0.004882200364432369,
"acc_norm,none": 0.7843059151563434,
"acc_norm_stderr,none": 0.004104623991846364
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736907020.9520104,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"hellaswag": "745f36a5a7a36e5192c010e2b43818ea1ff49739a6078fa6edbcf3bda680e5d7"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 617271.261912427,
"end_time": 617483.451207438,
"total_evaluation_time_seconds": "212.18929501099046"
}

317
evaluations/en/Falcon3-7B-Instruct/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,317 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.6612612612612613,
"acc_stderr,none": 0.0075941533560203575
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.5583982202447163,
"acc_stderr,none": 0.008282052379666472
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.761094674556213,
"acc_stderr,none": 0.008201801118670663
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.6977953410981698,
"acc_stderr,none": 0.006623347622611029
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.8410050251256281,
"acc_stderr,none": 0.005184872773495539
}
},
"group_subtasks": {
"ethics_utilitarianism": [],
"ethics_cm": [],
"ethics_virtue": [],
"ethics_justice": [],
"ethics_deontology": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_deontology": {
"original": 3596,
"effective": 3596
},
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_virtue": {
"original": 4975,
"effective": 4975
},
"ethics_cm": {
"original": 3885,
"effective": 3885
},
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736907313.3535528,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"ethics_deontology": "fad716ad4c1ccd0a69441ec78ee32ad04fbb04860bb2ede33329ebab0abfcd10",
"ethics_justice": "56acebbfada763de5832f4f4909e2b869d3f8233cee8640cae597b0a7dad223f",
"ethics_virtue": "3ed05bb2eac3d0663eaa0167a92917b09d04e9f6a50860f15ed101bb44d2ada9",
"ethics_cm": "14434d2a2b63a82cf13037549649099091dfcec2a0629f8438d454973f93ef17",
"ethics_utilitarianism": "25d711a4b0687249905b9da23ba457930c817c472b4f53388427a6f679289c8d"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 617563.658377943,
"end_time": 617709.608623462,
"total_evaluation_time_seconds": "145.95024551905226"
}

138
evaluations/en/Falcon3-7B-Instruct/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,138 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.5600739371534196,
"prompt_level_strict_acc_stderr,none": 0.02136070822080198,
"inst_level_strict_acc,none": 0.6858513189448441,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.6266173752310537,
"prompt_level_loose_acc_stderr,none": 0.020815238376834504,
"inst_level_loose_acc,none": 0.7350119904076738,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736891917.073872,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"ifeval": "35b1a968304ce1d8fa21032567a89deea9b44fc4851893dea1a34179b20df314"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 602167.479468507,
"end_time": 602798.440833874,
"total_evaluation_time_seconds": "630.9613653670531"
}

533
evaluations/en/Falcon3-7B-Instruct/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,533 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.3076,
"exact_match_stderr,none": 0.006198998754660659,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.4026958719460826,
"exact_match_stderr,none": 0.014241115293724816
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.350210970464135,
"exact_match_stderr,none": 0.021934133893619426
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.3173277661795407,
"exact_match_stderr,none": 0.02128855620995171
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.09745293466223699,
"exact_match_stderr,none": 0.009874818485404377
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.24444444444444444,
"exact_match_stderr,none": 0.018510958396334234
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.5120551090700345,
"exact_match_stderr,none": 0.016946659873163027
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.1391941391941392,
"exact_match_stderr,none": 0.014827394112308778
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.3076,
"exact_match_stderr,none": 0.006198998754660659,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x15110549ecb0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x15110549e050>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x15110549dcf0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x151105491360>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x151105490790>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x15116fad96c0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x15116fbe83a0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736902050.8686402,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"minerva_math_algebra": "185f34b170fd1ddec0f7e4c6f6b46ec8e3634ad4c99d822a3e2f0a964a15f0d5",
"minerva_math_counting_and_prob": "7edba0e802d0ed4e586e3511f6cc4f7d369268a05835a9a4160e9c79236c0718",
"minerva_math_geometry": "a089b5ed647abeb1874a75b3212f265db6f797cb85a56c4ee8b6dcba00bb946f",
"minerva_math_intermediate_algebra": "1f523afc1e3a8ca005120f5c859d3ca68c7cc592bddc4d583eab99c076f188d1",
"minerva_math_num_theory": "0d8bdb3a26388da49d3e8d8419869655a3a3247dde250e368e44534cf5bba0ea",
"minerva_math_prealgebra": "27c50c162f003f7257958233b7e6501b6250cf8c580dda185ddc2f76ff9ae866",
"minerva_math_precalc": "1f27730753ee7cd62d6de902471a10a0adb5e0254b7d6014f56f459820aec022"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 612301.351469343,
"end_time": 616050.463408958,
"total_evaluation_time_seconds": "3749.1119396151043"
}

3345
evaluations/en/Falcon3-7B-Instruct/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

1107
evaluations/en/Falcon3-7B-Instruct/mmlu_pro_5_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

134
evaluations/en/Falcon3-7B-Instruct/triviaqa_5_shot.json Normal file
View File

@@ -0,0 +1,134 @@
{
"results": {
"triviaqa": {
"alias": "triviaqa",
"exact_match,remove_whitespace": 0.5197837717342845,
"exact_match_stderr,remove_whitespace": 0.003729771668524104
}
},
"group_subtasks": {
"triviaqa": []
},
"configs": {
"triviaqa": {
"task": "triviaqa",
"dataset_path": "trivia_qa",
"dataset_name": "rc.nocontext",
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "Question: {{question}}?\nAnswer:",
"doc_to_target": "{{answer.aliases}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"\n",
".",
","
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "remove_whitespace",
"filter": [
{
"function": "remove_whitespace"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 3.0
}
}
},
"versions": {
"triviaqa": 3.0
},
"n-shot": {
"triviaqa": 5
},
"higher_is_better": {
"triviaqa": {
"exact_match": true
}
},
"n-samples": {
"triviaqa": {
"original": 17944,
"effective": 17944
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736892612.7161763,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"triviaqa": "670d2ae10dd71aa794fbdf7ab8e87b2005e2dda265045033795fd65031df1ea4"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 602862.940441801,
"end_time": 603179.077445082,
"total_evaluation_time_seconds": "316.1370032810373"
}

114
evaluations/en/Falcon3-7B-Instruct/truthfulqa_mc2_0_shot.json Normal file
View File

@@ -0,0 +1,114 @@
{
"results": {
"truthfulqa_mc2": {
"alias": "truthfulqa_mc2",
"acc,none": 0.5553251876617251,
"acc_stderr,none": 0.01592232780967959
}
},
"group_subtasks": {
"truthfulqa_mc2": []
},
"configs": {
"truthfulqa_mc2": {
"task": "truthfulqa_mc2",
"tag": [
"truthfulqa"
],
"dataset_path": "truthful_qa",
"dataset_name": "multiple_choice",
"validation_split": "validation",
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
"doc_to_target": 0,
"doc_to_choice": "{{mc2_targets.choices}}",
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 2.0
}
}
},
"versions": {
"truthfulqa_mc2": 2.0
},
"n-shot": {
"truthfulqa_mc2": 0
},
"higher_is_better": {
"truthfulqa_mc2": {
"acc": true
}
},
"n-samples": {
"truthfulqa_mc2": {
"original": 817,
"effective": 817
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736907663.6040406,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"truthfulqa_mc2": "b2a468babf2fac051de630e3e136ca3588387b755a38c843be1b929ca8bb21ab"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 617914.090583994,
"end_time": 617984.84129463,
"total_evaluation_time_seconds": "70.75071063591167"
}

114
evaluations/en/Falcon3-7B-Instruct/winogrande_0_shot.json Normal file
View File

@@ -0,0 +1,114 @@
{
"results": {
"winogrande": {
"alias": "winogrande",
"acc,none": 0.7008681925808997,
"acc_stderr,none": 0.012868639066091541
}
},
"group_subtasks": {
"winogrande": []
},
"configs": {
"winogrande": {
"task": "winogrande",
"dataset_path": "winogrande",
"dataset_name": "winogrande_xl",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "sentence",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"winogrande": 1.0
},
"n-shot": {
"winogrande": 0
},
"higher_is_better": {
"winogrande": {
"acc": true
}
},
"n-samples": {
"winogrande": {
"original": 1267,
"effective": 1267
}
},
"config": {
"model": "hf",
"model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 7455550464,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "5e10e017",
"date": 1736907812.9122443,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.0",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<|pad|>",
"2023"
],
"tokenizer_eos_token": [
"<|endoftext|>",
"11"
],
"tokenizer_bos_token": [
null,
"None"
],
"eot_token_id": 11,
"max_length": 32768,
"task_hashes": {
"winogrande": "e985cb5c0b87f5487bd3c1e824fda62a51869a8dc2feb550c4853fde00a3b617"
},
"model_source": "hf",
"model_name": "tiiuae/Falcon3-7B-Instruct",
"model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 618063.267604849,
"end_time": 618118.97434571,
"total_evaluation_time_seconds": "55.7067408610601"
}

1108
evaluations/en/Llama-3.3-70B-Instruct/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

117
evaluations/en/Llama-3.3-70B-Instruct/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,117 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.6117747440273038,
"acc_stderr,none": 0.014241614207414047,
"acc_norm,none": 0.6339590443686007,
"acc_norm_stderr,none": 0.014077223108470134
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737581843.4494154,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 123864.353343428,
"end_time": 123962.742418921,
"total_evaluation_time_seconds": "98.38907549300347"
}

119
evaluations/en/Llama-3.3-70B-Instruct/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,119 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.25892857142857145,
"acc_stderr,none": 0.020718879324472143,
"acc_norm,none": 0.25892857142857145,
"acc_norm_stderr,none": 0.020718879324472143
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737587163.2574375,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 129184.190027017,
"end_time": 129313.238046962,
"total_evaluation_time_seconds": "129.04801994499576"
}

153
evaluations/en/Llama-3.3-70B-Instruct/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,153 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.9082638362395754,
"exact_match_stderr,strict-match": 0.00795094214833935,
"exact_match,flexible-extract": 0.935557240333586,
"exact_match_stderr,flexible-extract": 0.0067633917284882555
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737587329.0756748,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 129350.110628712,
"end_time": 129590.582331698,
"total_evaluation_time_seconds": "240.4717029859894"
}

118
evaluations/en/Llama-3.3-70B-Instruct/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,118 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.657239593706433,
"acc_stderr,none": 0.004736621698861193,
"acc_norm,none": 0.843855805616411,
"acc_norm_stderr,none": 0.003622501370331856
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737582214.4104311,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 124235.149145965,
"end_time": 124763.573958303,
"total_evaluation_time_seconds": "528.4248123379948"
}

307
evaluations/en/Llama-3.3-70B-Instruct/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,307 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.8023166023166023,
"acc_stderr,none": 0.006390257774878015
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.6298665183537263,
"acc_stderr,none": 0.008052931418172102
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.8557692307692307,
"acc_stderr,none": 0.006757472246675016
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.8148918469217971,
"acc_stderr,none": 0.005601775490890298
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.9495477386934673,
"acc_stderr,none": 0.003103457695116678
}
},
"group_subtasks": {
"ethics_deontology": [],
"ethics_justice": [],
"ethics_cm": [],
"ethics_utilitarianism": [],
"ethics_virtue": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_virtue": {
"original": 4975,
"effective": 4975
},
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
},
"ethics_cm": {
"original": 3885,
"effective": 3885
},
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_deontology": {
"original": 3596,
"effective": 3596
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737580554.1132338,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 122574.978636081,
"end_time": 123057.366655506,
"total_evaluation_time_seconds": "482.3880194250087"
}

132
evaluations/en/Llama-3.3-70B-Instruct/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,132 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.6321626617375231,
"prompt_level_strict_acc_stderr,none": 0.02075130655602969,
"inst_level_strict_acc,none": 0.7278177458033573,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.7005545286506469,
"prompt_level_loose_acc_stderr,none": 0.019709834029672916,
"inst_level_loose_acc,none": 0.7781774580335732,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737584656.560232,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 126677.523972637,
"end_time": 126852.930489088,
"total_evaluation_time_seconds": "175.4065164509957"
}

521
evaluations/en/Llama-3.3-70B-Instruct/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,521 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.4642,
"exact_match_stderr,none": 0.006628889249601153,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.6293176074136478,
"exact_match_stderr,none": 0.01402469985709588
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.5253164556962026,
"exact_match_stderr,none": 0.02296053591387607
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.4154488517745303,
"exact_match_stderr,none": 0.022540113165977028
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.22591362126245848,
"exact_match_stderr,none": 0.013923956329164374
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.45925925925925926,
"exact_match_stderr,none": 0.021464912562702897
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.6383467278989667,
"exact_match_stderr,none": 0.016289767709994334
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.21611721611721613,
"exact_match_stderr,none": 0.017630799001234886
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.4642,
"exact_match_stderr,none": 0.006628889249601153,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a39a518a60>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a39a4ae9e0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a39a4ac8b0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a39b2a4d30>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a39b2a57e0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a39b229d80>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14a39b958670>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737583466.5454865,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 125487.461297843,
"end_time": 126234.645678455,
"total_evaluation_time_seconds": "747.1843806120014"
}

3283
evaluations/en/Llama-3.3-70B-Instruct/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

1103
evaluations/en/Llama-3.3-70B-Instruct/mmlu_pro_5_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

128
evaluations/en/Llama-3.3-70B-Instruct/triviaqa_5_shot.json Normal file
View File

@@ -0,0 +1,128 @@
{
"results": {
"triviaqa": {
"alias": "triviaqa",
"exact_match,remove_whitespace": 0.817041908158716,
"exact_match_stderr,remove_whitespace": 0.0028863596794662027
}
},
"group_subtasks": {
"triviaqa": []
},
"configs": {
"triviaqa": {
"task": "triviaqa",
"dataset_path": "trivia_qa",
"dataset_name": "rc.nocontext",
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "Question: {{question}}?\nAnswer:",
"doc_to_target": "{{answer.aliases}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"\n",
".",
","
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "remove_whitespace",
"filter": [
{
"function": "remove_whitespace"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 3.0
}
}
},
"versions": {
"triviaqa": 3.0
},
"n-shot": {
"triviaqa": 5
},
"higher_is_better": {
"triviaqa": {
"exact_match": true
}
},
"n-samples": {
"triviaqa": {
"original": 17944,
"effective": 17944
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737582778.909245,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 124799.725543077,
"end_time": 125319.396698907,
"total_evaluation_time_seconds": "519.6711558300012"
}

108
evaluations/en/Llama-3.3-70B-Instruct/truthfulqa_mc2_0_shot.json Normal file
View File

@@ -0,0 +1,108 @@
{
"results": {
"truthfulqa_mc2": {
"alias": "truthfulqa_mc2",
"acc,none": 0.6090721533173807,
"acc_stderr,none": 0.014847067973697343
}
},
"group_subtasks": {
"truthfulqa_mc2": []
},
"configs": {
"truthfulqa_mc2": {
"task": "truthfulqa_mc2",
"tag": [
"truthfulqa"
],
"dataset_path": "truthful_qa",
"dataset_name": "multiple_choice",
"validation_split": "validation",
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
"doc_to_target": 0,
"doc_to_choice": "{{mc2_targets.choices}}",
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 2.0
}
}
},
"versions": {
"truthfulqa_mc2": 2.0
},
"n-shot": {
"truthfulqa_mc2": 0
},
"higher_is_better": {
"truthfulqa_mc2": {
"acc": true
}
},
"n-samples": {
"truthfulqa_mc2": {
"original": 817,
"effective": 817
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737581194.728857,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 123215.544564302,
"end_time": 123421.64257545,
"total_evaluation_time_seconds": "206.09801114798756"
}

108
evaluations/en/Llama-3.3-70B-Instruct/winogrande_0_shot.json Normal file
View File

@@ -0,0 +1,108 @@
{
"results": {
"winogrande": {
"alias": "winogrande",
"acc,none": 0.7924230465666929,
"acc_stderr,none": 0.011398593419386783
}
},
"group_subtasks": {
"winogrande": []
},
"configs": {
"winogrande": {
"task": "winogrande",
"dataset_path": "winogrande",
"dataset_name": "winogrande_xl",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "sentence",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"winogrande": 1.0
},
"n-shot": {
"winogrande": 0
},
"higher_is_better": {
"winogrande": {
"acc": true
}
},
"n-samples": {
"winogrande": {
"original": 1267,
"effective": 1267
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.9,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "150ae04f",
"date": 1737581074.38925,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<|finetune_right_pad_id|>",
"128004"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {},
"model_source": "vllm",
"model_name": "meta-llama/Llama-3.3-70B-Instruct",
"model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 123095.348423816,
"end_time": 123177.388886054,
"total_evaluation_time_seconds": "82.04046223800106"
}

1130
evaluations/en/Meta-Llama-3.1-8B-Instruct/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

119
evaluations/en/Meta-Llama-3.1-8B-Instruct/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,119 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.5170648464163823,
"acc_stderr,none": 0.014602878388536598,
"acc_norm,none": 0.5511945392491467,
"acc_norm_stderr,none": 0.014534599585097667
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737961621.350289,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"arc_challenge": "09f9ae87a0905d63512cffc4aa91a55e44258fc35160e40fa1eb66fb75473e34"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 990761.352605304,
"end_time": 990811.547884618,
"total_evaluation_time_seconds": "50.19527931406628"
}

121
evaluations/en/Meta-Llama-3.1-8B-Instruct/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,121 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.27232142857142855,
"acc_stderr,none": 0.021055082129324165,
"acc_norm,none": 0.27232142857142855,
"acc_norm_stderr,none": 0.021055082129324165
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737961727.1741447,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"gpqa_main_n_shot": "4a64f5415ed03d5c5fec2b22dd8bfd718011928a30847c5b126c837aaf0c0619"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 990867.19129279,
"end_time": 990922.774824139,
"total_evaluation_time_seconds": "55.58353134896606"
}

155
evaluations/en/Meta-Llama-3.1-8B-Instruct/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,155 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.7649734647460197,
"exact_match_stderr,strict-match": 0.011679491349994874,
"exact_match,flexible-extract": 0.7869598180439727,
"exact_match_stderr,flexible-extract": 0.011278447856900771
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737961837.484743,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"gsm8k": "2330f4ebfcccaf66a892922df2819cdb1f118e448d076d3f42bdde4177678ac7"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 990977.464841778,
"end_time": 991047.570395286,
"total_evaluation_time_seconds": "70.10555350792129"
}

120
evaluations/en/Meta-Llama-3.1-8B-Instruct/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,120 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.5909181437960566,
"acc_stderr,none": 0.004906595857916792,
"acc_norm,none": 0.7927703644692292,
"acc_norm_stderr,none": 0.004044931315182791
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737962245.449226,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"hellaswag": "edcc7edd27a555d3f7cbca0641152b2c5e4eb6eb79c5e62d7fe5887f47814323"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 991385.417049995,
"end_time": 991536.278556097,
"total_evaluation_time_seconds": "150.86150610190816"
}

313
evaluations/en/Meta-Llama-3.1-8B-Instruct/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,313 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.6028314028314028,
"acc_stderr,none": 0.007851375973914774
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.6362625139043382,
"acc_stderr,none": 0.00802347957953013
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.6830621301775148,
"acc_stderr,none": 0.008949404717643246
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.552828618968386,
"acc_stderr,none": 0.007171255536806875
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.8592964824120602,
"acc_stderr,none": 0.0049302745463304706
}
},
"group_subtasks": {
"ethics_utilitarianism": [],
"ethics_deontology": [],
"ethics_virtue": [],
"ethics_justice": [],
"ethics_cm": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_cm": {
"original": 3885,
"effective": 3885
},
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_virtue": {
"original": 4975,
"effective": 4975
},
"ethics_deontology": {
"original": 3596,
"effective": 3596
},
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737961961.397722,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"ethics_cm": "088ead6c08bb523b9de2bf5098b07ad2d484b8d19d068937634e20e4a776db84",
"ethics_justice": "29e70305fd625a6fa42aa154ef0c4fcd7ffbfce91483485d61ef01ebaab02235",
"ethics_virtue": "b3e6efc9b8e5a591f9e9bd96c14a97d118c29455f4441e52d97b10b404513a55",
"ethics_deontology": "5311ba877c2291b107da9263731e4895484636a7fdce77b31855eb34cc6c2a37",
"ethics_utilitarianism": "50e3b75384c265c6c5fb9691f46a46b22a44ffb07d131e285b5f0a84b1025bc8"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 991101.332318416,
"end_time": 991237.205268011,
"total_evaluation_time_seconds": "135.87294959498104"
}

134
evaluations/en/Meta-Llama-3.1-8B-Instruct/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,134 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.4436229205175601,
"prompt_level_strict_acc_stderr,none": 0.021379361149596345,
"inst_level_strict_acc,none": 0.5851318944844125,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.49168207024029575,
"prompt_level_loose_acc_stderr,none": 0.021513596564021183,
"inst_level_loose_acc,none": 0.6187050359712231,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737968143.925328,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"ifeval": "a9cc24d7d92904c9f59225bb28b88b892d9ab82be222808ea7fa345ffd4500ae"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 1677873.808264766,
"end_time": 1678076.48068606,
"total_evaluation_time_seconds": "202.67242129403166"
}

529
evaluations/en/Meta-Llama-3.1-8B-Instruct/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,529 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.3426,
"exact_match_stderr,none": 0.00626883548076138,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.4928390901432182,
"exact_match_stderr,none": 0.014517208529270137
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.3059071729957806,
"exact_match_stderr,none": 0.021187174233958342
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.27348643006263046,
"exact_match_stderr,none": 0.02038805554382814
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.1362126245847176,
"exact_match_stderr,none": 0.011421123769972273
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.23703703703703705,
"exact_match_stderr,none": 0.01831746837581445
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.5889781859931114,
"exact_match_stderr,none": 0.016681012759620913
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.16117216117216118,
"exact_match_stderr,none": 0.015750095129187364
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.3426,
"exact_match_stderr,none": 0.00626883548076138,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14de24096200>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14de24094310>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14de240e84c0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14de2409ca60>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14de2409c700>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14de2555b760>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14de2567dfc0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737963129.649857,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"minerva_math_algebra": "5c955bbc89ad645142d61b1594b7c36b552b722edf416ae40fcc71a4c50bd24b",
"minerva_math_counting_and_prob": "44b9697d6c9aa5b4c364a427ece31698d9eb853f35b2b059c11a461b8886534e",
"minerva_math_geometry": "e3bc2da59c734f3345ac1db47104b32ddcaf82e460a2dc3449e2c88249e4e1fb",
"minerva_math_intermediate_algebra": "fba9ce144ffb78d824e4e4cc707e887c24afd73cc95ae48c38feef96e61fc77c",
"minerva_math_num_theory": "a54599f16065edfa4a097d2e6d0c7f71d92ece79ff5d4910abcc374456f6b352",
"minerva_math_prealgebra": "9d0a86e21bfe1ffa07f634fec45d83c27d6190dd7b452230e405b7640a28fd6f",
"minerva_math_precalc": "77e35064ebbe841cd39c111b65213ee245825d611c4bf7920b08c823d8db65ef"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 992269.559608006,
"end_time": 992486.51410904,
"total_evaluation_time_seconds": "216.95450103399344"
}

3289
evaluations/en/Meta-Llama-3.1-8B-Instruct/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

1103
evaluations/en/Meta-Llama-3.1-8B-Instruct/mmlu_pro_5_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

130
evaluations/en/Meta-Llama-3.1-8B-Instruct/triviaqa_5_shot.json Normal file
View File

@@ -0,0 +1,130 @@
{
"results": {
"triviaqa": {
"alias": "triviaqa",
"exact_match,remove_whitespace": 0.7004569772625947,
"exact_match_stderr,remove_whitespace": 0.0034195803141582057
}
},
"group_subtasks": {
"triviaqa": []
},
"configs": {
"triviaqa": {
"task": "triviaqa",
"dataset_path": "trivia_qa",
"dataset_name": "rc.nocontext",
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "Question: {{question}}?\nAnswer:",
"doc_to_target": "{{answer.aliases}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"\n",
".",
","
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "remove_whitespace",
"filter": [
{
"function": "remove_whitespace"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 3.0
}
}
},
"versions": {
"triviaqa": 3.0
},
"n-shot": {
"triviaqa": 5
},
"higher_is_better": {
"triviaqa": {
"exact_match": true
}
},
"n-samples": {
"triviaqa": {
"original": 17944,
"effective": 17944
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737962454.507693,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"triviaqa": "379fef744d809f91d62f54f7d164c285085ce50c8fe95f2fcb8d5e375dd23848"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 991594.319315193,
"end_time": 991790.491645356,
"total_evaluation_time_seconds": "196.17233016307"
}

110
evaluations/en/Meta-Llama-3.1-8B-Instruct/truthfulqa_mc2_0_shot.json Normal file
View File

@@ -0,0 +1,110 @@
{
"results": {
"truthfulqa_mc2": {
"alias": "truthfulqa_mc2",
"acc,none": 0.5405228643859059,
"acc_stderr,none": 0.014970095044069969
}
},
"group_subtasks": {
"truthfulqa_mc2": []
},
"configs": {
"truthfulqa_mc2": {
"task": "truthfulqa_mc2",
"tag": [
"truthfulqa"
],
"dataset_path": "truthful_qa",
"dataset_name": "multiple_choice",
"validation_split": "validation",
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
"doc_to_target": 0,
"doc_to_choice": "{{mc2_targets.choices}}",
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 2.0
}
}
},
"versions": {
"truthfulqa_mc2": 2.0
},
"n-shot": {
"truthfulqa_mc2": 0
},
"higher_is_better": {
"truthfulqa_mc2": {
"acc": true
}
},
"n-samples": {
"truthfulqa_mc2": {
"original": 817,
"effective": 817
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737963404.627917,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"truthfulqa_mc2": "a84d12f632c7780645b884ce110adebc1f8277817f5cf11484c396efe340e882"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 992544.394328261,
"end_time": 992613.654196921,
"total_evaluation_time_seconds": "69.2598686600104"
}

110
evaluations/en/Meta-Llama-3.1-8B-Instruct/winogrande_0_shot.json Normal file
View File

@@ -0,0 +1,110 @@
{
"results": {
"winogrande": {
"alias": "winogrande",
"acc,none": 0.739542225730071,
"acc_stderr,none": 0.012334833671998292
}
},
"group_subtasks": {
"winogrande": []
},
"configs": {
"winogrande": {
"task": "winogrande",
"dataset_path": "winogrande",
"dataset_name": "winogrande_xl",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "sentence",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"winogrande": 1.0
},
"n-shot": {
"winogrande": 0
},
"higher_is_better": {
"winogrande": {
"acc": true
}
},
"n-samples": {
"winogrande": {
"original": 1267,
"effective": 1267
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=meta-llama/Meta-Llama-3.1-8B-Instruct,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737962141.2910187,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_eos_token": [
"<|eot_id|>",
"128009"
],
"tokenizer_bos_token": [
"<|begin_of_text|>",
"128000"
],
"eot_token_id": 128009,
"max_length": 131072,
"task_hashes": {
"winogrande": "a5ea73eb24ab46d111fe5d21eed85b1e779c0b309d80d080c3caa21a851b6feb"
},
"model_source": "vllm",
"model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_name_sanitized": "meta-llama__Meta-Llama-3.1-8B-Instruct",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 991281.220101991,
"end_time": 991330.313812068,
"total_evaluation_time_seconds": "49.093710076995194"
}

1112
evaluations/en/Mistral-7B-Instruct-v0.3/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

121
evaluations/en/Mistral-7B-Instruct-v0.3/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,121 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.575938566552901,
"acc_stderr,none": 0.0144418896274644,
"acc_norm,none": 0.5887372013651877,
"acc_norm_stderr,none": 0.01437944106852208
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457484.5890195,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 932037.087947329,
"end_time": 932627.888443997,
"total_evaluation_time_seconds": "590.8004966679728"
}

123
evaluations/en/Mistral-7B-Instruct-v0.3/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,123 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.23214285714285715,
"acc_stderr,none": 0.01996935857569919,
"acc_norm,none": 0.23214285714285715,
"acc_norm_stderr,none": 0.01996935857569919
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732155399.0952759,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.86\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 635555.45223858,
"end_time": 636027.642566244,
"total_evaluation_time_seconds": "472.19032766402233"
}

157
evaluations/en/Mistral-7B-Instruct-v0.3/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,157 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.4836997725549659,
"exact_match_stderr,strict-match": 0.013765164147036959,
"exact_match,flexible-extract": 0.4844579226686884,
"exact_match_stderr,flexible-extract": 0.013765829454512888
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457438.5119252,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 632810.518285338,
"end_time": 642083.759931333,
"total_evaluation_time_seconds": "9273.241645995062"
}

122
evaluations/en/Mistral-7B-Instruct-v0.3/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,122 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.6486755626369249,
"acc_stderr,none": 0.0047640845971768965,
"acc_norm,none": 0.8293168691495718,
"acc_norm_stderr,none": 0.0037546293132753286
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457501.3892474,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 938256.524964757,
"end_time": 940502.86117875,
"total_evaluation_time_seconds": "2246.336213993025"
}

311
evaluations/en/Mistral-7B-Instruct-v0.3/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,311 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.6875160875160875,
"acc_stderr,none": 0.00743730605460123
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.6535038932146829,
"acc_stderr,none": 0.007936404996899458
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.6989644970414202,
"acc_stderr,none": 0.008822941393145468
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.6761647254575707,
"acc_stderr,none": 0.00674918404185245
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.9127638190954774,
"acc_stderr,none": 0.004001056094140476
}
},
"group_subtasks": {
"ethics_virtue": [],
"ethics_cm": [],
"ethics_utilitarianism": [],
"ethics_deontology": [],
"ethics_justice": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_deontology": {
"original": 3596,
"effective": 3596
},
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
},
"ethics_cm": {
"original": 3885,
"effective": 3885
},
"ethics_virtue": {
"original": 4975,
"effective": 4975
}
},
"config": {
"model": "hf",
"model_args": "pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "112b79143",
"date": 1739257708.3481266,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 1368142.451898874,
"end_time": 1369038.256261414,
"total_evaluation_time_seconds": "895.8043625399005"
}

132
evaluations/en/Mistral-7B-Instruct-v0.3/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,132 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.42513863216266173,
"prompt_level_strict_acc_stderr,none": 0.021274039805355742,
"inst_level_strict_acc,none": 0.5479616306954437,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.46395563770794823,
"prompt_level_loose_acc_stderr,none": 0.021460592823736722,
"inst_level_loose_acc,none": 0.5887290167865707,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=mistralai/Mistral-7B-Instruct-v0.3,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.4,download_dir=/tmp",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "8e1bd48d",
"date": 1735756099.6672652,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.47.1",
"upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "vllm",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 9944.313018783,
"end_time": 10022.302016336,
"total_evaluation_time_seconds": "77.98899755300044"
}

525
evaluations/en/Mistral-7B-Instruct-v0.3/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,525 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.1344,
"exact_match_stderr,none": 0.00469690840313393,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.1954507160909857,
"exact_match_stderr,none": 0.011514699662714494
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.12236286919831224,
"exact_match_stderr,none": 0.015067866025208529
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.09603340292275574,
"exact_match_stderr,none": 0.013476384772608527
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.04540420819490587,
"exact_match_stderr,none": 0.006931935965006335
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.08148148148148149,
"exact_match_stderr,none": 0.011783628281121686
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.2571756601607348,
"exact_match_stderr,none": 0.014818299496867965
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.04945054945054945,
"exact_match_stderr,none": 0.009286983354895582
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.1344,
"exact_match_stderr,none": 0.00469690840313393,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14c88c892290>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14c88c890310>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14c88c86d000>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14c88c813f40>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14c88c8104c0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14c9228fbac0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x14c8ad211090>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457421.434201,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 937481.096308053,
"end_time": 984028.729417881,
"total_evaluation_time_seconds": "46547.63310982799"
}

3283
evaluations/en/Mistral-7B-Instruct-v0.3/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

1092
evaluations/en/Mistral-7B-Instruct-v0.3/mmlu_pro_5_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

132
evaluations/en/Mistral-7B-Instruct-v0.3/triviaqa_5_shot.json Normal file
View File

@@ -0,0 +1,132 @@
{
"results": {
"triviaqa": {
"alias": "triviaqa",
"exact_match,remove_whitespace": 0.6797258136424431,
"exact_match_stderr,remove_whitespace": 0.003483215316023233
}
},
"group_subtasks": {
"triviaqa": []
},
"configs": {
"triviaqa": {
"task": "triviaqa",
"dataset_path": "trivia_qa",
"dataset_name": "rc.nocontext",
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "Question: {{question}}?\nAnswer:",
"doc_to_target": "{{answer.aliases}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"\n",
".",
","
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "remove_whitespace",
"filter": [
{
"function": "remove_whitespace"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 3.0
}
}
},
"versions": {
"triviaqa": 3.0
},
"n-shot": {
"triviaqa": 5
},
"higher_is_better": {
"triviaqa": {
"exact_match": true
}
},
"n-samples": {
"triviaqa": {
"original": 17944,
"effective": 17944
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732530019.7536964,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.88\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 705391.766851171,
"end_time": 709579.729863481,
"total_evaluation_time_seconds": "4187.963012309978"
}

112
evaluations/en/Mistral-7B-Instruct-v0.3/truthfulqa_mc2_0_shot.json Normal file
View File

@@ -0,0 +1,112 @@
{
"results": {
"truthfulqa_mc2": {
"alias": "truthfulqa_mc2",
"acc,none": 0.5969383260814474,
"acc_stderr,none": 0.015440420868691797
}
},
"group_subtasks": {
"truthfulqa_mc2": []
},
"configs": {
"truthfulqa_mc2": {
"task": "truthfulqa_mc2",
"tag": [
"truthfulqa"
],
"dataset_path": "truthful_qa",
"dataset_name": "multiple_choice",
"validation_split": "validation",
"doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
"doc_to_target": 0,
"doc_to_choice": "{{mc2_targets.choices}}",
"process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "question",
"metadata": {
"version": 2.0
}
}
},
"versions": {
"truthfulqa_mc2": 2.0
},
"n-shot": {
"truthfulqa_mc2": 0
},
"higher_is_better": {
"truthfulqa_mc2": {
"acc": true
}
},
"n-samples": {
"truthfulqa_mc2": {
"original": 817,
"effective": 817
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457521.7663252,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 938096.908966253,
"end_time": 938758.534434522,
"total_evaluation_time_seconds": "661.6254682689905"
}

112
evaluations/en/Mistral-7B-Instruct-v0.3/winogrande_0_shot.json Normal file
View File

@@ -0,0 +1,112 @@
{
"results": {
"winogrande": {
"alias": "winogrande",
"acc,none": 0.739542225730071,
"acc_stderr,none": 0.01233483367199829
}
},
"group_subtasks": {
"winogrande": []
},
"configs": {
"winogrande": {
"task": "winogrande",
"dataset_path": "winogrande",
"dataset_name": "winogrande_xl",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n",
"doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n",
"doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "sentence",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"winogrande": 1.0
},
"n-shot": {
"winogrande": 0
},
"higher_is_better": {
"winogrande": {
"acc": true
}
},
"n-samples": {
"winogrande": {
"original": 1267,
"effective": 1267
}
},
"config": {
"model": "hf",
"model_args": "parallelize=True,pretrained=mistralai/Mistral-7B-Instruct-v0.3,trust_remote_code=True,mm=False,trust_remote_code=True",
"model_num_parameters": 7248023552,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "e0bc86c23ce5aae1db576c8cca6f06f1f73af2db",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "3127d82f",
"date": 1732457457.0153227,
"pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.86\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
"transformers_version": "4.46.3",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 32768,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-7B-Instruct-v0.3",
"model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.3",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 940275.314023227,
"end_time": 940769.680795377,
"total_evaluation_time_seconds": "494.36677215003874"
}

1114
evaluations/en/Mistral-Nemo-Instruct-2407/agieval_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

123
evaluations/en/Mistral-Nemo-Instruct-2407/arc_challenge_0_shot.json Normal file
View File

@@ -0,0 +1,123 @@
{
"results": {
"arc_challenge": {
"alias": "arc_challenge",
"acc,none": 0.5622866894197952,
"acc_stderr,none": 0.01449757388110829,
"acc_norm,none": 0.590443686006826,
"acc_norm_stderr,none": 0.014370358632472444
}
},
"group_subtasks": {
"arc_challenge": []
},
"configs": {
"arc_challenge": {
"task": "arc_challenge",
"tag": [
"ai2_arc"
],
"dataset_path": "allenai/ai2_arc",
"dataset_name": "ARC-Challenge",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{choices.label.index(answerKey)}}",
"doc_to_choice": "{{choices.text}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
"metadata": {
"version": 1.0
}
}
},
"versions": {
"arc_challenge": 1.0
},
"n-shot": {
"arc_challenge": 0
},
"higher_is_better": {
"arc_challenge": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"arc_challenge": {
"original": 1172,
"effective": 1172
}
},
"config": {
"model": "hf",
"model_args": "pretrained=mistralai/Mistral-Nemo-Instruct-2407,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 12247782400,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "8aedd450f2583e9c67fae1929f6936b8fc5aef9c",
"batch_size": "auto",
"batch_sizes": [
64
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737893401.9579802,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 131072,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-Nemo-Instruct-2407",
"model_name_sanitized": "mistralai__Mistral-Nemo-Instruct-2407",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 5777.925846111,
"end_time": 5816.133359654,
"total_evaluation_time_seconds": "38.20751354299955"
}

121
evaluations/en/Mistral-Nemo-Instruct-2407/gpqa_main_n_shot_0_shot.json Normal file
View File

@@ -0,0 +1,121 @@
{
"results": {
"gpqa_main_n_shot": {
"alias": "gpqa_main_n_shot",
"acc,none": 0.24330357142857142,
"acc_stderr,none": 0.020294638625866786,
"acc_norm,none": 0.24330357142857142,
"acc_norm_stderr,none": 0.020294638625866786
}
},
"group_subtasks": {
"gpqa_main_n_shot": []
},
"configs": {
"gpqa_main_n_shot": {
"task": "gpqa_main_n_shot",
"tag": "gpqa",
"dataset_path": "Idavidrein/gpqa",
"dataset_name": "gpqa_main",
"training_split": "train",
"validation_split": "train",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n rng.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
"doc_to_target": "answer",
"doc_to_choice": [
"(A)",
"(B)",
"(C)",
"(D)"
],
"description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 2.0
}
}
},
"versions": {
"gpqa_main_n_shot": 2.0
},
"n-shot": {
"gpqa_main_n_shot": 0
},
"higher_is_better": {
"gpqa_main_n_shot": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"gpqa_main_n_shot": {
"original": 448,
"effective": 448
}
},
"config": {
"model": "vllm",
"model_args": "pretrained=mistralai/Mistral-Nemo-Instruct-2407,tensor_parallel_size=1,data_parallel_size=8,gpu_memory_utilization=0.8,download_dir=/tmp,enforce_eager=True",
"batch_size": 1,
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1738145952.0897527,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 96\nOn-line CPU(s) list: 0-95\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V12 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 2\nStepping: 0\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 3 MiB (96 instances)\nL1i cache: 3 MiB (96 instances)\nL2 cache: 48 MiB (96 instances)\nL3 cache: 384 MiB (24 instances)\nNUMA node(s): 4\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nNUMA node2 CPU(s): 48-71\nNUMA node3 CPU(s): 72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": null,
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 131072,
"task_hashes": {
"gpqa_main_n_shot": "4a64f5415ed03d5c5fec2b22dd8bfd718011928a30847c5b126c837aaf0c0619"
},
"model_source": "vllm",
"model_name": "mistralai/Mistral-Nemo-Instruct-2407",
"model_name_sanitized": "mistralai__Mistral-Nemo-Instruct-2407",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 600856.106946281,
"end_time": 600922.223087618,
"total_evaluation_time_seconds": "66.11614133697003"
}

157
evaluations/en/Mistral-Nemo-Instruct-2407/gsm8k_5_shot.json Normal file
View File

@@ -0,0 +1,157 @@
{
"results": {
"gsm8k": {
"alias": "gsm8k",
"exact_match,strict-match": 0.7194844579226687,
"exact_match_stderr,strict-match": 0.012374608490929554,
"exact_match,flexible-extract": 0.7429871114480667,
"exact_match_stderr,flexible-extract": 0.012036781757428675
}
},
"group_subtasks": {
"gsm8k": []
},
"configs": {
"gsm8k": {
"task": "gsm8k",
"tag": [
"math_word_problems"
],
"dataset_path": "gsm8k",
"dataset_name": "main",
"training_split": "train",
"test_split": "test",
"fewshot_split": "train",
"doc_to_text": "Question: {{question}}\nAnswer:",
"doc_to_target": "{{answer}}",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 5,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true,
"ignore_case": true,
"ignore_punctuation": false,
"regexes_to_ignore": [
",",
"\\$",
"(?s).*#### ",
"\\.$"
]
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Question:",
"</s>",
"<|im_end|>"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"filter_list": [
{
"name": "strict-match",
"filter": [
{
"function": "regex",
"regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
},
{
"function": "take_first"
}
]
},
{
"name": "flexible-extract",
"filter": [
{
"function": "regex",
"group_select": -1,
"regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
},
{
"function": "take_first"
}
]
}
],
"should_decontaminate": false,
"metadata": {
"version": 3.0
}
}
},
"versions": {
"gsm8k": 3.0
},
"n-shot": {
"gsm8k": 5
},
"higher_is_better": {
"gsm8k": {
"exact_match": true
}
},
"n-samples": {
"gsm8k": {
"original": 1319,
"effective": 1319
}
},
"config": {
"model": "hf",
"model_args": "pretrained=mistralai/Mistral-Nemo-Instruct-2407,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 12247782400,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "8aedd450f2583e9c67fae1929f6936b8fc5aef9c",
"batch_size": "auto",
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737956733.1439893,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 131072,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-Nemo-Instruct-2407",
"model_name_sanitized": "mistralai__Mistral-Nemo-Instruct-2407",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 69108.901463069,
"end_time": 72081.874727591,
"total_evaluation_time_seconds": "2972.973264521992"
}

124
evaluations/en/Mistral-Nemo-Instruct-2407/hellaswag_0_shot.json Normal file
View File

@@ -0,0 +1,124 @@
{
"results": {
"hellaswag": {
"alias": "hellaswag",
"acc,none": 0.6328420633339972,
"acc_stderr,none": 0.0048104493435723854,
"acc_norm,none": 0.823541127265485,
"acc_norm_stderr,none": 0.003804310123682686
}
},
"group_subtasks": {
"hellaswag": []
},
"configs": {
"hellaswag": {
"task": "hellaswag",
"tag": [
"multiple_choice"
],
"dataset_path": "hellaswag",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"validation_split": "validation",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "{{query}}",
"doc_to_target": "{{label}}",
"doc_to_choice": "choices",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"hellaswag": 1.0
},
"n-shot": {
"hellaswag": 0
},
"higher_is_better": {
"hellaswag": {
"acc": true,
"acc_norm": true
}
},
"n-samples": {
"hellaswag": {
"original": 10042,
"effective": 10042
}
},
"config": {
"model": "hf",
"model_args": "pretrained=mistralai/Mistral-Nemo-Instruct-2407,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 12247782400,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "8aedd450f2583e9c67fae1929f6936b8fc5aef9c",
"batch_size": "auto",
"batch_sizes": [
64
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737893612.0515287,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 131072,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-Nemo-Instruct-2407",
"model_name_sanitized": "mistralai__Mistral-Nemo-Instruct-2407",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 5987.885904716,
"end_time": 6264.313032231,
"total_evaluation_time_seconds": "276.4271275149995"
}

313
evaluations/en/Mistral-Nemo-Instruct-2407/hendrycks_ethics_0_shot.json Normal file
View File

@@ -0,0 +1,313 @@
{
"results": {
"ethics_cm": {
"alias": "ethics_cm",
"acc,none": 0.5446589446589447,
"acc_stderr,none": 0.007990815702906981
},
"ethics_deontology": {
"alias": "ethics_deontology",
"acc,none": 0.6115127919911012,
"acc_stderr,none": 0.008129085423675336
},
"ethics_justice": {
"alias": "ethics_justice",
"acc,none": 0.7688609467455622,
"acc_stderr,none": 0.008108444402646632
},
"ethics_utilitarianism": {
"alias": "ethics_utilitarianism",
"acc,none": 0.5405574043261231,
"acc_stderr,none": 0.007187857815072047
},
"ethics_virtue": {
"alias": "ethics_virtue",
"acc,none": 0.9272361809045226,
"acc_stderr,none": 0.003682985737376842
}
},
"group_subtasks": {
"ethics_deontology": [],
"ethics_cm": [],
"ethics_virtue": [],
"ethics_justice": [],
"ethics_utilitarianism": []
},
"configs": {
"ethics_cm": {
"task": "ethics_cm",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "commonsense",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_deontology": {
"task": "ethics_deontology",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "deontology",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_justice": {
"task": "ethics_justice",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "justice",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"unreasonable",
"reasonable"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_utilitarianism": {
"task": "ethics_utilitarianism",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "utilitarianism",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "def doc_to_text(doc) -> str:\n doc = _preproc_doc(doc)\n return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
"doc_to_target": "def doc_to_target(doc):\n doc = _preproc_doc(doc)\n return doc[\"label\"]\n",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"ethics_virtue": {
"task": "ethics_virtue",
"tag": [
"hendrycks_ethics"
],
"dataset_path": "EleutherAI/hendrycks_ethics",
"dataset_name": "virtue",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
"doc_to_target": "label",
"doc_to_choice": [
"no",
"yes"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "acc"
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"ethics_cm": 1.0,
"ethics_deontology": 1.0,
"ethics_justice": 1.0,
"ethics_utilitarianism": 1.0,
"ethics_virtue": 1.0
},
"n-shot": {
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_virtue": 0
},
"higher_is_better": {
"ethics_cm": {
"acc": true
},
"ethics_deontology": {
"acc": true
},
"ethics_justice": {
"acc": true
},
"ethics_utilitarianism": {
"acc": true
},
"ethics_virtue": {
"acc": true
}
},
"n-samples": {
"ethics_utilitarianism": {
"original": 4808,
"effective": 4808
},
"ethics_justice": {
"original": 2704,
"effective": 2704
},
"ethics_virtue": {
"original": 4975,
"effective": 4975
},
"ethics_cm": {
"original": 3885,
"effective": 3885
},
"ethics_deontology": {
"original": 3596,
"effective": 3596
}
},
"config": {
"model": "hf",
"model_args": "pretrained=mistralai/Mistral-Nemo-Instruct-2407,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 12247782400,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "8aedd450f2583e9c67fae1929f6936b8fc5aef9c",
"batch_size": "auto",
"batch_sizes": [
32
],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737892742.1856506,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 131072,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-Nemo-Instruct-2407",
"model_name_sanitized": "mistralai__Mistral-Nemo-Instruct-2407",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 5118.081982267,
"end_time": 5313.55855677,
"total_evaluation_time_seconds": "195.47657450299994"
}

136
evaluations/en/Mistral-Nemo-Instruct-2407/ifeval_0_shot.json Normal file
View File

@@ -0,0 +1,136 @@
{
"results": {
"ifeval": {
"alias": "ifeval",
"prompt_level_strict_acc,none": 0.30129390018484287,
"prompt_level_strict_acc_stderr,none": 0.019744473483514293,
"inst_level_strict_acc,none": 0.38968824940047964,
"inst_level_strict_acc_stderr,none": "N/A",
"prompt_level_loose_acc,none": 0.3585951940850277,
"prompt_level_loose_acc_stderr,none": 0.020638182918873243,
"inst_level_loose_acc,none": 0.45083932853717024,
"inst_level_loose_acc_stderr,none": "N/A"
}
},
"group_subtasks": {
"ifeval": []
},
"configs": {
"ifeval": {
"task": "ifeval",
"dataset_path": "google/IFEval",
"test_split": "train",
"doc_to_text": "prompt",
"doc_to_target": 0,
"process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"num_fewshot": 0,
"metric_list": [
{
"metric": "prompt_level_strict_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_strict_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
},
{
"metric": "prompt_level_loose_acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "inst_level_loose_acc",
"aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [],
"do_sample": false,
"temperature": 0.0,
"max_gen_toks": 1280
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 4.0
}
}
},
"versions": {
"ifeval": 4.0
},
"n-shot": {
"ifeval": 0
},
"higher_is_better": {
"ifeval": {
"prompt_level_strict_acc": true,
"inst_level_strict_acc": true,
"prompt_level_loose_acc": true,
"inst_level_loose_acc": true
}
},
"n-samples": {
"ifeval": {
"original": 541,
"effective": 541
}
},
"config": {
"model": "hf",
"model_args": "pretrained=mistralai/Mistral-Nemo-Instruct-2407,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 12247782400,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "8aedd450f2583e9c67fae1929f6936b8fc5aef9c",
"batch_size": "auto",
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737924166.1102595,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 131072,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-Nemo-Instruct-2407",
"model_name_sanitized": "mistralai__Mistral-Nemo-Instruct-2407",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 36541.988769304,
"end_time": 38833.188633169,
"total_evaluation_time_seconds": "2291.1998638649966"
}

525
evaluations/en/Mistral-Nemo-Instruct-2407/minerva_math_4_shot.json Normal file
View File

@@ -0,0 +1,525 @@
{
"results": {
"minerva_math": {
"exact_match,none": 0.2962,
"exact_match_stderr,none": 0.006122935392545511,
"alias": "minerva_math"
},
"minerva_math_algebra": {
"alias": " - minerva_math_algebra",
"exact_match,none": 0.4128053917438922,
"exact_match_stderr,none": 0.014296224701563264
},
"minerva_math_counting_and_prob": {
"alias": " - minerva_math_counting_and_prob",
"exact_match,none": 0.2552742616033755,
"exact_match_stderr,none": 0.020048003331023533
},
"minerva_math_geometry": {
"alias": " - minerva_math_geometry",
"exact_match,none": 0.24425887265135698,
"exact_match_stderr,none": 0.01965159270337075
},
"minerva_math_intermediate_algebra": {
"alias": " - minerva_math_intermediate_algebra",
"exact_match,none": 0.12513842746400886,
"exact_match_stderr,none": 0.011016959383289181
},
"minerva_math_num_theory": {
"alias": " - minerva_math_num_theory",
"exact_match,none": 0.1962962962962963,
"exact_match_stderr,none": 0.017108410215595875
},
"minerva_math_prealgebra": {
"alias": " - minerva_math_prealgebra",
"exact_match,none": 0.521239954075775,
"exact_match_stderr,none": 0.016936285753255634
},
"minerva_math_precalc": {
"alias": " - minerva_math_precalc",
"exact_match,none": 0.14652014652014653,
"exact_match_stderr,none": 0.01514771264919227
}
},
"groups": {
"minerva_math": {
"exact_match,none": 0.2962,
"exact_match_stderr,none": 0.006122935392545511,
"alias": "minerva_math"
}
},
"group_subtasks": {
"minerva_math": [
"minerva_math_algebra",
"minerva_math_counting_and_prob",
"minerva_math_geometry",
"minerva_math_intermediate_algebra",
"minerva_math_num_theory",
"minerva_math_prealgebra",
"minerva_math_precalc"
]
},
"configs": {
"minerva_math_algebra": {
"task": "minerva_math_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x150b79d10670>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_counting_and_prob": {
"task": "minerva_math_counting_and_prob",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "counting_and_probability",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x150b79da25f0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_geometry": {
"task": "minerva_math_geometry",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "geometry",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x150b79da08b0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_intermediate_algebra": {
"task": "minerva_math_intermediate_algebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "intermediate_algebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x150b79d20670>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_num_theory": {
"task": "minerva_math_num_theory",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "number_theory",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x150b79d213f0>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_prealgebra": {
"task": "minerva_math_prealgebra",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "prealgebra",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x150b7a5c9990>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
},
"minerva_math_precalc": {
"task": "minerva_math_precalc",
"tag": [
"math_word_problems"
],
"group": [
"math_word_problems"
],
"dataset_path": "EleutherAI/hendrycks_math",
"dataset_name": "precalculus",
"dataset_kwargs": {
"trust_remote_code": true
},
"training_split": "train",
"test_split": "test",
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n",
"doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
"doc_to_target": "{{answer if few_shot is undefined else solution}}",
"process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n results = {\n \"exact_match\": retval,\n }\n return results\n",
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"fewshot_config": {
"sampler": "first_n",
"samples": "<function list_fewshot_samples at 0x150b7b768310>"
},
"num_fewshot": 4,
"metric_list": [
{
"metric": "exact_match",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "generate_until",
"generation_kwargs": {
"until": [
"Problem:"
],
"do_sample": false,
"temperature": 0.0
},
"repeats": 1,
"should_decontaminate": false,
"metadata": {
"version": 1.0
}
}
},
"versions": {
"minerva_math": 1.0,
"minerva_math_algebra": 1.0,
"minerva_math_counting_and_prob": 1.0,
"minerva_math_geometry": 1.0,
"minerva_math_intermediate_algebra": 1.0,
"minerva_math_num_theory": 1.0,
"minerva_math_prealgebra": 1.0,
"minerva_math_precalc": 1.0
},
"n-shot": {
"minerva_math_algebra": 4,
"minerva_math_counting_and_prob": 4,
"minerva_math_geometry": 4,
"minerva_math_intermediate_algebra": 4,
"minerva_math_num_theory": 4,
"minerva_math_prealgebra": 4,
"minerva_math_precalc": 4
},
"higher_is_better": {
"minerva_math": {
"exact_match": true
},
"minerva_math_algebra": {
"exact_match": true
},
"minerva_math_counting_and_prob": {
"exact_match": true
},
"minerva_math_geometry": {
"exact_match": true
},
"minerva_math_intermediate_algebra": {
"exact_match": true
},
"minerva_math_num_theory": {
"exact_match": true
},
"minerva_math_prealgebra": {
"exact_match": true
},
"minerva_math_precalc": {
"exact_match": true
}
},
"n-samples": {
"minerva_math_algebra": {
"original": 1187,
"effective": 1187
},
"minerva_math_counting_and_prob": {
"original": 474,
"effective": 474
},
"minerva_math_geometry": {
"original": 479,
"effective": 479
},
"minerva_math_intermediate_algebra": {
"original": 903,
"effective": 903
},
"minerva_math_num_theory": {
"original": 540,
"effective": 540
},
"minerva_math_prealgebra": {
"original": 871,
"effective": 871
},
"minerva_math_precalc": {
"original": 546,
"effective": 546
}
},
"config": {
"model": "hf",
"model_args": "pretrained=mistralai/Mistral-Nemo-Instruct-2407,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
"model_num_parameters": 12247782400,
"model_dtype": "torch.bfloat16",
"model_revision": "main",
"model_sha": "8aedd450f2583e9c67fae1929f6936b8fc5aef9c",
"batch_size": "auto",
"batch_sizes": [],
"device": null,
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null,
"random_seed": 0,
"numpy_seed": 1234,
"torch_seed": 1234,
"fewshot_seed": 1234
},
"git_hash": "788a3672",
"date": 1737896212.8039174,
"pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.89\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
"transformers_version": "4.48.1",
"upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
"tokenizer_pad_token": [
"<unk>",
"0"
],
"tokenizer_eos_token": [
"</s>",
"2"
],
"tokenizer_bos_token": [
"<s>",
"1"
],
"eot_token_id": 2,
"max_length": 131072,
"task_hashes": {},
"model_source": "hf",
"model_name": "mistralai/Mistral-Nemo-Instruct-2407",
"model_name_sanitized": "mistralai__Mistral-Nemo-Instruct-2407",
"system_instruction": null,
"system_instruction_sha": null,
"fewshot_as_multiturn": false,
"chat_template": null,
"chat_template_sha": null,
"start_time": 8588.58337239,
"end_time": 21876.84113091,
"total_evaluation_time_seconds": "13288.257758520002"
}

3289
evaluations/en/Mistral-Nemo-Instruct-2407/mmlu_0_shot.json Normal file
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More