初始化项目，由ModelHub XC社区提供模型
Model: absltnull/predBor-v0.5 Source: Original Platform
This commit is contained in:
ModelHub XC
202
results_gpt2-orao.json
Normal file
202
results_gpt2-orao.json
Normal file
@@ -0,0 +1,202 @@
|
||||
{
|
||||
"results": {
|
||||
"hellaswag": {
|
||||
"name": "hellaswag",
|
||||
"alias": "hellaswag",
|
||||
"sample_len": 10042,
|
||||
"acc,none": 0.26926906990639315,
|
||||
"acc_stderr,none": 0.00442673471880892,
|
||||
"acc_norm,none": 0.2680740888269269,
|
||||
"acc_norm_stderr,none": 0.004420511215131016
|
||||
},
|
||||
"arc_challenge": {
|
||||
"name": "arc_challenge",
|
||||
"alias": "arc_challenge",
|
||||
"sample_len": 1172,
|
||||
"acc,none": 0.1945392491467577,
|
||||
"acc_stderr,none": 0.011567709174648728,
|
||||
"acc_norm,none": 0.2593856655290102,
|
||||
"acc_norm_stderr,none": 0.012808273573927099
|
||||
}
|
||||
},
|
||||
"group_subtasks": {},
|
||||
"configs": {
|
||||
"arc_challenge": {
|
||||
"task": "arc_challenge",
|
||||
"dataset_path": "allenai/ai2_arc",
|
||||
"dataset_name": "ARC-Challenge",
|
||||
"training_split": "train",
|
||||
"validation_split": "validation",
|
||||
"test_split": "test",
|
||||
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||
"unsafe_code": false,
|
||||
"doc_to_choice": "{{choices.text}}",
|
||||
"description": "",
|
||||
"target_delimiter": " ",
|
||||
"fewshot_delimiter": "\n\n",
|
||||
"fewshot_config": {
|
||||
"sampler": "default",
|
||||
"split": null,
|
||||
"process_docs": null,
|
||||
"fewshot_indices": null,
|
||||
"samples": null,
|
||||
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||
"doc_to_choice": "{{choices.text}}",
|
||||
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||
"gen_prefix": null,
|
||||
"fewshot_delimiter": "\n\n",
|
||||
"target_delimiter": " "
|
||||
},
|
||||
"num_fewshot": 0,
|
||||
"metric_list": [
|
||||
{
|
||||
"metric": "acc",
|
||||
"aggregation": "mean",
|
||||
"higher_is_better": true
|
||||
},
|
||||
{
|
||||
"metric": "acc_norm",
|
||||
"aggregation": "mean",
|
||||
"higher_is_better": true
|
||||
}
|
||||
],
|
||||
"output_type": "multiple_choice",
|
||||
"repeats": 1,
|
||||
"should_decontaminate": true,
|
||||
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
|
||||
"metadata": {
|
||||
"version": 1.0,
|
||||
"pretrained": "jerteh/gpt2-orao",
|
||||
"config_source": "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\arc\\arc_challenge.yaml"
|
||||
}
|
||||
},
|
||||
"hellaswag": {
|
||||
"task": "hellaswag",
|
||||
"dataset_path": "Rowan/hellaswag",
|
||||
"training_split": "train",
|
||||
"validation_split": "validation",
|
||||
"process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n",
|
||||
"doc_to_text": "{{query}}",
|
||||
"doc_to_target": "{{label}}",
|
||||
"unsafe_code": false,
|
||||
"doc_to_choice": "choices",
|
||||
"description": "",
|
||||
"target_delimiter": " ",
|
||||
"fewshot_delimiter": "\n\n",
|
||||
"fewshot_config": {
|
||||
"sampler": "default",
|
||||
"split": null,
|
||||
"process_docs": "<function process_docs at 0x000001BBC6DBF760>",
|
||||
"fewshot_indices": null,
|
||||
"samples": null,
|
||||
"doc_to_text": "{{query}}",
|
||||
"doc_to_choice": "choices",
|
||||
"doc_to_target": "{{label}}",
|
||||
"gen_prefix": null,
|
||||
"fewshot_delimiter": "\n\n",
|
||||
"target_delimiter": " "
|
||||
},
|
||||
"num_fewshot": 0,
|
||||
"metric_list": [
|
||||
{
|
||||
"metric": "acc",
|
||||
"aggregation": "mean",
|
||||
"higher_is_better": true
|
||||
},
|
||||
{
|
||||
"metric": "acc_norm",
|
||||
"aggregation": "mean",
|
||||
"higher_is_better": true
|
||||
}
|
||||
],
|
||||
"output_type": "multiple_choice",
|
||||
"repeats": 1,
|
||||
"should_decontaminate": false,
|
||||
"metadata": {
|
||||
"version": 1.0,
|
||||
"pretrained": "jerteh/gpt2-orao",
|
||||
"config_source": "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\hellaswag\\hellaswag.yaml"
|
||||
}
|
||||
}
|
||||
},
|
||||
"versions": {
|
||||
"arc_challenge": 1.0,
|
||||
"hellaswag": 1.0
|
||||
},
|
||||
"n-shot": {
|
||||
"arc_challenge": 0,
|
||||
"hellaswag": 0
|
||||
},
|
||||
"higher_is_better": {
|
||||
"arc_challenge": {
|
||||
"acc": true,
|
||||
"acc_norm": true
|
||||
},
|
||||
"hellaswag": {
|
||||
"acc": true,
|
||||
"acc_norm": true
|
||||
}
|
||||
},
|
||||
"n-samples": {
|
||||
"hellaswag": {
|
||||
"original": 10042,
|
||||
"effective": 10042
|
||||
},
|
||||
"arc_challenge": {
|
||||
"original": 1172,
|
||||
"effective": 1172
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"model": "hf",
|
||||
"model_args": {
|
||||
"pretrained": "jerteh/gpt2-orao"
|
||||
},
|
||||
"model_num_parameters": 772615680,
|
||||
"model_dtype": "torch.float32",
|
||||
"model_revision": "main",
|
||||
"model_sha": "a7dfec6b4290bcb48e579e13e39d20d0ea574449",
|
||||
"batch_size": "4",
|
||||
"batch_sizes": [],
|
||||
"device": "cuda:0",
|
||||
"use_cache": null,
|
||||
"limit": null,
|
||||
"bootstrap_iters": 100000,
|
||||
"gen_kwargs": {},
|
||||
"random_seed": 0,
|
||||
"numpy_seed": 1234,
|
||||
"torch_seed": 1234,
|
||||
"fewshot_seed": 1234
|
||||
},
|
||||
"git_hash": null,
|
||||
"date": 1782051514.3003025,
|
||||
"pretty_env_info": "PyTorch version: 2.11.0+cu130\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Microsoft Windows 11 Pro (10.0.26200 64-bit)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: N/A\n\nPython version: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 64 bit (AMD64)] (64-bit runtime)\nPython platform: Windows-10-10.0.26200-SP0\nIs CUDA available: True\nCUDA runtime version: 13.0.48\r\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 2050\nNvidia driver version: 591.86\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nName: 12th Gen Intel(R) Core(TM) i5-12450H\nManufacturer: GenuineIntel\nFamily: 205\nArchitecture: 9\nProcessorType: 3\nDeviceID: CPU0\nCurrentClockSpeed: 2000\nMaxClockSpeed: 2000\nL2CacheSize: 7168\nL2CacheSpeed: None\nRevision: None\n\nVersions of relevant libraries:\n[pip3] flash_attn==2.8.3+cu130torch2.11\n[pip3] numpy==2.2.6\n[pip3] optree==0.18.0\n[pip3] rotary-embedding-torch==0.6.4\n[pip3] torch==2.11.0+cu130\n[pip3] torchaudio==2.11.0+cu130\n[pip3] torchcrepe==0.0.20\n[pip3] torchgen==0.0.1\n[pip3] torchvision==0.26.0+cu130\n[conda] Could not collect",
|
||||
"transformers_version": "5.12.0",
|
||||
"lm_eval_version": "0.4.12",
|
||||
"upper_git_hash": null,
|
||||
"tokenizer_pad_token": [
|
||||
"<|endoftext|>",
|
||||
"49152"
|
||||
],
|
||||
"tokenizer_eos_token": [
|
||||
"<|endoftext|>",
|
||||
"49152"
|
||||
],
|
||||
"tokenizer_bos_token": [
|
||||
"<|endoftext|>",
|
||||
"49152"
|
||||
],
|
||||
"eot_token_id": 49152,
|
||||
"max_length": 1024,
|
||||
"task_hashes": {},
|
||||
"model_source": "hf",
|
||||
"model_name": "jerteh/gpt2-orao",
|
||||
"model_name_sanitized": "jerteh__gpt2-orao",
|
||||
"system_instruction": null,
|
||||
"system_instruction_sha": null,
|
||||
"fewshot_as_multiturn": null,
|
||||
"chat_template": null,
|
||||
"chat_template_sha": null,
|
||||
"total_evaluation_time_seconds": "4028.2001800000007"
|
||||
}
|
||||
Reference in New Issue
Block a user