初始化项目，由ModelHub XC社区提供模型
Model: lomahony/eleuther-pythia6.9b-hh-sft Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
20
README.md
Normal file
20
README.md
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
---
|
||||||
|
language:
|
||||||
|
- en
|
||||||
|
tags:
|
||||||
|
- pytorch
|
||||||
|
- causal-lm
|
||||||
|
- pythia
|
||||||
|
license: apache-2.0
|
||||||
|
datasets:
|
||||||
|
- Anthropic/hh-rlhf
|
||||||
|
---
|
||||||
|
|
||||||
|
[Pythia-6.9b](https://huggingface.co/EleutherAI/pythia-6.9b) supervised finetuned with [Anthropic-hh-rlhf dataset](https://huggingface.co/datasets/Anthropic/hh-rlhf) for 1 epoch.
|
||||||
|
|
||||||
|
[wandb log](https://wandb.ai/pythia_dpo/Pythia_LOM/runs/vd9e3it0?workspace=user-lauraomahony)
|
||||||
|
|
||||||
|
Benchmark evaluations included in repo done using [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness/tree/big-refactor).
|
||||||
|
|
||||||
|
See [Pythia-6.9b](https://huggingface.co/EleutherAI/pythia-6.9b) for model details [(paper)](https://arxiv.org/abs/2101.00027).
|
||||||
|
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
bootstrapping for stddev: perplexity
|
||||||
|
hf (pretrained=EleutherAI/pythia-6.9b), limit: None, num_fewshot: 0, batch_size: 4
|
||||||
|
| Task |Version|Filter| Metric | Value | |Stderr|
|
||||||
|
|--------------|-------|------|---------------|------:|---|-----:|
|
||||||
|
|arc_challenge |Yaml |none |acc | 0.3140|± |0.0136|
|
||||||
|
| | |none |acc_norm | 0.3515|± |0.0140|
|
||||||
|
|arc_easy |Yaml |none |acc | 0.6705|± |0.0096|
|
||||||
|
| | |none |acc_norm | 0.6128|± |0.0100|
|
||||||
|
|boolq |Yaml |none |acc | 0.6352|± |0.0084|
|
||||||
|
|hellaswag |Yaml |none |acc | 0.4798|± |0.0050|
|
||||||
|
| | |none |acc_norm | 0.6389|± |0.0048|
|
||||||
|
|lambada_openai|Yaml |none |perplexity | 4.4566|± |0.1000|
|
||||||
|
| | |none |acc | 0.6713|± |0.0065|
|
||||||
|
|openbookqa |Yaml |none |acc | 0.2560|± |0.0195|
|
||||||
|
| | |none |acc_norm | 0.3720|± |0.0216|
|
||||||
|
|piqa |Yaml |none |acc | 0.7524|± |0.0101|
|
||||||
|
| | |none |acc_norm | 0.7639|± |0.0099|
|
||||||
|
|sciq |Yaml |none |acc | 0.8930|± |0.0098|
|
||||||
|
| | |none |acc_norm | 0.8380|± |0.0117|
|
||||||
|
|wikitext |Yaml |none |word_perplexity|17.6830| | |
|
||||||
|
| | |none |byte_perplexity| 1.6057| | |
|
||||||
|
| | |none |bits_per_byte | 0.6832| | |
|
||||||
|
|winogrande |Yaml |none |acc | 0.6077|± |0.0137|
|
||||||
|
|
||||||
404
base-6.9b-eval-files/EleutherAI-pythia-6.9b-0shot/results.json
Normal file
404
base-6.9b-eval-files/EleutherAI-pythia-6.9b-0shot/results.json
Normal file
@@ -0,0 +1,404 @@
|
|||||||
|
{
|
||||||
|
"results": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"acc,none": 0.31399317406143346,
|
||||||
|
"acc_stderr,none": 0.013562691224726305,
|
||||||
|
"acc_norm,none": 0.3515358361774744,
|
||||||
|
"acc_norm_stderr,none": 0.013952413699600938
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"acc,none": 0.6704545454545454,
|
||||||
|
"acc_stderr,none": 0.00964518419095386,
|
||||||
|
"acc_norm,none": 0.6127946127946128,
|
||||||
|
"acc_norm_stderr,none": 0.009995312065890353
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"acc,none": 0.6351681957186545,
|
||||||
|
"acc_stderr,none": 0.00841944098496366
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"acc,none": 0.4797849034056961,
|
||||||
|
"acc_stderr,none": 0.004985701593897999,
|
||||||
|
"acc_norm,none": 0.6389165504879506,
|
||||||
|
"acc_norm_stderr,none": 0.0047933305256562106
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"perplexity,none": 4.456621453732154,
|
||||||
|
"perplexity_stderr,none": 0.10000881772967252,
|
||||||
|
"acc,none": 0.6712594605084417,
|
||||||
|
"acc_stderr,none": 0.006544612151352774
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"acc,none": 0.256,
|
||||||
|
"acc_stderr,none": 0.019536923574747605,
|
||||||
|
"acc_norm,none": 0.372,
|
||||||
|
"acc_norm_stderr,none": 0.0216371979857224
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"acc,none": 0.7524483133841132,
|
||||||
|
"acc_stderr,none": 0.010069703966857088,
|
||||||
|
"acc_norm,none": 0.763873775843308,
|
||||||
|
"acc_norm_stderr,none": 0.009908965890558214
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"acc,none": 0.893,
|
||||||
|
"acc_stderr,none": 0.009779910359847167,
|
||||||
|
"acc_norm,none": 0.838,
|
||||||
|
"acc_norm_stderr,none": 0.011657267771304427
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"word_perplexity,none": 17.682958079421635,
|
||||||
|
"byte_perplexity,none": 1.6057045697141277,
|
||||||
|
"bits_per_byte,none": 0.6832064787735104
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"acc,none": 0.6077348066298343,
|
||||||
|
"acc_stderr,none": 0.013722400462000888
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"configs": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"task": "arc_challenge",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Challenge",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"task": "arc_easy",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Easy",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"task": "boolq",
|
||||||
|
"group": [
|
||||||
|
"super-glue-lm-eval-v1"
|
||||||
|
],
|
||||||
|
"dataset_path": "super_glue",
|
||||||
|
"dataset_name": "boolq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": [
|
||||||
|
"no",
|
||||||
|
"yes"
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "passage"
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"task": "hellaswag",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "hellaswag",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{% set text = activity_label ~ ': ' ~ ctx_a ~ ' ' ~ ctx_b.capitalize() %}{{text|trim|replace(' [title]', '. ')|regex_replace('\\[.*?\\]', '')|replace(' ', ' ')}}",
|
||||||
|
"doc_to_target": "{{label}}",
|
||||||
|
"doc_to_choice": "{{endings|map('trim')|map('replace', ' [title]', '. ')|map('regex_replace', '\\[.*?\\]', '')|map('replace', ' ', ' ')|list}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": false
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"task": "lambada_openai",
|
||||||
|
"group": [
|
||||||
|
"lambada",
|
||||||
|
"loglikelihood",
|
||||||
|
"perplexity"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/lambada_openai",
|
||||||
|
"dataset_name": "default",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
|
||||||
|
"doc_to_target": "{{' '+text.split(' ')[-1]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "perplexity",
|
||||||
|
"aggregation": "perplexity",
|
||||||
|
"higher_is_better": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{text}}"
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"task": "openbookqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "openbookqa",
|
||||||
|
"dataset_name": "main",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "question_stem",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey.lstrip())}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "question_stem"
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"task": "piqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "piqa",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "Question: {{goal}}\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": "{{[sol1, sol2]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "goal"
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"task": "sciq",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "sciq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": 3,
|
||||||
|
"doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{support}} {{question}}"
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"task": "wikitext",
|
||||||
|
"group": [
|
||||||
|
"perplexity",
|
||||||
|
"loglikelihood_rolling"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/wikitext_document_level",
|
||||||
|
"dataset_name": "wikitext-2-raw-v1",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "",
|
||||||
|
"doc_to_target": "<function wikitext_detokenizer at 0x7ff390401120>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "word_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "byte_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "bits_per_byte"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood_rolling",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{page}}"
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"task": "winogrande",
|
||||||
|
"dataset_path": "winogrande",
|
||||||
|
"dataset_name": "winogrande_xl",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "<function doc_to_text at 0x7ff390401360>",
|
||||||
|
"doc_to_target": "<function doc_to_target at 0x7ff3904016c0>",
|
||||||
|
"doc_to_choice": "<function doc_to_choice at 0x7ff390401a20>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "sentence"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"versions": {
|
||||||
|
"arc_challenge": "Yaml",
|
||||||
|
"arc_easy": "Yaml",
|
||||||
|
"boolq": "Yaml",
|
||||||
|
"hellaswag": "Yaml",
|
||||||
|
"lambada_openai": "Yaml",
|
||||||
|
"openbookqa": "Yaml",
|
||||||
|
"piqa": "Yaml",
|
||||||
|
"sciq": "Yaml",
|
||||||
|
"wikitext": "Yaml",
|
||||||
|
"winogrande": "Yaml"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"model": "hf",
|
||||||
|
"model_args": "pretrained=EleutherAI/pythia-6.9b",
|
||||||
|
"batch_size": "4",
|
||||||
|
"batch_sizes": [],
|
||||||
|
"device": null,
|
||||||
|
"use_cache": null,
|
||||||
|
"limit": null,
|
||||||
|
"bootstrap_iters": 100000
|
||||||
|
},
|
||||||
|
"git_hash": "d1a44c8"
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
bootstrapping for stddev: perplexity
|
||||||
|
hf (pretrained=EleutherAI/pythia-6.9b), limit: None, num_fewshot: 5, batch_size: 4
|
||||||
|
| Task |Version|Filter| Metric | Value | |Stderr|
|
||||||
|
|--------------|-------|------|---------------|------:|---|-----:|
|
||||||
|
|arc_challenge |Yaml |none |acc | 0.3541|± |0.0140|
|
||||||
|
| | |none |acc_norm | 0.3891|± |0.0142|
|
||||||
|
|arc_easy |Yaml |none |acc | 0.6944|± |0.0095|
|
||||||
|
| | |none |acc_norm | 0.7045|± |0.0094|
|
||||||
|
|boolq |Yaml |none |acc | 0.6575|± |0.0083|
|
||||||
|
|hellaswag |Yaml |none |acc | 0.4804|± |0.0050|
|
||||||
|
| | |none |acc_norm | 0.6509|± |0.0048|
|
||||||
|
|lambada_openai|Yaml |none |perplexity | 5.6328|± |0.1331|
|
||||||
|
| | |none |acc | 0.6231|± |0.0068|
|
||||||
|
|openbookqa |Yaml |none |acc | 0.2800|± |0.0201|
|
||||||
|
| | |none |acc_norm | 0.3540|± |0.0214|
|
||||||
|
|piqa |Yaml |none |acc | 0.7644|± |0.0099|
|
||||||
|
| | |none |acc_norm | 0.7633|± |0.0099|
|
||||||
|
|sciq |Yaml |none |acc | 0.9470|± |0.0071|
|
||||||
|
| | |none |acc_norm | 0.9550|± |0.0066|
|
||||||
|
|wikitext |Yaml |none |word_perplexity|17.6830| | |
|
||||||
|
| | |none |byte_perplexity| 1.6057| | |
|
||||||
|
| | |none |bits_per_byte | 0.6832| | |
|
||||||
|
|winogrande |Yaml |none |acc | 0.6069|± |0.0137|
|
||||||
|
|
||||||
404
base-6.9b-eval-files/EleutherAI-pythia-6.9b-5shot/results.json
Normal file
404
base-6.9b-eval-files/EleutherAI-pythia-6.9b-5shot/results.json
Normal file
@@ -0,0 +1,404 @@
|
|||||||
|
{
|
||||||
|
"results": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"acc,none": 0.35409556313993173,
|
||||||
|
"acc_stderr,none": 0.01397545412275655,
|
||||||
|
"acc_norm,none": 0.3890784982935154,
|
||||||
|
"acc_norm_stderr,none": 0.014247309976045609
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"acc,none": 0.6944444444444444,
|
||||||
|
"acc_stderr,none": 0.00945218121359346,
|
||||||
|
"acc_norm,none": 0.7045454545454546,
|
||||||
|
"acc_norm_stderr,none": 0.009361987126556446
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"acc,none": 0.6574923547400612,
|
||||||
|
"acc_stderr,none": 0.008299903219506771
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"acc,none": 0.4803823939454292,
|
||||||
|
"acc_stderr,none": 0.004985939292819588,
|
||||||
|
"acc_norm,none": 0.650866361282613,
|
||||||
|
"acc_norm_stderr,none": 0.004757220449283696
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"perplexity,none": 5.632830092794682,
|
||||||
|
"perplexity_stderr,none": 0.13309255473880457,
|
||||||
|
"acc,none": 0.6231321560256161,
|
||||||
|
"acc_stderr,none": 0.006751444407117095
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"acc,none": 0.28,
|
||||||
|
"acc_stderr,none": 0.020099950647503233,
|
||||||
|
"acc_norm,none": 0.354,
|
||||||
|
"acc_norm_stderr,none": 0.021407582047916447
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"acc,none": 0.764417845484222,
|
||||||
|
"acc_stderr,none": 0.009901067586473907,
|
||||||
|
"acc_norm,none": 0.7633297062023939,
|
||||||
|
"acc_norm_stderr,none": 0.009916841655042809
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"acc,none": 0.947,
|
||||||
|
"acc_stderr,none": 0.007088105617246445,
|
||||||
|
"acc_norm,none": 0.955,
|
||||||
|
"acc_norm_stderr,none": 0.006558812241406067
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"word_perplexity,none": 17.682958079421635,
|
||||||
|
"byte_perplexity,none": 1.6057045697141277,
|
||||||
|
"bits_per_byte,none": 0.6832064787735104
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"acc,none": 0.6069455406471981,
|
||||||
|
"acc_stderr,none": 0.013727276249108442
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"configs": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"task": "arc_challenge",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Challenge",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"task": "arc_easy",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Easy",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"task": "boolq",
|
||||||
|
"group": [
|
||||||
|
"super-glue-lm-eval-v1"
|
||||||
|
],
|
||||||
|
"dataset_path": "super_glue",
|
||||||
|
"dataset_name": "boolq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": [
|
||||||
|
"no",
|
||||||
|
"yes"
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "passage"
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"task": "hellaswag",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "hellaswag",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{% set text = activity_label ~ ': ' ~ ctx_a ~ ' ' ~ ctx_b.capitalize() %}{{text|trim|replace(' [title]', '. ')|regex_replace('\\[.*?\\]', '')|replace(' ', ' ')}}",
|
||||||
|
"doc_to_target": "{{label}}",
|
||||||
|
"doc_to_choice": "{{endings|map('trim')|map('replace', ' [title]', '. ')|map('regex_replace', '\\[.*?\\]', '')|map('replace', ' ', ' ')|list}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": false
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"task": "lambada_openai",
|
||||||
|
"group": [
|
||||||
|
"lambada",
|
||||||
|
"loglikelihood",
|
||||||
|
"perplexity"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/lambada_openai",
|
||||||
|
"dataset_name": "default",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
|
||||||
|
"doc_to_target": "{{' '+text.split(' ')[-1]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "perplexity",
|
||||||
|
"aggregation": "perplexity",
|
||||||
|
"higher_is_better": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{text}}"
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"task": "openbookqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "openbookqa",
|
||||||
|
"dataset_name": "main",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "question_stem",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey.lstrip())}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "question_stem"
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"task": "piqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "piqa",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "Question: {{goal}}\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": "{{[sol1, sol2]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "goal"
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"task": "sciq",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "sciq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": 3,
|
||||||
|
"doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{support}} {{question}}"
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"task": "wikitext",
|
||||||
|
"group": [
|
||||||
|
"perplexity",
|
||||||
|
"loglikelihood_rolling"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/wikitext_document_level",
|
||||||
|
"dataset_name": "wikitext-2-raw-v1",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "",
|
||||||
|
"doc_to_target": "<function wikitext_detokenizer at 0x7fd39a70d120>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "word_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "byte_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "bits_per_byte"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood_rolling",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{page}}"
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"task": "winogrande",
|
||||||
|
"dataset_path": "winogrande",
|
||||||
|
"dataset_name": "winogrande_xl",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "<function doc_to_text at 0x7fd39a70d360>",
|
||||||
|
"doc_to_target": "<function doc_to_target at 0x7fd39a70d6c0>",
|
||||||
|
"doc_to_choice": "<function doc_to_choice at 0x7fd39a70da20>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "sentence"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"versions": {
|
||||||
|
"arc_challenge": "Yaml",
|
||||||
|
"arc_easy": "Yaml",
|
||||||
|
"boolq": "Yaml",
|
||||||
|
"hellaswag": "Yaml",
|
||||||
|
"lambada_openai": "Yaml",
|
||||||
|
"openbookqa": "Yaml",
|
||||||
|
"piqa": "Yaml",
|
||||||
|
"sciq": "Yaml",
|
||||||
|
"wikitext": "Yaml",
|
||||||
|
"winogrande": "Yaml"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"model": "hf",
|
||||||
|
"model_args": "pretrained=EleutherAI/pythia-6.9b",
|
||||||
|
"batch_size": "4",
|
||||||
|
"batch_sizes": [],
|
||||||
|
"device": null,
|
||||||
|
"use_cache": null,
|
||||||
|
"limit": null,
|
||||||
|
"bootstrap_iters": 100000
|
||||||
|
},
|
||||||
|
"git_hash": "d1a44c8"
|
||||||
|
}
|
||||||
24
config.json
Normal file
24
config.json
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"GPTNeoXForCausalLM"
|
||||||
|
],
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_size": 4096,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 16384,
|
||||||
|
"layer_norm_eps": 1e-05,
|
||||||
|
"max_position_embeddings": 2048,
|
||||||
|
"model_type": "gpt_neox",
|
||||||
|
"num_attention_heads": 32,
|
||||||
|
"num_hidden_layers": 32,
|
||||||
|
"rotary_emb_base": 10000,
|
||||||
|
"rotary_pct": 0.25,
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"torch_dtype": "float16",
|
||||||
|
"transformers_version": "4.24.0",
|
||||||
|
"use_cache": true,
|
||||||
|
"use_parallel_residual": true,
|
||||||
|
"vocab_size": 50432
|
||||||
|
}
|
||||||
3
pytorch_model.bin
Normal file
3
pytorch_model.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:8f5419f0be233d31d804941419b36ef8071c15adb0aaf7bf6ea267286b1ab306
|
||||||
|
size 13848994429
|
||||||
24
sft-6.9b-eval-files/sft-pythia-6.9b-0shot-shelloutput.txt
Normal file
24
sft-6.9b-eval-files/sft-pythia-6.9b-0shot-shelloutput.txt
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
bootstrapping for stddev: perplexity
|
||||||
|
hf (pretrained=lomahony/eleuther-pythia6.9b-hh-sft), limit: None, num_fewshot: 0, batch_size: 4
|
||||||
|
| Task |Version|Filter| Metric | Value | |Stderr|
|
||||||
|
|--------------|-------|------|---------------|------:|---|-----:|
|
||||||
|
|arc_challenge |Yaml |none |acc | 0.3080|± |0.0135|
|
||||||
|
| | |none |acc_norm | 0.3379|± |0.0138|
|
||||||
|
|arc_easy |Yaml |none |acc | 0.6789|± |0.0096|
|
||||||
|
| | |none |acc_norm | 0.6048|± |0.0100|
|
||||||
|
|boolq |Yaml |none |acc | 0.6657|± |0.0083|
|
||||||
|
|hellaswag |Yaml |none |acc | 0.4847|± |0.0050|
|
||||||
|
| | |none |acc_norm | 0.6404|± |0.0048|
|
||||||
|
|lambada_openai|Yaml |none |perplexity | 3.9049|± |0.0904|
|
||||||
|
| | |none |acc | 0.6920|± |0.0064|
|
||||||
|
|openbookqa |Yaml |none |acc | 0.2640|± |0.0197|
|
||||||
|
| | |none |acc_norm | 0.3740|± |0.0217|
|
||||||
|
|piqa |Yaml |none |acc | 0.7601|± |0.0100|
|
||||||
|
| | |none |acc_norm | 0.7661|± |0.0099|
|
||||||
|
|sciq |Yaml |none |acc | 0.9010|± |0.0094|
|
||||||
|
| | |none |acc_norm | 0.8290|± |0.0119|
|
||||||
|
|wikitext |Yaml |none |word_perplexity|18.0722| | |
|
||||||
|
| | |none |byte_perplexity| 1.6115| | |
|
||||||
|
| | |none |bits_per_byte | 0.6884| | |
|
||||||
|
|winogrande |Yaml |none |acc | 0.6164|± |0.0137|
|
||||||
|
|
||||||
404
sft-6.9b-eval-files/sft-pythia-6.9b-0shot/results.json
Normal file
404
sft-6.9b-eval-files/sft-pythia-6.9b-0shot/results.json
Normal file
@@ -0,0 +1,404 @@
|
|||||||
|
{
|
||||||
|
"results": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"acc,none": 0.30802047781569963,
|
||||||
|
"acc_stderr,none": 0.013491429517292038,
|
||||||
|
"acc_norm,none": 0.3378839590443686,
|
||||||
|
"acc_norm_stderr,none": 0.013822047922283512
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"acc,none": 0.6788720538720538,
|
||||||
|
"acc_stderr,none": 0.009580787536986797,
|
||||||
|
"acc_norm,none": 0.6047979797979798,
|
||||||
|
"acc_norm_stderr,none": 0.010031894052790976
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"acc,none": 0.6657492354740061,
|
||||||
|
"acc_stderr,none": 0.00825057245508343
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"acc,none": 0.48466440948018324,
|
||||||
|
"acc_stderr,none": 0.004987433862274563,
|
||||||
|
"acc_norm,none": 0.6404102768372834,
|
||||||
|
"acc_norm_stderr,none": 0.004788994060654275
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"perplexity,none": 3.9048816842205616,
|
||||||
|
"perplexity_stderr,none": 0.0904411518306548,
|
||||||
|
"acc,none": 0.6920240636522415,
|
||||||
|
"acc_stderr,none": 0.006431778256505186
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"acc,none": 0.264,
|
||||||
|
"acc_stderr,none": 0.019732885585922094,
|
||||||
|
"acc_norm,none": 0.374,
|
||||||
|
"acc_norm_stderr,none": 0.021660710347204484
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"acc,none": 0.7600652883569097,
|
||||||
|
"acc_stderr,none": 0.009963625892809545,
|
||||||
|
"acc_norm,none": 0.766050054406964,
|
||||||
|
"acc_norm_stderr,none": 0.009877236895137463
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"acc,none": 0.901,
|
||||||
|
"acc_stderr,none": 0.009449248027662744,
|
||||||
|
"acc_norm,none": 0.829,
|
||||||
|
"acc_norm_stderr,none": 0.011912216456264583
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"word_perplexity,none": 18.072185827315998,
|
||||||
|
"byte_perplexity,none": 1.6114783437385283,
|
||||||
|
"bits_per_byte,none": 0.6883848005035924
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"acc,none": 0.6164167324388319,
|
||||||
|
"acc_stderr,none": 0.01366627588953902
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"configs": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"task": "arc_challenge",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Challenge",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"task": "arc_easy",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Easy",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"task": "boolq",
|
||||||
|
"group": [
|
||||||
|
"super-glue-lm-eval-v1"
|
||||||
|
],
|
||||||
|
"dataset_path": "super_glue",
|
||||||
|
"dataset_name": "boolq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": [
|
||||||
|
"no",
|
||||||
|
"yes"
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "passage"
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"task": "hellaswag",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "hellaswag",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{% set text = activity_label ~ ': ' ~ ctx_a ~ ' ' ~ ctx_b.capitalize() %}{{text|trim|replace(' [title]', '. ')|regex_replace('\\[.*?\\]', '')|replace(' ', ' ')}}",
|
||||||
|
"doc_to_target": "{{label}}",
|
||||||
|
"doc_to_choice": "{{endings|map('trim')|map('replace', ' [title]', '. ')|map('regex_replace', '\\[.*?\\]', '')|map('replace', ' ', ' ')|list}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": false
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"task": "lambada_openai",
|
||||||
|
"group": [
|
||||||
|
"lambada",
|
||||||
|
"loglikelihood",
|
||||||
|
"perplexity"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/lambada_openai",
|
||||||
|
"dataset_name": "default",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
|
||||||
|
"doc_to_target": "{{' '+text.split(' ')[-1]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "perplexity",
|
||||||
|
"aggregation": "perplexity",
|
||||||
|
"higher_is_better": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{text}}"
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"task": "openbookqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "openbookqa",
|
||||||
|
"dataset_name": "main",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "question_stem",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey.lstrip())}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "question_stem"
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"task": "piqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "piqa",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "Question: {{goal}}\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": "{{[sol1, sol2]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "goal"
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"task": "sciq",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "sciq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": 3,
|
||||||
|
"doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{support}} {{question}}"
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"task": "wikitext",
|
||||||
|
"group": [
|
||||||
|
"perplexity",
|
||||||
|
"loglikelihood_rolling"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/wikitext_document_level",
|
||||||
|
"dataset_name": "wikitext-2-raw-v1",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "",
|
||||||
|
"doc_to_target": "<function wikitext_detokenizer at 0x7f5ef3dc5120>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "word_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "byte_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "bits_per_byte"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood_rolling",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{page}}"
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"task": "winogrande",
|
||||||
|
"dataset_path": "winogrande",
|
||||||
|
"dataset_name": "winogrande_xl",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "<function doc_to_text at 0x7f5ef3dc5360>",
|
||||||
|
"doc_to_target": "<function doc_to_target at 0x7f5ef3dc56c0>",
|
||||||
|
"doc_to_choice": "<function doc_to_choice at 0x7f5ef3dc5a20>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 0,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "sentence"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"versions": {
|
||||||
|
"arc_challenge": "Yaml",
|
||||||
|
"arc_easy": "Yaml",
|
||||||
|
"boolq": "Yaml",
|
||||||
|
"hellaswag": "Yaml",
|
||||||
|
"lambada_openai": "Yaml",
|
||||||
|
"openbookqa": "Yaml",
|
||||||
|
"piqa": "Yaml",
|
||||||
|
"sciq": "Yaml",
|
||||||
|
"wikitext": "Yaml",
|
||||||
|
"winogrande": "Yaml"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"model": "hf",
|
||||||
|
"model_args": "pretrained=lomahony/eleuther-pythia6.9b-hh-sft",
|
||||||
|
"batch_size": "4",
|
||||||
|
"batch_sizes": [],
|
||||||
|
"device": null,
|
||||||
|
"use_cache": null,
|
||||||
|
"limit": null,
|
||||||
|
"bootstrap_iters": 100000
|
||||||
|
},
|
||||||
|
"git_hash": "d1a44c8"
|
||||||
|
}
|
||||||
24
sft-6.9b-eval-files/sft-pythia-6.9b-5shot-shelloutput.txt
Normal file
24
sft-6.9b-eval-files/sft-pythia-6.9b-5shot-shelloutput.txt
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
bootstrapping for stddev: perplexity
|
||||||
|
hf (pretrained=lomahony/eleuther-pythia6.9b-hh-sft), limit: None, num_fewshot: 5, batch_size: 4
|
||||||
|
| Task |Version|Filter| Metric | Value | |Stderr|
|
||||||
|
|--------------|-------|------|---------------|------:|---|-----:|
|
||||||
|
|arc_challenge |Yaml |none |acc | 0.3584|± |0.0140|
|
||||||
|
| | |none |acc_norm | 0.3848|± |0.0142|
|
||||||
|
|arc_easy |Yaml |none |acc | 0.7104|± |0.0093|
|
||||||
|
| | |none |acc_norm | 0.7134|± |0.0093|
|
||||||
|
|boolq |Yaml |none |acc | 0.6664|± |0.0082|
|
||||||
|
|hellaswag |Yaml |none |acc | 0.4829|± |0.0050|
|
||||||
|
| | |none |acc_norm | 0.6446|± |0.0048|
|
||||||
|
|lambada_openai|Yaml |none |perplexity | 5.0215|± |0.1228|
|
||||||
|
| | |none |acc | 0.6385|± |0.0067|
|
||||||
|
|openbookqa |Yaml |none |acc | 0.2760|± |0.0200|
|
||||||
|
| | |none |acc_norm | 0.3700|± |0.0216|
|
||||||
|
|piqa |Yaml |none |acc | 0.7688|± |0.0098|
|
||||||
|
| | |none |acc_norm | 0.7709|± |0.0098|
|
||||||
|
|sciq |Yaml |none |acc | 0.9520|± |0.0068|
|
||||||
|
| | |none |acc_norm | 0.9540|± |0.0066|
|
||||||
|
|wikitext |Yaml |none |word_perplexity|18.0722| | |
|
||||||
|
| | |none |byte_perplexity| 1.6115| | |
|
||||||
|
| | |none |bits_per_byte | 0.6884| | |
|
||||||
|
|winogrande |Yaml |none |acc | 0.6156|± |0.0137|
|
||||||
|
|
||||||
404
sft-6.9b-eval-files/sft-pythia-6.9b-5shot/results.json
Normal file
404
sft-6.9b-eval-files/sft-pythia-6.9b-5shot/results.json
Normal file
@@ -0,0 +1,404 @@
|
|||||||
|
{
|
||||||
|
"results": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"acc,none": 0.3583617747440273,
|
||||||
|
"acc_stderr,none": 0.014012883334859871,
|
||||||
|
"acc_norm,none": 0.3848122866894198,
|
||||||
|
"acc_norm_stderr,none": 0.014218371065251107
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"acc,none": 0.7104377104377104,
|
||||||
|
"acc_stderr,none": 0.0093068389121739,
|
||||||
|
"acc_norm,none": 0.7133838383838383,
|
||||||
|
"acc_norm_stderr,none": 0.009278551100969293
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"acc,none": 0.6663608562691131,
|
||||||
|
"acc_stderr,none": 0.008246805985556873
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"acc,none": 0.48287193786098387,
|
||||||
|
"acc_stderr,none": 0.004986852842576715,
|
||||||
|
"acc_norm,none": 0.6445927106154152,
|
||||||
|
"acc_norm_stderr,none": 0.004776583530909558
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"perplexity,none": 5.021473818972182,
|
||||||
|
"perplexity_stderr,none": 0.12279168525921821,
|
||||||
|
"acc,none": 0.6384630312439356,
|
||||||
|
"acc_stderr,none": 0.006693546839258681
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"acc,none": 0.276,
|
||||||
|
"acc_stderr,none": 0.020011219298073528,
|
||||||
|
"acc_norm,none": 0.37,
|
||||||
|
"acc_norm_stderr,none": 0.021613289165165785
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"acc,none": 0.7687704026115343,
|
||||||
|
"acc_stderr,none": 0.009837063180625329,
|
||||||
|
"acc_norm,none": 0.7709466811751904,
|
||||||
|
"acc_norm_stderr,none": 0.009804509865175504
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"acc,none": 0.952,
|
||||||
|
"acc_stderr,none": 0.006763264133666668,
|
||||||
|
"acc_norm,none": 0.954,
|
||||||
|
"acc_norm_stderr,none": 0.006627814717380702
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"word_perplexity,none": 18.072185827315998,
|
||||||
|
"byte_perplexity,none": 1.6114783437385283,
|
||||||
|
"bits_per_byte,none": 0.6883848005035924
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"acc,none": 0.6156274664561957,
|
||||||
|
"acc_stderr,none": 0.01367156760083619
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"configs": {
|
||||||
|
"arc_challenge": {
|
||||||
|
"task": "arc_challenge",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Challenge",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"arc_easy": {
|
||||||
|
"task": "arc_easy",
|
||||||
|
"group": [
|
||||||
|
"ai2_arc",
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "ai2_arc",
|
||||||
|
"dataset_name": "ARC-Easy",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "Question: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey)}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
|
||||||
|
},
|
||||||
|
"boolq": {
|
||||||
|
"task": "boolq",
|
||||||
|
"group": [
|
||||||
|
"super-glue-lm-eval-v1"
|
||||||
|
],
|
||||||
|
"dataset_path": "super_glue",
|
||||||
|
"dataset_name": "boolq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": [
|
||||||
|
"no",
|
||||||
|
"yes"
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "passage"
|
||||||
|
},
|
||||||
|
"hellaswag": {
|
||||||
|
"task": "hellaswag",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "hellaswag",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "{% set text = activity_label ~ ': ' ~ ctx_a ~ ' ' ~ ctx_b.capitalize() %}{{text|trim|replace(' [title]', '. ')|regex_replace('\\[.*?\\]', '')|replace(' ', ' ')}}",
|
||||||
|
"doc_to_target": "{{label}}",
|
||||||
|
"doc_to_choice": "{{endings|map('trim')|map('replace', ' [title]', '. ')|map('regex_replace', '\\[.*?\\]', '')|map('replace', ' ', ' ')|list}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": false
|
||||||
|
},
|
||||||
|
"lambada_openai": {
|
||||||
|
"task": "lambada_openai",
|
||||||
|
"group": [
|
||||||
|
"lambada",
|
||||||
|
"loglikelihood",
|
||||||
|
"perplexity"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/lambada_openai",
|
||||||
|
"dataset_name": "default",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
|
||||||
|
"doc_to_target": "{{' '+text.split(' ')[-1]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "perplexity",
|
||||||
|
"aggregation": "perplexity",
|
||||||
|
"higher_is_better": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{text}}"
|
||||||
|
},
|
||||||
|
"openbookqa": {
|
||||||
|
"task": "openbookqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "openbookqa",
|
||||||
|
"dataset_name": "main",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "question_stem",
|
||||||
|
"doc_to_target": "{{choices.label.index(answerKey.lstrip())}}",
|
||||||
|
"doc_to_choice": "{{choices.text}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "question_stem"
|
||||||
|
},
|
||||||
|
"piqa": {
|
||||||
|
"task": "piqa",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "piqa",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "Question: {{goal}}\nAnswer:",
|
||||||
|
"doc_to_target": "label",
|
||||||
|
"doc_to_choice": "{{[sol1, sol2]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "goal"
|
||||||
|
},
|
||||||
|
"sciq": {
|
||||||
|
"task": "sciq",
|
||||||
|
"group": [
|
||||||
|
"multiple_choice"
|
||||||
|
],
|
||||||
|
"dataset_path": "sciq",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
|
||||||
|
"doc_to_target": 3,
|
||||||
|
"doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "acc_norm",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{support}} {{question}}"
|
||||||
|
},
|
||||||
|
"wikitext": {
|
||||||
|
"task": "wikitext",
|
||||||
|
"group": [
|
||||||
|
"perplexity",
|
||||||
|
"loglikelihood_rolling"
|
||||||
|
],
|
||||||
|
"dataset_path": "EleutherAI/wikitext_document_level",
|
||||||
|
"dataset_name": "wikitext-2-raw-v1",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"test_split": "test",
|
||||||
|
"doc_to_text": "",
|
||||||
|
"doc_to_target": "<function wikitext_detokenizer at 0x7fed99361120>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "word_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "byte_perplexity"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metric": "bits_per_byte"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "loglikelihood_rolling",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "{{page}}"
|
||||||
|
},
|
||||||
|
"winogrande": {
|
||||||
|
"task": "winogrande",
|
||||||
|
"dataset_path": "winogrande",
|
||||||
|
"dataset_name": "winogrande_xl",
|
||||||
|
"training_split": "train",
|
||||||
|
"validation_split": "validation",
|
||||||
|
"doc_to_text": "<function doc_to_text at 0x7fed99361360>",
|
||||||
|
"doc_to_target": "<function doc_to_target at 0x7fed993616c0>",
|
||||||
|
"doc_to_choice": "<function doc_to_choice at 0x7fed99361a20>",
|
||||||
|
"description": "",
|
||||||
|
"target_delimiter": " ",
|
||||||
|
"fewshot_delimiter": "\n\n",
|
||||||
|
"num_fewshot": 5,
|
||||||
|
"metric_list": [
|
||||||
|
{
|
||||||
|
"metric": "acc",
|
||||||
|
"aggregation": "mean",
|
||||||
|
"higher_is_better": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"output_type": "multiple_choice",
|
||||||
|
"repeats": 1,
|
||||||
|
"should_decontaminate": true,
|
||||||
|
"doc_to_decontamination_query": "sentence"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"versions": {
|
||||||
|
"arc_challenge": "Yaml",
|
||||||
|
"arc_easy": "Yaml",
|
||||||
|
"boolq": "Yaml",
|
||||||
|
"hellaswag": "Yaml",
|
||||||
|
"lambada_openai": "Yaml",
|
||||||
|
"openbookqa": "Yaml",
|
||||||
|
"piqa": "Yaml",
|
||||||
|
"sciq": "Yaml",
|
||||||
|
"wikitext": "Yaml",
|
||||||
|
"winogrande": "Yaml"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"model": "hf",
|
||||||
|
"model_args": "pretrained=lomahony/eleuther-pythia6.9b-hh-sft",
|
||||||
|
"batch_size": "4",
|
||||||
|
"batch_sizes": [],
|
||||||
|
"device": null,
|
||||||
|
"use_cache": null,
|
||||||
|
"limit": null,
|
||||||
|
"bootstrap_iters": 100000
|
||||||
|
},
|
||||||
|
"git_hash": "d1a44c8"
|
||||||
|
}
|
||||||
5
special_tokens_map.json
Normal file
5
special_tokens_map.json
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"unk_token": "<|endoftext|>"
|
||||||
|
}
|
||||||
100528
tokenizer.json
Normal file
100528
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
9
tokenizer_config.json
Normal file
9
tokenizer_config.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"name_or_path": "EleutherAI/gpt-neox-20b",
|
||||||
|
"special_tokens_map_file": "/admin/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/4e49eadb5d14bd22f314ec3f45b69a87b88c7691/special_tokens_map.json",
|
||||||
|
"tokenizer_class": "GPTNeoXTokenizer",
|
||||||
|
"unk_token": "<|endoftext|>"
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user