516 lines
17 KiB
Markdown
516 lines
17 KiB
Markdown
---
|
|
license: apache-2.0
|
|
library_name: transformers
|
|
tags:
|
|
- merge
|
|
pipeline_tag: text-generation
|
|
model-index:
|
|
- name: TheTop-5x7B-Instruct-T-v0.1
|
|
results:
|
|
- task:
|
|
type: text-generation
|
|
name: Text Generation
|
|
dataset:
|
|
name: AI2 Reasoning Challenge (25-Shot)
|
|
type: ai2_arc
|
|
config: ARC-Challenge
|
|
split: test
|
|
args:
|
|
num_few_shot: 25
|
|
metrics:
|
|
- type: acc_norm
|
|
value: 73.63
|
|
name: normalized accuracy
|
|
source:
|
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-T-v0.1
|
|
name: Open LLM Leaderboard
|
|
- task:
|
|
type: text-generation
|
|
name: Text Generation
|
|
dataset:
|
|
name: HellaSwag (10-Shot)
|
|
type: hellaswag
|
|
split: validation
|
|
args:
|
|
num_few_shot: 10
|
|
metrics:
|
|
- type: acc_norm
|
|
value: 88.85
|
|
name: normalized accuracy
|
|
source:
|
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-T-v0.1
|
|
name: Open LLM Leaderboard
|
|
- task:
|
|
type: text-generation
|
|
name: Text Generation
|
|
dataset:
|
|
name: MMLU (5-Shot)
|
|
type: cais/mmlu
|
|
config: all
|
|
split: test
|
|
args:
|
|
num_few_shot: 5
|
|
metrics:
|
|
- type: acc
|
|
value: 64.22
|
|
name: accuracy
|
|
source:
|
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-T-v0.1
|
|
name: Open LLM Leaderboard
|
|
- task:
|
|
type: text-generation
|
|
name: Text Generation
|
|
dataset:
|
|
name: TruthfulQA (0-shot)
|
|
type: truthful_qa
|
|
config: multiple_choice
|
|
split: validation
|
|
args:
|
|
num_few_shot: 0
|
|
metrics:
|
|
- type: mc2
|
|
value: 70.78
|
|
source:
|
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-T-v0.1
|
|
name: Open LLM Leaderboard
|
|
- task:
|
|
type: text-generation
|
|
name: Text Generation
|
|
dataset:
|
|
name: Winogrande (5-shot)
|
|
type: winogrande
|
|
config: winogrande_xl
|
|
split: validation
|
|
args:
|
|
num_few_shot: 5
|
|
metrics:
|
|
- type: acc
|
|
value: 85.79
|
|
name: accuracy
|
|
source:
|
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-T-v0.1
|
|
name: Open LLM Leaderboard
|
|
- task:
|
|
type: text-generation
|
|
name: Text Generation
|
|
dataset:
|
|
name: GSM8k (5-shot)
|
|
type: gsm8k
|
|
config: main
|
|
split: test
|
|
args:
|
|
num_few_shot: 5
|
|
metrics:
|
|
- type: acc
|
|
value: 66.49
|
|
name: accuracy
|
|
source:
|
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-T-v0.1
|
|
name: Open LLM Leaderboard
|
|
---
|
|
|
|
Merge of top 7B models with TIES method
|
|
|
|
> mergekit is a toolkit for merging pre-trained language models. mergekit uses an out-of-core approach to perform unreasonably elaborate merges in resource-constrained situations. Merges can be run entirely on CPU or accelerated with as little as 8 GB of VRAM. Many merging algorithms are supported, with more coming as they catch my attention.
|
|
|
|
## Eval
|
|
|
|
|
|
|
|
|
|
```python
|
|
{
|
|
"all": {
|
|
"acc": 0.6487801278765712,
|
|
"acc_stderr": 0.03219011246717665,
|
|
"acc_norm": 0.6479445077777353,
|
|
"acc_norm_stderr": 0.032868022907407396,
|
|
"mc1": 0.5862913096695227,
|
|
"mc1_stderr": 0.0172408618120998,
|
|
"mc2": 0.7078078883926877,
|
|
"mc2_stderr": 0.015097515102384168
|
|
},
|
|
"harness|arc:challenge|25": {
|
|
"acc": 0.7167235494880546,
|
|
"acc_stderr": 0.013167478735134575,
|
|
"acc_norm": 0.7363481228668942,
|
|
"acc_norm_stderr": 0.012875929151297044
|
|
},
|
|
"harness|hellaswag|10": {
|
|
"acc": 0.7321250746863175,
|
|
"acc_stderr": 0.004419469983939178,
|
|
"acc_norm": 0.8884684325831508,
|
|
"acc_norm_stderr": 0.0031414591751392717
|
|
},
|
|
"harness|hendrycksTest-abstract_algebra|5": {
|
|
"acc": 0.31,
|
|
"acc_stderr": 0.04648231987117316,
|
|
"acc_norm": 0.31,
|
|
"acc_norm_stderr": 0.04648231987117316
|
|
},
|
|
"harness|hendrycksTest-anatomy|5": {
|
|
"acc": 0.6518518518518519,
|
|
"acc_stderr": 0.041153246103369526,
|
|
"acc_norm": 0.6518518518518519,
|
|
"acc_norm_stderr": 0.041153246103369526
|
|
},
|
|
"harness|hendrycksTest-astronomy|5": {
|
|
"acc": 0.7039473684210527,
|
|
"acc_stderr": 0.03715062154998904,
|
|
"acc_norm": 0.7039473684210527,
|
|
"acc_norm_stderr": 0.03715062154998904
|
|
},
|
|
"harness|hendrycksTest-business_ethics|5": {
|
|
"acc": 0.61,
|
|
"acc_stderr": 0.04902071300001975,
|
|
"acc_norm": 0.61,
|
|
"acc_norm_stderr": 0.04902071300001975
|
|
},
|
|
"harness|hendrycksTest-clinical_knowledge|5": {
|
|
"acc": 0.7132075471698113,
|
|
"acc_stderr": 0.02783491252754407,
|
|
"acc_norm": 0.7132075471698113,
|
|
"acc_norm_stderr": 0.02783491252754407
|
|
},
|
|
"harness|hendrycksTest-college_biology|5": {
|
|
"acc": 0.75,
|
|
"acc_stderr": 0.03621034121889507,
|
|
"acc_norm": 0.75,
|
|
"acc_norm_stderr": 0.03621034121889507
|
|
},
|
|
"harness|hendrycksTest-college_chemistry|5": {
|
|
"acc": 0.46,
|
|
"acc_stderr": 0.05009082659620333,
|
|
"acc_norm": 0.46,
|
|
"acc_norm_stderr": 0.05009082659620333
|
|
},
|
|
"harness|hendrycksTest-college_computer_science|5": {
|
|
"acc": 0.55,
|
|
"acc_stderr": 0.05,
|
|
"acc_norm": 0.55,
|
|
"acc_norm_stderr": 0.05
|
|
},
|
|
"harness|hendrycksTest-college_mathematics|5": {
|
|
"acc": 0.3,
|
|
"acc_stderr": 0.046056618647183814,
|
|
"acc_norm": 0.3,
|
|
"acc_norm_stderr": 0.046056618647183814
|
|
},
|
|
"harness|hendrycksTest-college_medicine|5": {
|
|
"acc": 0.6589595375722543,
|
|
"acc_stderr": 0.036146654241808254,
|
|
"acc_norm": 0.6589595375722543,
|
|
"acc_norm_stderr": 0.036146654241808254
|
|
},
|
|
"harness|hendrycksTest-college_physics|5": {
|
|
"acc": 0.43137254901960786,
|
|
"acc_stderr": 0.04928099597287534,
|
|
"acc_norm": 0.43137254901960786,
|
|
"acc_norm_stderr": 0.04928099597287534
|
|
},
|
|
"harness|hendrycksTest-computer_security|5": {
|
|
"acc": 0.77,
|
|
"acc_stderr": 0.04229525846816506,
|
|
"acc_norm": 0.77,
|
|
"acc_norm_stderr": 0.04229525846816506
|
|
},
|
|
"harness|hendrycksTest-conceptual_physics|5": {
|
|
"acc": 0.548936170212766,
|
|
"acc_stderr": 0.032529096196131965,
|
|
"acc_norm": 0.548936170212766,
|
|
"acc_norm_stderr": 0.032529096196131965
|
|
},
|
|
"harness|hendrycksTest-econometrics|5": {
|
|
"acc": 0.49122807017543857,
|
|
"acc_stderr": 0.04702880432049615,
|
|
"acc_norm": 0.49122807017543857,
|
|
"acc_norm_stderr": 0.04702880432049615
|
|
},
|
|
"harness|hendrycksTest-electrical_engineering|5": {
|
|
"acc": 0.5517241379310345,
|
|
"acc_stderr": 0.04144311810878152,
|
|
"acc_norm": 0.5517241379310345,
|
|
"acc_norm_stderr": 0.04144311810878152
|
|
},
|
|
"harness|hendrycksTest-elementary_mathematics|5": {
|
|
"acc": 0.4126984126984127,
|
|
"acc_stderr": 0.025355741263055277,
|
|
"acc_norm": 0.4126984126984127,
|
|
"acc_norm_stderr": 0.025355741263055277
|
|
},
|
|
"harness|hendrycksTest-formal_logic|5": {
|
|
"acc": 0.49206349206349204,
|
|
"acc_stderr": 0.044715725362943486,
|
|
"acc_norm": 0.49206349206349204,
|
|
"acc_norm_stderr": 0.044715725362943486
|
|
},
|
|
"harness|hendrycksTest-global_facts|5": {
|
|
"acc": 0.35,
|
|
"acc_stderr": 0.047937248544110196,
|
|
"acc_norm": 0.35,
|
|
"acc_norm_stderr": 0.047937248544110196
|
|
},
|
|
"harness|hendrycksTest-high_school_biology|5": {
|
|
"acc": 0.7967741935483871,
|
|
"acc_stderr": 0.02289168798455496,
|
|
"acc_norm": 0.7967741935483871,
|
|
"acc_norm_stderr": 0.02289168798455496
|
|
},
|
|
"harness|hendrycksTest-high_school_chemistry|5": {
|
|
"acc": 0.5024630541871922,
|
|
"acc_stderr": 0.035179450386910616,
|
|
"acc_norm": 0.5024630541871922,
|
|
"acc_norm_stderr": 0.035179450386910616
|
|
},
|
|
"harness|hendrycksTest-high_school_computer_science|5": {
|
|
"acc": 0.7,
|
|
"acc_stderr": 0.046056618647183814,
|
|
"acc_norm": 0.7,
|
|
"acc_norm_stderr": 0.046056618647183814
|
|
},
|
|
"harness|hendrycksTest-high_school_european_history|5": {
|
|
"acc": 0.7575757575757576,
|
|
"acc_stderr": 0.03346409881055953,
|
|
"acc_norm": 0.7575757575757576,
|
|
"acc_norm_stderr": 0.03346409881055953
|
|
},
|
|
"harness|hendrycksTest-high_school_geography|5": {
|
|
"acc": 0.803030303030303,
|
|
"acc_stderr": 0.028335609732463362,
|
|
"acc_norm": 0.803030303030303,
|
|
"acc_norm_stderr": 0.028335609732463362
|
|
},
|
|
"harness|hendrycksTest-high_school_government_and_politics|5": {
|
|
"acc": 0.9067357512953368,
|
|
"acc_stderr": 0.020986854593289733,
|
|
"acc_norm": 0.9067357512953368,
|
|
"acc_norm_stderr": 0.020986854593289733
|
|
},
|
|
"harness|hendrycksTest-high_school_macroeconomics|5": {
|
|
"acc": 0.6487179487179487,
|
|
"acc_stderr": 0.024203665177902803,
|
|
"acc_norm": 0.6487179487179487,
|
|
"acc_norm_stderr": 0.024203665177902803
|
|
},
|
|
"harness|hendrycksTest-high_school_mathematics|5": {
|
|
"acc": 0.3333333333333333,
|
|
"acc_stderr": 0.02874204090394848,
|
|
"acc_norm": 0.3333333333333333,
|
|
"acc_norm_stderr": 0.02874204090394848
|
|
},
|
|
"harness|hendrycksTest-high_school_microeconomics|5": {
|
|
"acc": 0.6554621848739496,
|
|
"acc_stderr": 0.03086868260412162,
|
|
"acc_norm": 0.6554621848739496,
|
|
"acc_norm_stderr": 0.03086868260412162
|
|
},
|
|
"harness|hendrycksTest-high_school_physics|5": {
|
|
"acc": 0.32450331125827814,
|
|
"acc_stderr": 0.038227469376587525,
|
|
"acc_norm": 0.32450331125827814,
|
|
"acc_norm_stderr": 0.038227469376587525
|
|
},
|
|
"harness|hendrycksTest-high_school_psychology|5": {
|
|
"acc": 0.8403669724770643,
|
|
"acc_stderr": 0.015703498348461763,
|
|
"acc_norm": 0.8403669724770643,
|
|
"acc_norm_stderr": 0.015703498348461763
|
|
},
|
|
"harness|hendrycksTest-high_school_statistics|5": {
|
|
"acc": 0.5046296296296297,
|
|
"acc_stderr": 0.03409825519163572,
|
|
"acc_norm": 0.5046296296296297,
|
|
"acc_norm_stderr": 0.03409825519163572
|
|
},
|
|
"harness|hendrycksTest-high_school_us_history|5": {
|
|
"acc": 0.8235294117647058,
|
|
"acc_stderr": 0.026756401538078962,
|
|
"acc_norm": 0.8235294117647058,
|
|
"acc_norm_stderr": 0.026756401538078962
|
|
},
|
|
"harness|hendrycksTest-high_school_world_history|5": {
|
|
"acc": 0.7721518987341772,
|
|
"acc_stderr": 0.02730348459906944,
|
|
"acc_norm": 0.7721518987341772,
|
|
"acc_norm_stderr": 0.02730348459906944
|
|
},
|
|
"harness|hendrycksTest-human_aging|5": {
|
|
"acc": 0.6816143497757847,
|
|
"acc_stderr": 0.03126580522513713,
|
|
"acc_norm": 0.6816143497757847,
|
|
"acc_norm_stderr": 0.03126580522513713
|
|
},
|
|
"harness|hendrycksTest-human_sexuality|5": {
|
|
"acc": 0.7862595419847328,
|
|
"acc_stderr": 0.0359546161177469,
|
|
"acc_norm": 0.7862595419847328,
|
|
"acc_norm_stderr": 0.0359546161177469
|
|
},
|
|
"harness|hendrycksTest-international_law|5": {
|
|
"acc": 0.7851239669421488,
|
|
"acc_stderr": 0.037494924487096966,
|
|
"acc_norm": 0.7851239669421488,
|
|
"acc_norm_stderr": 0.037494924487096966
|
|
},
|
|
"harness|hendrycksTest-jurisprudence|5": {
|
|
"acc": 0.7777777777777778,
|
|
"acc_stderr": 0.0401910747255735,
|
|
"acc_norm": 0.7777777777777778,
|
|
"acc_norm_stderr": 0.0401910747255735
|
|
},
|
|
"harness|hendrycksTest-logical_fallacies|5": {
|
|
"acc": 0.7423312883435583,
|
|
"acc_stderr": 0.03436150827846917,
|
|
"acc_norm": 0.7423312883435583,
|
|
"acc_norm_stderr": 0.03436150827846917
|
|
},
|
|
"harness|hendrycksTest-machine_learning|5": {
|
|
"acc": 0.42857142857142855,
|
|
"acc_stderr": 0.04697113923010212,
|
|
"acc_norm": 0.42857142857142855,
|
|
"acc_norm_stderr": 0.04697113923010212
|
|
},
|
|
"harness|hendrycksTest-management|5": {
|
|
"acc": 0.7475728155339806,
|
|
"acc_stderr": 0.04301250399690878,
|
|
"acc_norm": 0.7475728155339806,
|
|
"acc_norm_stderr": 0.04301250399690878
|
|
},
|
|
"harness|hendrycksTest-marketing|5": {
|
|
"acc": 0.8846153846153846,
|
|
"acc_stderr": 0.02093019318517933,
|
|
"acc_norm": 0.8846153846153846,
|
|
"acc_norm_stderr": 0.02093019318517933
|
|
},
|
|
"harness|hendrycksTest-medical_genetics|5": {
|
|
"acc": 0.7,
|
|
"acc_stderr": 0.046056618647183814,
|
|
"acc_norm": 0.7,
|
|
"acc_norm_stderr": 0.046056618647183814
|
|
},
|
|
"harness|hendrycksTest-miscellaneous|5": {
|
|
"acc": 0.80970625798212,
|
|
"acc_stderr": 0.014036945850381396,
|
|
"acc_norm": 0.80970625798212,
|
|
"acc_norm_stderr": 0.014036945850381396
|
|
},
|
|
"harness|hendrycksTest-moral_disputes|5": {
|
|
"acc": 0.7369942196531792,
|
|
"acc_stderr": 0.023703099525258172,
|
|
"acc_norm": 0.7369942196531792,
|
|
"acc_norm_stderr": 0.023703099525258172
|
|
},
|
|
"harness|hendrycksTest-moral_scenarios|5": {
|
|
"acc": 0.47150837988826816,
|
|
"acc_stderr": 0.016695329746015796,
|
|
"acc_norm": 0.47150837988826816,
|
|
"acc_norm_stderr": 0.016695329746015796
|
|
},
|
|
"harness|hendrycksTest-nutrition|5": {
|
|
"acc": 0.7189542483660131,
|
|
"acc_stderr": 0.025738854797818733,
|
|
"acc_norm": 0.7189542483660131,
|
|
"acc_norm_stderr": 0.025738854797818733
|
|
},
|
|
"harness|hendrycksTest-philosophy|5": {
|
|
"acc": 0.7170418006430869,
|
|
"acc_stderr": 0.025583062489984813,
|
|
"acc_norm": 0.7170418006430869,
|
|
"acc_norm_stderr": 0.025583062489984813
|
|
},
|
|
"harness|hendrycksTest-prehistory|5": {
|
|
"acc": 0.7407407407407407,
|
|
"acc_stderr": 0.024383665531035457,
|
|
"acc_norm": 0.7407407407407407,
|
|
"acc_norm_stderr": 0.024383665531035457
|
|
},
|
|
"harness|hendrycksTest-professional_accounting|5": {
|
|
"acc": 0.475177304964539,
|
|
"acc_stderr": 0.029790719243829727,
|
|
"acc_norm": 0.475177304964539,
|
|
"acc_norm_stderr": 0.029790719243829727
|
|
},
|
|
"harness|hendrycksTest-professional_law|5": {
|
|
"acc": 0.470013037809648,
|
|
"acc_stderr": 0.01274724896707906,
|
|
"acc_norm": 0.470013037809648,
|
|
"acc_norm_stderr": 0.01274724896707906
|
|
},
|
|
"harness|hendrycksTest-professional_medicine|5": {
|
|
"acc": 0.6691176470588235,
|
|
"acc_stderr": 0.028582709753898445,
|
|
"acc_norm": 0.6691176470588235,
|
|
"acc_norm_stderr": 0.028582709753898445
|
|
},
|
|
"harness|hendrycksTest-professional_psychology|5": {
|
|
"acc": 0.6584967320261438,
|
|
"acc_stderr": 0.019184639328092487,
|
|
"acc_norm": 0.6584967320261438,
|
|
"acc_norm_stderr": 0.019184639328092487
|
|
},
|
|
"harness|hendrycksTest-public_relations|5": {
|
|
"acc": 0.6818181818181818,
|
|
"acc_stderr": 0.044612721759105085,
|
|
"acc_norm": 0.6818181818181818,
|
|
"acc_norm_stderr": 0.044612721759105085
|
|
},
|
|
"harness|hendrycksTest-security_studies|5": {
|
|
"acc": 0.7306122448979592,
|
|
"acc_stderr": 0.02840125202902294,
|
|
"acc_norm": 0.7306122448979592,
|
|
"acc_norm_stderr": 0.02840125202902294
|
|
},
|
|
"harness|hendrycksTest-sociology|5": {
|
|
"acc": 0.835820895522388,
|
|
"acc_stderr": 0.026193923544454125,
|
|
"acc_norm": 0.835820895522388,
|
|
"acc_norm_stderr": 0.026193923544454125
|
|
},
|
|
"harness|hendrycksTest-us_foreign_policy|5": {
|
|
"acc": 0.85,
|
|
"acc_stderr": 0.03588702812826371,
|
|
"acc_norm": 0.85,
|
|
"acc_norm_stderr": 0.03588702812826371
|
|
},
|
|
"harness|hendrycksTest-virology|5": {
|
|
"acc": 0.5542168674698795,
|
|
"acc_stderr": 0.03869543323472101,
|
|
"acc_norm": 0.5542168674698795,
|
|
"acc_norm_stderr": 0.03869543323472101
|
|
},
|
|
"harness|hendrycksTest-world_religions|5": {
|
|
"acc": 0.8245614035087719,
|
|
"acc_stderr": 0.029170885500727665,
|
|
"acc_norm": 0.8245614035087719,
|
|
"acc_norm_stderr": 0.029170885500727665
|
|
},
|
|
"harness|truthfulqa:mc|0": {
|
|
"mc1": 0.5862913096695227,
|
|
"mc1_stderr": 0.0172408618120998,
|
|
"mc2": 0.7078078883926877,
|
|
"mc2_stderr": 0.015097515102384168
|
|
},
|
|
"harness|winogrande|5": {
|
|
"acc": 0.8579321231254933,
|
|
"acc_stderr": 0.009812000391679367
|
|
},
|
|
"harness|gsm8k|5": {
|
|
"acc": 0.6648976497346475,
|
|
"acc_stderr": 0.013001948176422954
|
|
}
|
|
}
|
|
```
|
|
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
|
|
Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_MaziyarPanahi__TheTop-5x7B-Instruct-T-v0.1)
|
|
|
|
| Metric |Value|
|
|
|---------------------------------|----:|
|
|
|Avg. |74.96|
|
|
|AI2 Reasoning Challenge (25-Shot)|73.63|
|
|
|HellaSwag (10-Shot) |88.85|
|
|
|MMLU (5-Shot) |64.22|
|
|
|TruthfulQA (0-shot) |70.78|
|
|
|Winogrande (5-shot) |85.79|
|
|
|GSM8k (5-shot) |66.49|
|
|
|