nm-testing/Llama-2-7b-pruned40-retrained
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: nm-testing/Llama-2-7b-pruned40-retrained
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-06-16 08:19:13 +08:00
commit 5daa9d3078
19 changed files with 94441 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
.gitattributes vendored Normal file
View File

@@ -0,0 +1,39 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
pytorch_model-00001-of-00004.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00002-of-00004.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00003-of-00004.bin filter=lfs diff=lfs merge=lfs -text
pytorch_model-00004-of-00004.bin filter=lfs diff=lfs merge=lfs -text

19
README.md Normal file
View File

@@ -0,0 +1,19 @@
---
base_model: meta-llama/Llama-2-7b-hf
datasets:
- cerebras/SlimPajama-627B
---
Checkpoint of a [Llama 2 7B](https://huggingface.co/meta-llama/Llama-2-7b-hf) model that has had 50% of the parameters pruned in one-shot with SparseGPT, then retrained for 40B tokens from SlimPajama while maintaining sparsity.
* Model: Llama 2
* Size: 7B
* LR: 3.00E-4
* Dataset: SlimPajama
* Retrained tokens: 40B
* Notes: no warmup + decay to 0.0
* Eval Harness:
* CommonSense Reasoning: 62.2 (97.65%)
* Reading Comprehension: 57.7 (98.30%)
* World Knowledge: 42.4 (97.65%)
* Math: 6.1 (74.39%)
* Code: 16.2 (98.78%)

29
arc_challenge_25shot.json Normal file
View File

@@ -0,0 +1,29 @@
{
"results": {
"arc_challenge": {
"acc": 0.4778156996587031,
"acc_stderr": 0.014597001927076136,
"acc_norm": 0.5136518771331058,
"acc_norm_stderr": 0.014605943429860945
}
},
"versions": {
"arc_challenge": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": {
"pretrained": "/network/alexandre/research/cerebras/retrained_s50",
"trust_remote_code": true,
"dtype": "bfloat16"
},
"num_fewshot": 25,
"batch_size": "8",
"batch_sizes": [],
"device": "cuda:3",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}

26
config.json Normal file
View File

@@ -0,0 +1,26 @@
{
"architectures": [
"LlamaForCausalLM"
],
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"tokenizer_class": "LlamaTokenizerFast",
"torch_dtype": "bfloat16",
"transformers_version": "4.33.1",
"use_cache": true,
"vocab_size": 32000
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

27
gsm8k_5shot.json Normal file
View File

@@ -0,0 +1,27 @@
{
"results": {
"gsm8k": {
"acc": 0.10917361637604246,
"acc_stderr": 0.00859008930051115
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": {
"pretrained": "/network/alexandre/research/cerebras/retrained_s50",
"trust_remote_code": true,
"dtype": "bfloat16"
},
"num_fewshot": 5,
"batch_size": "8",
"batch_sizes": [],
"device": "cuda:1",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}

29
hellaswag_10shot.json Normal file
View File

@@ -0,0 +1,29 @@
{
"results": {
"hellaswag": {
"acc": 0.5736904999004182,
"acc_stderr": 0.004935291975579188,
"acc_norm": 0.7689703246365266,
"acc_norm_stderr": 0.004206299481687387
}
},
"versions": {
"hellaswag": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": {
"pretrained": "/network/alexandre/research/cerebras/retrained_s50",
"trust_remote_code": true,
"dtype": "bfloat16"
},
"num_fewshot": 10,
"batch_size": "8",
"batch_sizes": [],
"device": "cuda:2",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}

421
mmlu_5shot.json Normal file
View File

@@ -0,0 +1,421 @@
{
"results": {
"hendrycksTest-abstract_algebra": {
"acc": 0.33,
"acc_stderr": 0.04725815626252605,
"acc_norm": 0.33,
"acc_norm_stderr": 0.04725815626252605
},
"hendrycksTest-anatomy": {
"acc": 0.45185185185185184,
"acc_stderr": 0.04299268905480864,
"acc_norm": 0.45185185185185184,
"acc_norm_stderr": 0.04299268905480864
},
"hendrycksTest-astronomy": {
"acc": 0.42105263157894735,
"acc_stderr": 0.040179012759817494,
"acc_norm": 0.42105263157894735,
"acc_norm_stderr": 0.040179012759817494
},
"hendrycksTest-business_ethics": {
"acc": 0.5,
"acc_stderr": 0.050251890762960605,
"acc_norm": 0.5,
"acc_norm_stderr": 0.050251890762960605
},
"hendrycksTest-clinical_knowledge": {
"acc": 0.4679245283018868,
"acc_stderr": 0.030709486992556545,
"acc_norm": 0.4679245283018868,
"acc_norm_stderr": 0.030709486992556545
},
"hendrycksTest-college_biology": {
"acc": 0.4513888888888889,
"acc_stderr": 0.04161402398403279,
"acc_norm": 0.4513888888888889,
"acc_norm_stderr": 0.04161402398403279
},
"hendrycksTest-college_chemistry": {
"acc": 0.34,
"acc_stderr": 0.04760952285695235,
"acc_norm": 0.34,
"acc_norm_stderr": 0.04760952285695235
},
"hendrycksTest-college_computer_science": {
"acc": 0.37,
"acc_stderr": 0.04852365870939099,
"acc_norm": 0.37,
"acc_norm_stderr": 0.04852365870939099
},
"hendrycksTest-college_mathematics": {
"acc": 0.35,
"acc_stderr": 0.0479372485441102,
"acc_norm": 0.35,
"acc_norm_stderr": 0.0479372485441102
},
"hendrycksTest-college_medicine": {
"acc": 0.4161849710982659,
"acc_stderr": 0.03758517775404947,
"acc_norm": 0.4161849710982659,
"acc_norm_stderr": 0.03758517775404947
},
"hendrycksTest-college_physics": {
"acc": 0.20588235294117646,
"acc_stderr": 0.04023382273617746,
"acc_norm": 0.20588235294117646,
"acc_norm_stderr": 0.04023382273617746
},
"hendrycksTest-computer_security": {
"acc": 0.56,
"acc_stderr": 0.04988876515698589,
"acc_norm": 0.56,
"acc_norm_stderr": 0.04988876515698589
},
"hendrycksTest-conceptual_physics": {
"acc": 0.37872340425531914,
"acc_stderr": 0.03170995606040655,
"acc_norm": 0.37872340425531914,
"acc_norm_stderr": 0.03170995606040655
},
"hendrycksTest-econometrics": {
"acc": 0.3333333333333333,
"acc_stderr": 0.044346007015849245,
"acc_norm": 0.3333333333333333,
"acc_norm_stderr": 0.044346007015849245
},
"hendrycksTest-electrical_engineering": {
"acc": 0.38620689655172413,
"acc_stderr": 0.04057324734419035,
"acc_norm": 0.38620689655172413,
"acc_norm_stderr": 0.04057324734419035
},
"hendrycksTest-elementary_mathematics": {
"acc": 0.2830687830687831,
"acc_stderr": 0.023201392938194974,
"acc_norm": 0.2830687830687831,
"acc_norm_stderr": 0.023201392938194974
},
"hendrycksTest-formal_logic": {
"acc": 0.2222222222222222,
"acc_stderr": 0.037184890068181146,
"acc_norm": 0.2222222222222222,
"acc_norm_stderr": 0.037184890068181146
},
"hendrycksTest-global_facts": {
"acc": 0.34,
"acc_stderr": 0.047609522856952365,
"acc_norm": 0.34,
"acc_norm_stderr": 0.047609522856952365
},
"hendrycksTest-high_school_biology": {
"acc": 0.4838709677419355,
"acc_stderr": 0.028429203176724555,
"acc_norm": 0.4838709677419355,
"acc_norm_stderr": 0.028429203176724555
},
"hendrycksTest-high_school_chemistry": {
"acc": 0.28078817733990147,
"acc_stderr": 0.03161856335358608,
"acc_norm": 0.28078817733990147,
"acc_norm_stderr": 0.03161856335358608
},
"hendrycksTest-high_school_computer_science": {
"acc": 0.43,
"acc_stderr": 0.049756985195624284,
"acc_norm": 0.43,
"acc_norm_stderr": 0.049756985195624284
},
"hendrycksTest-high_school_european_history": {
"acc": 0.5636363636363636,
"acc_stderr": 0.03872592983524754,
"acc_norm": 0.5636363636363636,
"acc_norm_stderr": 0.03872592983524754
},
"hendrycksTest-high_school_geography": {
"acc": 0.4898989898989899,
"acc_stderr": 0.035616254886737454,
"acc_norm": 0.4898989898989899,
"acc_norm_stderr": 0.035616254886737454
},
"hendrycksTest-high_school_government_and_politics": {
"acc": 0.6787564766839378,
"acc_stderr": 0.033699508685490674,
"acc_norm": 0.6787564766839378,
"acc_norm_stderr": 0.033699508685490674
},
"hendrycksTest-high_school_macroeconomics": {
"acc": 0.4,
"acc_stderr": 0.024838811988033165,
"acc_norm": 0.4,
"acc_norm_stderr": 0.024838811988033165
},
"hendrycksTest-high_school_mathematics": {
"acc": 0.2740740740740741,
"acc_stderr": 0.027195934804085622,
"acc_norm": 0.2740740740740741,
"acc_norm_stderr": 0.027195934804085622
},
"hendrycksTest-high_school_microeconomics": {
"acc": 0.3865546218487395,
"acc_stderr": 0.0316314580755238,
"acc_norm": 0.3865546218487395,
"acc_norm_stderr": 0.0316314580755238
},
"hendrycksTest-high_school_physics": {
"acc": 0.2847682119205298,
"acc_stderr": 0.03684881521389023,
"acc_norm": 0.2847682119205298,
"acc_norm_stderr": 0.03684881521389023
},
"hendrycksTest-high_school_psychology": {
"acc": 0.6073394495412844,
"acc_stderr": 0.020937505161201096,
"acc_norm": 0.6073394495412844,
"acc_norm_stderr": 0.020937505161201096
},
"hendrycksTest-high_school_statistics": {
"acc": 0.4027777777777778,
"acc_stderr": 0.03344887382997866,
"acc_norm": 0.4027777777777778,
"acc_norm_stderr": 0.03344887382997866
},
"hendrycksTest-high_school_us_history": {
"acc": 0.5490196078431373,
"acc_stderr": 0.03492406104163613,
"acc_norm": 0.5490196078431373,
"acc_norm_stderr": 0.03492406104163613
},
"hendrycksTest-high_school_world_history": {
"acc": 0.6244725738396625,
"acc_stderr": 0.03152256243091156,
"acc_norm": 0.6244725738396625,
"acc_norm_stderr": 0.03152256243091156
},
"hendrycksTest-human_aging": {
"acc": 0.48878923766816146,
"acc_stderr": 0.033549366530984746,
"acc_norm": 0.48878923766816146,
"acc_norm_stderr": 0.033549366530984746
},
"hendrycksTest-human_sexuality": {
"acc": 0.4961832061068702,
"acc_stderr": 0.04385162325601553,
"acc_norm": 0.4961832061068702,
"acc_norm_stderr": 0.04385162325601553
},
"hendrycksTest-international_law": {
"acc": 0.6446280991735537,
"acc_stderr": 0.04369236326573981,
"acc_norm": 0.6446280991735537,
"acc_norm_stderr": 0.04369236326573981
},
"hendrycksTest-jurisprudence": {
"acc": 0.48148148148148145,
"acc_stderr": 0.04830366024635331,
"acc_norm": 0.48148148148148145,
"acc_norm_stderr": 0.04830366024635331
},
"hendrycksTest-logical_fallacies": {
"acc": 0.4171779141104294,
"acc_stderr": 0.038741028598180814,
"acc_norm": 0.4171779141104294,
"acc_norm_stderr": 0.038741028598180814
},
"hendrycksTest-machine_learning": {
"acc": 0.3125,
"acc_stderr": 0.043994650575715215,
"acc_norm": 0.3125,
"acc_norm_stderr": 0.043994650575715215
},
"hendrycksTest-management": {
"acc": 0.5242718446601942,
"acc_stderr": 0.049449010929737795,
"acc_norm": 0.5242718446601942,
"acc_norm_stderr": 0.049449010929737795
},
"hendrycksTest-marketing": {
"acc": 0.6581196581196581,
"acc_stderr": 0.03107502852650775,
"acc_norm": 0.6581196581196581,
"acc_norm_stderr": 0.03107502852650775
},
"hendrycksTest-medical_genetics": {
"acc": 0.47,
"acc_stderr": 0.050161355804659205,
"acc_norm": 0.47,
"acc_norm_stderr": 0.050161355804659205
},
"hendrycksTest-miscellaneous": {
"acc": 0.5938697318007663,
"acc_stderr": 0.017562037406478916,
"acc_norm": 0.5938697318007663,
"acc_norm_stderr": 0.017562037406478916
},
"hendrycksTest-moral_disputes": {
"acc": 0.4884393063583815,
"acc_stderr": 0.026911898686377906,
"acc_norm": 0.4884393063583815,
"acc_norm_stderr": 0.026911898686377906
},
"hendrycksTest-moral_scenarios": {
"acc": 0.23798882681564246,
"acc_stderr": 0.014242630070574915,
"acc_norm": 0.23798882681564246,
"acc_norm_stderr": 0.014242630070574915
},
"hendrycksTest-nutrition": {
"acc": 0.5065359477124183,
"acc_stderr": 0.028627470550556047,
"acc_norm": 0.5065359477124183,
"acc_norm_stderr": 0.028627470550556047
},
"hendrycksTest-philosophy": {
"acc": 0.5562700964630225,
"acc_stderr": 0.02821768355665231,
"acc_norm": 0.5562700964630225,
"acc_norm_stderr": 0.02821768355665231
},
"hendrycksTest-prehistory": {
"acc": 0.49382716049382713,
"acc_stderr": 0.027818623962583295,
"acc_norm": 0.49382716049382713,
"acc_norm_stderr": 0.027818623962583295
},
"hendrycksTest-professional_accounting": {
"acc": 0.3404255319148936,
"acc_stderr": 0.028267657482650144,
"acc_norm": 0.3404255319148936,
"acc_norm_stderr": 0.028267657482650144
},
"hendrycksTest-professional_law": {
"acc": 0.3318122555410691,
"acc_stderr": 0.012026088259897632,
"acc_norm": 0.3318122555410691,
"acc_norm_stderr": 0.012026088259897632
},
"hendrycksTest-professional_medicine": {
"acc": 0.4963235294117647,
"acc_stderr": 0.030372015885428195,
"acc_norm": 0.4963235294117647,
"acc_norm_stderr": 0.030372015885428195
},
"hendrycksTest-professional_psychology": {
"acc": 0.39215686274509803,
"acc_stderr": 0.01975172650876263,
"acc_norm": 0.39215686274509803,
"acc_norm_stderr": 0.01975172650876263
},
"hendrycksTest-public_relations": {
"acc": 0.5454545454545454,
"acc_stderr": 0.04769300568972743,
"acc_norm": 0.5454545454545454,
"acc_norm_stderr": 0.04769300568972743
},
"hendrycksTest-security_studies": {
"acc": 0.46530612244897956,
"acc_stderr": 0.03193207024425314,
"acc_norm": 0.46530612244897956,
"acc_norm_stderr": 0.03193207024425314
},
"hendrycksTest-sociology": {
"acc": 0.5671641791044776,
"acc_stderr": 0.03503490923673281,
"acc_norm": 0.5671641791044776,
"acc_norm_stderr": 0.03503490923673281
},
"hendrycksTest-us_foreign_policy": {
"acc": 0.68,
"acc_stderr": 0.04688261722621504,
"acc_norm": 0.68,
"acc_norm_stderr": 0.04688261722621504
},
"hendrycksTest-virology": {
"acc": 0.39156626506024095,
"acc_stderr": 0.037998574544796375,
"acc_norm": 0.39156626506024095,
"acc_norm_stderr": 0.037998574544796375
},
"hendrycksTest-world_religions": {
"acc": 0.6549707602339181,
"acc_stderr": 0.03645981377388807,
"acc_norm": 0.6549707602339181,
"acc_norm_stderr": 0.03645981377388807
}
},
"versions": {
"hendrycksTest-abstract_algebra": 1,
"hendrycksTest-anatomy": 1,
"hendrycksTest-astronomy": 1,
"hendrycksTest-business_ethics": 1,
"hendrycksTest-clinical_knowledge": 1,
"hendrycksTest-college_biology": 1,
"hendrycksTest-college_chemistry": 1,
"hendrycksTest-college_computer_science": 1,
"hendrycksTest-college_mathematics": 1,
"hendrycksTest-college_medicine": 1,
"hendrycksTest-college_physics": 1,
"hendrycksTest-computer_security": 1,
"hendrycksTest-conceptual_physics": 1,
"hendrycksTest-econometrics": 1,
"hendrycksTest-electrical_engineering": 1,
"hendrycksTest-elementary_mathematics": 1,
"hendrycksTest-formal_logic": 1,
"hendrycksTest-global_facts": 1,
"hendrycksTest-high_school_biology": 1,
"hendrycksTest-high_school_chemistry": 1,
"hendrycksTest-high_school_computer_science": 1,
"hendrycksTest-high_school_european_history": 1,
"hendrycksTest-high_school_geography": 1,
"hendrycksTest-high_school_government_and_politics": 1,
"hendrycksTest-high_school_macroeconomics": 1,
"hendrycksTest-high_school_mathematics": 1,
"hendrycksTest-high_school_microeconomics": 1,
"hendrycksTest-high_school_physics": 1,
"hendrycksTest-high_school_psychology": 1,
"hendrycksTest-high_school_statistics": 1,
"hendrycksTest-high_school_us_history": 1,
"hendrycksTest-high_school_world_history": 1,
"hendrycksTest-human_aging": 1,
"hendrycksTest-human_sexuality": 1,
"hendrycksTest-international_law": 1,
"hendrycksTest-jurisprudence": 1,
"hendrycksTest-logical_fallacies": 1,
"hendrycksTest-machine_learning": 1,
"hendrycksTest-management": 1,
"hendrycksTest-marketing": 1,
"hendrycksTest-medical_genetics": 1,
"hendrycksTest-miscellaneous": 1,
"hendrycksTest-moral_disputes": 1,
"hendrycksTest-moral_scenarios": 1,
"hendrycksTest-nutrition": 1,
"hendrycksTest-philosophy": 1,
"hendrycksTest-prehistory": 1,
"hendrycksTest-professional_accounting": 1,
"hendrycksTest-professional_law": 1,
"hendrycksTest-professional_medicine": 1,
"hendrycksTest-professional_psychology": 1,
"hendrycksTest-public_relations": 1,
"hendrycksTest-security_studies": 1,
"hendrycksTest-sociology": 1,
"hendrycksTest-us_foreign_policy": 1,
"hendrycksTest-virology": 1,
"hendrycksTest-world_religions": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": {
"pretrained": "/network/alexandre/research/cerebras/retrained_s50",
"trust_remote_code": true,
"dtype": "bfloat16"
},
"num_fewshot": 5,
"batch_size": "4",
"batch_sizes": [],
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}

3
pytorch_model-00001-of-00004.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5d0081ccf624b7afad51dbbddf16e6a151d004de97983155bc4e530b2561eb54
size 7944340669

3
pytorch_model-00002-of-00004.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:15f435a57d2f29bb068d3543d58f27ba42a926f91a664b978d676f92c95f746f
size 7961152715

3
pytorch_model-00003-of-00004.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8cc6499ac3376d973734bf6c689ba28cc4789c733e1149eb59b906a26aeeff74
size 7915015709

3
pytorch_model-00004-of-00004.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:57c9829fe0157a77aec903df157b783513d7798d03583ecc2062a52436d503cf
size 3133271521

330
pytorch_model.bin.index.json Normal file
View File

@@ -0,0 +1,330 @@
{
"metadata": {
"total_size": 26953670656
},
"weight_map": {
"model.embed_tokens.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00004.bin",
"model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
"model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00004.bin",
"model.layers.18.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.18.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
"model.layers.19.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.19.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.19.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.20.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.21.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.25.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.26.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.26.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.27.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.27.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00004.bin",
"model.layers.28.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
"model.layers.28.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00004.bin",
"model.layers.29.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.29.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00004.bin",
"model.layers.30.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.30.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00004.bin",
"model.layers.31.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.layers.31.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
"model.norm.weight": "pytorch_model-00004-of-00004.bin",
"lm_head.weight": "pytorch_model-00004-of-00004.bin"
}
}

23
special_tokens_map.json Normal file
View File

@@ -0,0 +1,23 @@
{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}

93391
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
tokenizer.model (Stored with Git LFS) Normal file
View File

Binary file not shown.

35
tokenizer_config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"add_bos_token": true,
"add_eos_token": false,
"bos_token": {
"__type": "AddedToken",
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"clean_up_tokenization_spaces": false,
"eos_token": {
"__type": "AddedToken",
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"legacy": false,
"model_max_length": 1000000000000000019884624838656,
"pad_token": null,
"padding_side": "right",
"sp_model_kwargs": {},
"tokenizer_class": "LlamaTokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}

29
truthfulqa_mc_0shot.json Normal file
View File

@@ -0,0 +1,29 @@
{
"results": {
"truthfulqa_mc": {
"mc1": 0.24724602203182375,
"mc1_stderr": 0.015102404797359652,
"mc2": 0.38074259720731923,
"mc2_stderr": 0.013465768407546208
}
},
"versions": {
"truthfulqa_mc": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": {
"pretrained": "/network/alexandre/research/cerebras/retrained_s50",
"trust_remote_code": true,
"dtype": "bfloat16"
},
"num_fewshot": 0,
"batch_size": "8",
"batch_sizes": [],
"device": "cuda:2",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}

27
winogrande_5shot.json Normal file
View File

@@ -0,0 +1,27 @@
{
"results": {
"winogrande": {
"acc": 0.7198105761641673,
"acc_stderr": 0.012621707979798499
}
},
"versions": {
"winogrande": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": {
"pretrained": "/network/alexandre/research/cerebras/retrained_s50",
"trust_remote_code": true,
"dtype": "bfloat16"
},
"num_fewshot": 5,
"batch_size": "8",
"batch_sizes": [],
"device": "cuda:1",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}