初始化项目，由ModelHub XC社区提供模型

Model: Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended Source: Original Platform
2026-05-16 09:11:29 +08:00
commit 6374a2d94e
14 changed files with 91645 additions and 0 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
--- a/README.md
+++ b/README.md
@@ -0,0 +1,247 @@
+---
+license: apache-2.0
+library_name: transformers
+tags:
+- mergekit
+- merge
+base_model:
+- openchat/openchat-3.5-0106
+model-index:
+- name: OpenChat-3.5-0106_8.99B_40Layers-Appended
+  results:
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: IFEval (0-Shot)
+      type: HuggingFaceH4/ifeval
+      args:
+        num_few_shot: 0
+    metrics:
+    - type: inst_level_strict_acc and prompt_level_strict_acc
+      value: 59.61
+      name: strict accuracy
+    source:
+      url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: BBH (3-Shot)
+      type: BBH
+      args:
+        num_few_shot: 3
+    metrics:
+    - type: acc_norm
+      value: 24.06
+      name: normalized accuracy
+    source:
+      url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: MATH Lvl 5 (4-Shot)
+      type: hendrycks/competition_math
+      args:
+        num_few_shot: 4
+    metrics:
+    - type: exact_match
+      value: 6.8
+      name: exact match
+    source:
+      url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: GPQA (0-shot)
+      type: Idavidrein/gpqa
+      args:
+        num_few_shot: 0
+    metrics:
+    - type: acc_norm
+      value: 7.61
+      name: acc_norm
+    source:
+      url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: MuSR (0-shot)
+      type: TAUR-Lab/MuSR
+      args:
+        num_few_shot: 0
+    metrics:
+    - type: acc_norm
+      value: 11.78
+      name: acc_norm
+    source:
+      url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: MMLU-PRO (5-shot)
+      type: TIGER-Lab/MMLU-Pro
+      config: main
+      split: test
+      args:
+        num_few_shot: 5
+    metrics:
+    - type: acc
+      value: 25.44
+      name: accuracy
+    source:
+      url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended
+      name: Open LLM Leaderboard
+---
+<p align="center">
+  <a href="https://ko-fi.com/pretergeek">Buy me a Ko-Fi</a> •
+  <a href="https://patreon.com/Pretergeek">Support my work using Patreon</a>
+</p>
+
+# OpenChat-3.5-0106_8.99B_40Layers-Appended
+
+This is NOT your usual frankenmerge created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the passthrough merge method, but employing a variation of the Block Expansion method described in the paper [LLaMA Pro: Progressive LLaMA with Block Expansion](https://arxiv.org/abs/2401.02415).
+
+The authors of the paper added new layers interleaved in between the original layers of the model, setting the parameters of the o_proj and down_proj layers to zero. This effectively adds layers that will just output their input (as if they were "transparent") allowing the model to remain functional even without further training. These new layers can then be targeted during training or fine-tuning without risking catastrophic forgetting, if you follow the author's training method to freeze the original layers and only train the new layers.
+
+I used the same method but added the new layers to the end of the model. My rationale is that the level of abstraction increases with each layer of the model. So, while new layers spread along the original layers will help the model to learn new tasks, adding layers to the end of the model and then re-training/fine-tuning the model on tasks it already performs well could improve the models understanding of those task and perform them better by employing more complex reasoning.
+
+This model has not yet received additional training, so it should perform close to the original model.
+
+### Models Merged
+
+The following models were included in the merge:
+* [openchat/openchat-3.5-0106](https://huggingface.co/openchat/openchat-3.5-0106)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+slices:
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [0, 32]
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+merge_method: passthrough
+dtype: bfloat16
+
+```
+# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard)
+Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_Pretergeek__OpenChat-3.5-0106_BlockExpansion-40Layers-End)
+
+|      Metric       |Value|
+|-------------------|----:|
+|Avg.               |22.55|
+|IFEval (0-Shot)    |59.61|
+|BBH (3-Shot)       |24.06|
+|MATH Lvl 5 (4-Shot)| 6.80|
+|GPQA (0-shot)      | 7.61|
+|MuSR (0-shot)      |11.78|
+|MMLU-PRO (5-shot)  |25.44|
+
+## Citation
+```
+@misc{wu2024llamaproprogressivellama,
+      title={LLaMA Pro: Progressive LLaMA with Block Expansion}, 
+      author={Chengyue Wu and Yukang Gan and Yixiao Ge and Zeyu Lu and Jiahao Wang and Ye Feng and Ying Shan and Ping Luo},
+      year={2024},
+      eprint={2401.02415},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2401.02415}, 
+}
+```
--- a/added_tokens.json
+++ b/added_tokens.json
@@ -0,0 +1,4 @@
+{
+  "<|end_of_turn|>": 32000,
+  "<|pad_0|>": 32001
+}
--- a/config.json
+++ b/config.json
@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "openchat/openchat-3.5-0106",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.4",
+  "use_cache": true,
+  "vocab_size": 32002
+}
--- a/mergekit_config.yml
+++ b/mergekit_config.yml
@@ -0,0 +1,86 @@
+slices:
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [0, 32]
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+  - sources:
+    - model: openchat/openchat-3.5-0106
+      layer_range: [31, 32]
+      parameters:
+        scale:
+          - filter: o_proj
+            value: 0.0
+          - filter: down_proj
+            value: 0.0
+          - value: 1.0
+merge_method: passthrough
+dtype: bfloat16
--- a/model-00001-of-00004.safetensors
+++ b/model-00001-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dca56018c0ebd2816b61de2831f329b9dfe335f16ddae30f8fcb8593b5d00efe
+size 4886579776
--- a/model-00002-of-00004.safetensors
+++ b/model-00002-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e74af30c0657c317d4539723543c1dbde68d76d959ecd171979220df68cdf48c
+size 4915916176
--- a/model-00003-of-00004.safetensors
+++ b/model-00003-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bc36bcb43621d5e7031f17f0a416380a1d4247b442e37c57fc725bc15c83b1a
+size 4974659936
--- a/model-00004-of-00004.safetensors
+++ b/model-00004-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c93716cfcf11122a15cf8ef49c05bb1eab1da3d86080fc0194c157af9e05259b
+size 3196175288
--- a/model.safetensors.index.json
+++ b/model.safetensors.index.json
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@@ -0,0 +1,27 @@
+{
+  "additional_special_tokens": [
+    "<|end_of_turn|>",
+    "<|pad_0|>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_turn|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
--- a/tokenizer.json
+++ b/tokenizer.json
--- a/tokenizer.model
+++ b/tokenizer.model
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -0,0 +1,63 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|end_of_turn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|pad_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|end_of_turn|>",
+    "<|pad_0|>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|end_of_turn|>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}