From eeb8f0ff2f34ae43156a1656981eed508c62c7f2 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 7 May 2026 06:37:41 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: louisbrulenaudet/Pearl-7B-0210-ties Source: Original Platform --- .gitattributes | 58 +++++++++++ README.md | 160 +++++++++++++++++++++++++++++++ config.json | 26 +++++ configuration.json | 1 + mergekit_config.yml | 25 +++++ model-00001-of-00008.safetensors | 3 + model-00002-of-00008.safetensors | 3 + model-00003-of-00008.safetensors | 3 + model-00004-of-00008.safetensors | 3 + model-00005-of-00008.safetensors | 3 + model-00006-of-00008.safetensors | 3 + model-00007-of-00008.safetensors | 3 + model-00008-of-00008.safetensors | 3 + model.safetensors.index.json | 1 + special_tokens_map.json | 23 +++++ tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 42 ++++++++ 18 files changed, 366 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 mergekit_config.yml create mode 100644 model-00001-of-00008.safetensors create mode 100644 model-00002-of-00008.safetensors create mode 100644 model-00003-of-00008.safetensors create mode 100644 model-00004-of-00008.safetensors create mode 100644 model-00005-of-00008.safetensors create mode 100644 model-00006-of-00008.safetensors create mode 100644 model-00007-of-00008.safetensors create mode 100644 model-00008-of-00008.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6bb6f6e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,58 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text + +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +model-00008-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +model-00005-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +model-00001-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +model-00004-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +model-00002-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +model-00006-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +model-00007-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +model-00003-of-00008.safetensors filter=lfs diff=lfs merge=lfs -text +tokenizer.model filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2e3540b --- /dev/null +++ b/README.md @@ -0,0 +1,160 @@ +--- +tags: +- merge +- mergekit +- louisbrulenaudet/Pearl-7B-slerp +- WizardLM/WizardMath-7B-V1.1 +- cognitivecomputations/WestLake-7B-v2-laser +- CultriX/NeuralTrix-7B-dpo +- chemistry +- biology +- math +base_model: +- louisbrulenaudet/Pearl-7B-slerp +- WizardLM/WizardMath-7B-V1.1 +- cognitivecomputations/WestLake-7B-v2-laser +- CultriX/NeuralTrix-7B-dpo +license: apache-2.0 +language: +- en +library_name: transformers +pipeline_tag: text-generation +model-index: +- name: Pearl-7B-0210-ties + results: + - task: + type: text-generation + metrics: + - name: Average + type: Average + value: 74.66 + - name: ARC + type: ARC + value: 71.08 + - name: GSM8K + type: GSM8K + value: 69.98 + - name: Winogrande + type: Winogrande + value: 83.98 + - name: TruthfulQA + type: TruthfulQA + value: 70.47 + - name: HellaSwag + type: HellaSwag + value: 88.63 + source: + name: Open LLM Leaderboard + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard +--- +
+ +# Pearl-7B-0210-ties, an xtraordinary 7B model + +**03-22-2024 - To date, louisbrulenaudet/Pearl-34B-ties is the "Best 🤝 base merges and moerges model of around 30B" on the Open LLM Leaderboard.** + +Pearl-7B-0210-ties is a merge of the following models: +* [louisbrulenaudet/Pearl-7B-slerp](https://huggingface.co/louisbrulenaudet/Pearl-7B-slerp) +* [WizardLM/WizardMath-7B-V1.1](https://huggingface.co/WizardLM/WizardMath-7B-V1.1) +* [cognitivecomputations/WestLake-7B-v2-laser](https://huggingface.co/cognitivecomputations/WestLake-7B-v2-laser) +* [CultriX/NeuralTrix-7B-dpo](https://huggingface.co/CultriX/NeuralTrix-7B-dpo) + +Evaluation + +The evaluation was performed using the HuggingFace Open LLM Leaderboard. + +| Model | Average | ARC | HellaSwag | MMLU | TruthfulQA | Winogrande | GSM8K | #Params (B) | +|--------------------------------------------------|---------|-------|-----------|-------|------------|------------|-------|--------------| +| louisbrulenaudet/Pearl-34B-ties | 75.48 | 70.99 | 84.83 | 76.63 | 70.32 | 82.64 | 67.48 | 34.39 | +| louisbrulenaudet/Pearl-7B-0211-ties | 75.11 | 71.42 | 88.86 | 63.91 | 71.46 | 84.37 | 70.66 | 7.24 | +| NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO | 73.35 | 71.08 | 87.29 | 72.17 | 54.83 | 83.11 | 71.65 | 46.7 | +| argilla/notus-8x7b-experiment | 73.18 | 70.99 | 87.73 | 71.33 | 65.79 | 81.61 | 61.64 | 46.7 | +| louisbrulenaudet/Pearl-7B-slerp | 72.75 | 68.00 | 87.16 | 64.04 | 62.35 | 81.29 | 73.62 | 7.24 | +| mistralai/Mixtral-8x7B-Instruct-v0.1 | 72.7 | 70.14 | 87.55 | 71.4 | 64.98 | 81.06 | 61.11 | 46.7 | +| microsoft/Orca-2-13b | 61.98 | 60.92 | 79.85 | 60.3 | 56.42 | 76.56 | 37.83 | 13 | +| microsoft/phi-2 | 61.33 | 61.09 | 75.11 | 58.11 | 44.47 | 74.35 | 54.81 | 2.78 | + +### Ties merging + +TIES-Merging is a method designed to facilitate the efficient merging of multiple task-specific models into a consolidated multitask model. It addresses two primary challenges encountered in the process of model merging with a focus on maintaining objectivity. + +One key challenge tackled by TIES-Merging involves addressing redundancy in model parameters. This is achieved by identifying and eliminating redundant parameters within task-specific models, emphasizing the changes made during fine-tuning and selectively retaining the top-k% most significant changes while discarding the rest. + +Another challenge pertains to conflicts arising from disagreements between parameter signs across different models. TIES-Merging resolves these conflicts by creating a unified sign vector representing the most dominant direction of change across all models. + +The TIES-Merging process consists of three steps: + +- Trim: Reduces redundancy in task-specific models by retaining a fraction of the most significant parameters (density parameter) and resetting the remaining parameters to zero. +- Elect Sign: Resolves sign conflicts across different models by creating a unified sign vector based on the most dominant direction (positive or negative) in terms of cumulative magnitude. +- Disjoint Merge: Averages parameter values aligned with the unified sign vector, excluding zero values. + +## Configuration + +```yaml +models: + - model: OpenPipe/mistral-ft-optimized-1227 + - model: louisbrulenaudet/Pearl-7B-slerp + parameters: + density: 0.5 + weight: 0.4 + - model: WizardLM/WizardMath-7B-V1.1 + parameters: + density: 0.5 + weight: 0.2 + - model: cognitivecomputations/WestLake-7B-v2-laser + parameters: + density: 0.5 + weight: 0.2 + - model: CultriX/NeuralTrix-7B-dpo + parameters: + density: 0.5 + weight: 0.2 +merge_method: ties +base_model: OpenPipe/mistral-ft-optimized-1227 +parameters: + normalize: true + int8_mask: true +dtype: float16 +``` + +## Usage + +```python +!pip install -qU transformers accelerate + +from transformers import AutoTokenizer +import transformers +import torch + +model = "louisbrulenaudet/Pearl-7B-0210-ties" +messages = [{"role": "user", "content": "What is a large language model?"}] + +tokenizer = AutoTokenizer.from_pretrained(model) +prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +pipeline = transformers.pipeline( + "text-generation", + model=model, + torch_dtype=torch.float16, + device_map="auto", +) + +outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95) +print(outputs[0]["generated_text"]) +``` + +## Citing & Authors + +If you use this code in your research, please use the following BibTeX entry. + +```BibTeX +@misc{louisbrulenaudet2023, + author = {Louis Brulé Naudet}, + title = {Pearl-7B-0210-ties, an xtraordinary 7B model}, + year = {2023} + howpublished = {\url{https://huggingface.co/louisbrulenaudet/Pearl-7B-0210-ties}}, +} +``` + +## Feedback + +If you have any feedback, please reach out at [louisbrulenaudet@icloud.com](mailto:louisbrulenaudet@icloud.com). \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..0168488 --- /dev/null +++ b/config.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "OpenPipe/mistral-ft-optimized-1227", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 32768, + "model_type": "mistral", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-05, + "rope_theta": 10000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.37.2", + "use_cache": true, + "vocab_size": 32000 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/mergekit_config.yml b/mergekit_config.yml new file mode 100644 index 0000000..e64d695 --- /dev/null +++ b/mergekit_config.yml @@ -0,0 +1,25 @@ + +models: + - model: OpenPipe/mistral-ft-optimized-1227 + - model: louisbrulenaudet/Pearl-7B-slerp + parameters: + density: 0.5 + weight: 0.4 + - model: WizardLM/WizardMath-7B-V1.1 + parameters: + density: 0.5 + weight: 0.2 + - model: cognitivecomputations/WestLake-7B-v2-laser + parameters: + density: 0.5 + weight: 0.2 + - model: CultriX/NeuralTrix-7B-dpo + parameters: + density: 0.5 + weight: 0.2 +merge_method: ties +base_model: OpenPipe/mistral-ft-optimized-1227 +parameters: + normalize: true + int8_mask: true +dtype: float16 diff --git a/model-00001-of-00008.safetensors b/model-00001-of-00008.safetensors new file mode 100644 index 0000000..dc2fea7 --- /dev/null +++ b/model-00001-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4902ba1db8ddd8fa355a67a3bfecd933f592a5b2bb78c44f513b3a1954fbc7ed +size 1912681544 diff --git a/model-00002-of-00008.safetensors b/model-00002-of-00008.safetensors new file mode 100644 index 0000000..78f4551 --- /dev/null +++ b/model-00002-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac4193befb04049225d4b332f2feb71f9a859ce6a6812336fcb5f6004cc5a2f +size 1979781416 diff --git a/model-00003-of-00008.safetensors b/model-00003-of-00008.safetensors new file mode 100644 index 0000000..4e8b6f9 --- /dev/null +++ b/model-00003-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f74dda44067a9b02c3ea4cb654474abca571780043a9dc9e0ea3ceec825d8d0 +size 1946243928 diff --git a/model-00004-of-00008.safetensors b/model-00004-of-00008.safetensors new file mode 100644 index 0000000..fb201b8 --- /dev/null +++ b/model-00004-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96e492364d72d14eb337985a8393bb29123b78b4bacf7a9daebc604806908953 +size 1979781384 diff --git a/model-00005-of-00008.safetensors b/model-00005-of-00008.safetensors new file mode 100644 index 0000000..1bb3b39 --- /dev/null +++ b/model-00005-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbe6e369c9982f7396b545d38f3c74ca11576ab939a8c00f108131c7d0833d6c +size 1862349040 diff --git a/model-00006-of-00008.safetensors b/model-00006-of-00008.safetensors new file mode 100644 index 0000000..1e6e1f3 --- /dev/null +++ b/model-00006-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:233e95c0036ce1070955ab29c5e555102eef8c599f23c99d23f358e0028d5c83 +size 1916866472 diff --git a/model-00007-of-00008.safetensors b/model-00007-of-00008.safetensors new file mode 100644 index 0000000..6eb4905 --- /dev/null +++ b/model-00007-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f889bcf4291a0e6217fc58dfd18c9765ab5372985509cc4d0dab63ccd7edd21 +size 1979781416 diff --git a/model-00008-of-00008.safetensors b/model-00008-of-00008.safetensors new file mode 100644 index 0000000..f7d545e --- /dev/null +++ b/model-00008-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ce2850ecc432ac9c39a2b674dfcf9ac877df031216c22775f06c287f9154469 +size 906012456 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..912c02f --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1 @@ +{"metadata": {"mergekit_version": "0.0.4.1"}, "weight_map": {"model.layers.22.self_attn.o_proj.weight": "model-00001-of-00008.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00008.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00008.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.mlp.down_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.mlp.up_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00008.safetensors", "model.layers.21.input_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.20.mlp.down_proj.weight": "model-00001-of-00008.safetensors", "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00008.safetensors", "model.layers.20.mlp.up_proj.weight": "model-00001-of-00008.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00008.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00008.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00008.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00008.safetensors", "model.layers.20.input_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.19.mlp.down_proj.weight": "model-00001-of-00008.safetensors", "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00008.safetensors", "model.layers.19.mlp.up_proj.weight": "model-00001-of-00008.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00008.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00008.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00008.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00008.safetensors", "model.layers.19.input_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.18.mlp.down_proj.weight": "model-00001-of-00008.safetensors", "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00008.safetensors", "model.layers.18.mlp.up_proj.weight": "model-00001-of-00008.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00008.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00008.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00008.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00008.safetensors", "model.layers.18.input_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00008.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00008.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00008.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00008.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00008.safetensors", "model.layers.17.mlp.down_proj.weight": "model-00002-of-00008.safetensors", "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00008.safetensors", "model.layers.17.mlp.up_proj.weight": "model-00002-of-00008.safetensors", "model.layers.17.input_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.16.mlp.down_proj.weight": "model-00002-of-00008.safetensors", "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00008.safetensors", "model.layers.16.mlp.up_proj.weight": "model-00002-of-00008.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00008.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00008.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00008.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00008.safetensors", "model.layers.16.input_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.15.mlp.down_proj.weight": "model-00002-of-00008.safetensors", "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00008.safetensors", "model.layers.15.mlp.up_proj.weight": "model-00002-of-00008.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00008.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00008.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00008.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00008.safetensors", "model.layers.15.input_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.14.mlp.down_proj.weight": "model-00002-of-00008.safetensors", "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00008.safetensors", "model.layers.14.mlp.up_proj.weight": "model-00002-of-00008.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00008.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00008.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00008.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00008.safetensors", "model.layers.14.input_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.13.mlp.down_proj.weight": "model-00002-of-00008.safetensors", "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00008.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00008.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00008.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00008.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00008.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00008.safetensors", "model.layers.13.mlp.up_proj.weight": "model-00003-of-00008.safetensors", "model.layers.13.input_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.12.mlp.down_proj.weight": "model-00003-of-00008.safetensors", "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00008.safetensors", "model.layers.12.mlp.up_proj.weight": "model-00003-of-00008.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00008.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00008.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00008.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00008.safetensors", "model.layers.12.input_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.11.mlp.down_proj.weight": "model-00003-of-00008.safetensors", "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00008.safetensors", "model.layers.11.mlp.up_proj.weight": "model-00003-of-00008.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00008.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00008.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00008.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00008.safetensors", "model.layers.11.input_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.10.mlp.down_proj.weight": "model-00003-of-00008.safetensors", "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00008.safetensors", "model.layers.10.mlp.up_proj.weight": "model-00003-of-00008.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00008.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00008.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00008.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00008.safetensors", "model.layers.10.input_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.9.mlp.down_proj.weight": "model-00003-of-00008.safetensors", "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00008.safetensors", "model.layers.9.mlp.up_proj.weight": "model-00003-of-00008.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00008.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00008.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00008.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00008.safetensors", "model.layers.9.input_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00008.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00008.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00008.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00008.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00008.safetensors", "model.layers.8.mlp.down_proj.weight": "model-00004-of-00008.safetensors", "model.layers.8.mlp.gate_proj.weight": "model-00004-of-00008.safetensors", "model.layers.8.mlp.up_proj.weight": "model-00004-of-00008.safetensors", "model.layers.8.input_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.7.mlp.down_proj.weight": "model-00004-of-00008.safetensors", "model.layers.7.mlp.gate_proj.weight": "model-00004-of-00008.safetensors", "model.layers.7.mlp.up_proj.weight": "model-00004-of-00008.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00004-of-00008.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00004-of-00008.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00004-of-00008.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00004-of-00008.safetensors", "model.layers.7.input_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.6.mlp.down_proj.weight": "model-00004-of-00008.safetensors", "model.layers.6.mlp.gate_proj.weight": "model-00004-of-00008.safetensors", "model.layers.6.mlp.up_proj.weight": "model-00004-of-00008.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00004-of-00008.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00004-of-00008.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00004-of-00008.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00004-of-00008.safetensors", "model.layers.6.input_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.5.mlp.down_proj.weight": "model-00004-of-00008.safetensors", "model.layers.5.mlp.gate_proj.weight": "model-00004-of-00008.safetensors", "model.layers.5.mlp.up_proj.weight": "model-00004-of-00008.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00004-of-00008.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00004-of-00008.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00004-of-00008.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00004-of-00008.safetensors", "model.layers.5.input_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.4.mlp.down_proj.weight": "model-00004-of-00008.safetensors", "model.layers.4.mlp.gate_proj.weight": "model-00004-of-00008.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00004-of-00008.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00004-of-00008.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00004-of-00008.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00004-of-00008.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00004-of-00008.safetensors", "model.layers.4.mlp.up_proj.weight": "model-00005-of-00008.safetensors", "model.layers.4.input_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.3.mlp.down_proj.weight": "model-00005-of-00008.safetensors", "model.layers.3.mlp.gate_proj.weight": "model-00005-of-00008.safetensors", "model.layers.3.mlp.up_proj.weight": "model-00005-of-00008.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00005-of-00008.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00005-of-00008.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00005-of-00008.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00005-of-00008.safetensors", "model.layers.3.input_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.2.mlp.down_proj.weight": "model-00005-of-00008.safetensors", "model.layers.2.mlp.gate_proj.weight": "model-00005-of-00008.safetensors", "model.layers.2.mlp.up_proj.weight": "model-00005-of-00008.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00005-of-00008.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00005-of-00008.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00005-of-00008.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00005-of-00008.safetensors", "model.layers.2.input_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.1.mlp.down_proj.weight": "model-00005-of-00008.safetensors", "model.layers.1.mlp.gate_proj.weight": "model-00005-of-00008.safetensors", "model.layers.1.mlp.up_proj.weight": "model-00005-of-00008.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00005-of-00008.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00005-of-00008.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00005-of-00008.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00005-of-00008.safetensors", "model.layers.1.input_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.0.mlp.down_proj.weight": "model-00005-of-00008.safetensors", "model.layers.0.mlp.gate_proj.weight": "model-00005-of-00008.safetensors", "model.layers.0.mlp.up_proj.weight": "model-00005-of-00008.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00005-of-00008.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00005-of-00008.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00005-of-00008.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00005-of-00008.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00005-of-00008.safetensors", "model.layers.0.input_layernorm.weight": "model-00005-of-00008.safetensors", "lm_head.weight": "model-00006-of-00008.safetensors", "model.norm.weight": "model-00006-of-00008.safetensors", "model.layers.31.mlp.down_proj.weight": "model-00006-of-00008.safetensors", "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00008.safetensors", "model.layers.31.mlp.up_proj.weight": "model-00006-of-00008.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00008.safetensors", "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00008.safetensors", "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00008.safetensors", "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00008.safetensors", "model.layers.31.input_layernorm.weight": "model-00006-of-00008.safetensors", "model.layers.30.mlp.down_proj.weight": "model-00006-of-00008.safetensors", "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00008.safetensors", "model.layers.30.mlp.up_proj.weight": "model-00006-of-00008.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00008.safetensors", "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00008.safetensors", "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00008.safetensors", "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00008.safetensors", "model.layers.30.input_layernorm.weight": "model-00006-of-00008.safetensors", "model.layers.29.mlp.down_proj.weight": "model-00006-of-00008.safetensors", "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00008.safetensors", "model.layers.29.mlp.up_proj.weight": "model-00006-of-00008.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00008.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00008.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00008.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00008.safetensors", "model.layers.29.input_layernorm.weight": "model-00006-of-00008.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00008.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00008.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00008.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00008.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00008.safetensors", "model.embed_tokens.weight": "model-00006-of-00008.safetensors", "model.layers.28.mlp.down_proj.weight": "model-00007-of-00008.safetensors", "model.layers.28.mlp.gate_proj.weight": "model-00007-of-00008.safetensors", "model.layers.28.mlp.up_proj.weight": "model-00007-of-00008.safetensors", "model.layers.28.input_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.27.mlp.down_proj.weight": "model-00007-of-00008.safetensors", "model.layers.27.mlp.gate_proj.weight": "model-00007-of-00008.safetensors", "model.layers.27.mlp.up_proj.weight": "model-00007-of-00008.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00007-of-00008.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00007-of-00008.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00007-of-00008.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00007-of-00008.safetensors", "model.layers.27.input_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.26.mlp.down_proj.weight": "model-00007-of-00008.safetensors", "model.layers.26.mlp.gate_proj.weight": "model-00007-of-00008.safetensors", "model.layers.26.mlp.up_proj.weight": "model-00007-of-00008.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00007-of-00008.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00007-of-00008.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00007-of-00008.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00007-of-00008.safetensors", "model.layers.26.input_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.25.mlp.down_proj.weight": "model-00007-of-00008.safetensors", "model.layers.25.mlp.gate_proj.weight": "model-00007-of-00008.safetensors", "model.layers.25.mlp.up_proj.weight": "model-00007-of-00008.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00007-of-00008.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00007-of-00008.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00007-of-00008.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00007-of-00008.safetensors", "model.layers.25.input_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.24.mlp.down_proj.weight": "model-00007-of-00008.safetensors", "model.layers.24.mlp.gate_proj.weight": "model-00007-of-00008.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00007-of-00008.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00007-of-00008.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00007-of-00008.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00007-of-00008.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00007-of-00008.safetensors", "model.layers.24.mlp.up_proj.weight": "model-00008-of-00008.safetensors", "model.layers.24.input_layernorm.weight": "model-00008-of-00008.safetensors", "model.layers.23.mlp.down_proj.weight": "model-00008-of-00008.safetensors", "model.layers.23.mlp.gate_proj.weight": "model-00008-of-00008.safetensors", "model.layers.23.mlp.up_proj.weight": "model-00008-of-00008.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00008-of-00008.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00008-of-00008.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00008-of-00008.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00008-of-00008.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00008-of-00008.safetensors", "model.layers.23.input_layernorm.weight": "model-00008-of-00008.safetensors", "model.layers.22.mlp.down_proj.weight": "model-00008-of-00008.safetensors", "model.layers.22.mlp.gate_proj.weight": "model-00008-of-00008.safetensors", "model.layers.22.mlp.up_proj.weight": "model-00008-of-00008.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00008-of-00008.safetensors", "model.layers.22.input_layernorm.weight": "model-00008-of-00008.safetensors"}} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..451134b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..61607de --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc4f0bd70b3709312d9d1d9e5ba674794b6bc5abc17429897a540f93882f25fc +size 1795303 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..8b443ef --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055 +size 493443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..81730cd --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,42 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}