commit a4d6b52d5278221397ee55ed7329d9cd3fdd310c Author: ModelHub XC Date: Thu Jul 2 19:48:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: speakleash/Bielik-Minitron-7B-v3.0-Instruct Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6b87ddd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,54 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text + +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +tokenizer.model filter=lfs diff=lfs merge=lfs -text +model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..69c364d --- /dev/null +++ b/README.md @@ -0,0 +1,204 @@ +--- +license: apache-2.0 +base_model: +- speakleash/Bielik-11B-v3-Base-20250730 +language: +- multilingual +- pl +- en +- sq +- bel +- bs +- bg +- hr +- cs +- da +- et +- fi +- fr +- el +- es +- is +- lt +- nl +- de +- 'no' +- pt +- ru +- ro +- sr +- hbs +- sv +- sk +- sl +- tr +- uk +- hu +- it +- lv +library_name: transformers +inference: + parameters: + temperature: 0.2 +widget: +- messages: + - role: user + content: Co przedstawia polskie godło? +extra_gated_fields: + I agree to be contacted for feedback about Bielik models: checkbox +pipeline_tag: text-generation +--- + +

+ +

+ +# Bielik-Minitron-7B-v3.0-Instruct + +Bielik-Minitron-7B-v3.0-Instruct is a generative text model featuring 7.35 billion parameters. It is an instruct-aligned version of the pruned [Bielik-11B-v3-Base-20250730](https://huggingface.co/speakleash/Bielik-11B-v3-Base-20250730) model. This model stands as a testament to a unique trilateral collaboration between the open-science project SpeakLeash, the High Performance Computing (HPC) center ACK Cyfronet AGH, and NVIDIA. + +By leveraging NVIDIA's Minitron methodology, the team employed the NVIDIA Model Optimizer and the NVIDIA NeMo Framework to execute a sophisticated two-stage compression process involving structured pruning and knowledge distillation. This technical synergy allowed for a significant reduction in parameter count while maintaining high-tier performance benchmarks. + +Developed and trained on a massive multilingual corpus spanning 32 European languages, with a specific emphasis on Polish data curated by the SpeakLeash team, this project utilizes Poland's large-scale computing infrastructure within the PLGrid environment. The training was conducted on the Athena and Helios supercomputers at ACK Cyfronet AGH, supported by computational grant PLG/2024/016951. This access to cutting-edge NVIDIA hardware and software resources was essential for the complex machine learning processes required to produce a model of this scale. As a result, the model exhibits an exceptional ability to process Polish and other European languages, providing accurate responses and performing complex linguistic tasks with high precision and significantly improved inference speed. + +📚 Technical report: [Bielik-Minitron-7B: Compressing Large Language Models via Structured Pruning and Knowledge Distillation for the Polish Language](http://arxiv.org/abs/2603.11881) + +## Model + +The model is a compressed 7.35B parameter version of the [Bielik 11B v3](https://arxiv.org/abs/2601.11579) model, specifically optimized for European languages. Leveraging a two-stage compression methodology inspired by the NVIDIA Minitron approach, we combined structured hybrid pruning and knowledge distillation to reduce the model's parameter count by 33.4% (from 11.04B to 7.35B). We utilized the NVIDIA Model Optimizer for structural pruning and the NVIDIA NeMo Framework for logit-based distillation to facilitate quality recovery. Following distillation, the model underwent a rigorous alignment pipeline consisting of Supervised Fine-Tuning (SFT), Direct Preference Optimization ([DPO-P](https://arxiv.org/abs/2402.13228)), and Reinforcement Learning via Group Relative Policy Optimization (GRPO). The final model recovered approximately 90% of the baseline model's performance while providing up to a 50% inference speedup. + +The post-distillation pipeline was designed to mirror the treatment of the [Bielik-11B-v3.0-Instruct](https://huggingface.co/speakleash/Bielik-11B-v3.0-Instruct) model, ensuring that efficiency gains from pruning did not come at the expense of instruction-following precision or safety. + +Supervised Fine-Tuning and Preference Alignment (DPO-P) stages were conducted using [ALLaMo](https://github.com/chrisociepa/allamo), an original open-source framework implemented by [Krzysztof Ociepa](https://www.linkedin.com/in/krzysztof-ociepa-44886550/). This framework allows users to train language models with architectures similar to LLaMA and Mistral in a fast and efficient way. + +The Reinforcement Learning stage utilized [Group Relative Policy Optimization (GRPO)](https://arxiv.org/abs/2402.03300) and its variant, [Dr. GRPO](https://arxiv.org/abs/2503.20783); these were chosen to improve token efficiency by reducing the tendency of models to artificially increase response length to maximize rewards. RL training was executed using the [Volcano Engine Reinforcement Learning (VERL)](https://arxiv.org/abs/2409.19256) framework, providing a scalable and modular environment. The training corpus contained curated problems spanning logic, STEM, mathematics, and tool-use domains. All samples were selected based on the availability of Reinforcement Learning from Verifiable Rewards (RLVR), ensuring that each problem had a definitive, verifiable solution. + + +### Model description: + +* **Developed by:** [SpeakLeash](https://speakleash.org/) & [ACK Cyfronet AGH](https://www.cyfronet.pl/) +* **Language:** Multilingual (32 European languages, optimized for Polish) +* **Model type:** causal decoder-only +* **Pruned and finetuned from:** [Bielik-11B-v3-Base-20250730](https://huggingface.co/speakleash/Bielik-11B-v3-Base-20250730) +* **License:** Apache 2.0 + + +### Chat template + +Bielik-Minitron-7B-v3.0-Instruct uses [ChatML](https://github.com/cognitivecomputations/OpenChatML) as the prompt format. + +E.g. +``` +prompt = "<|im_start|> user\nJakie mamy pory roku?<|im_end|> \n<|im_start|> assistant\n" +completion = "W Polsce mamy 4 pory roku: wiosna, lato, jesień i zima.<|im_end|> \n" +``` + +This format is available as a [chat template](https://huggingface.co/docs/transformers/main/chat_templating) via the `apply_chat_template()` method: + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +device = "cuda" # the device to load the model onto + +model_name = "speakleash/Bielik-Minitron-7B-v3.0-Instruct" + +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16) + +messages = [ + {"role": "system", "content": "Odpowiadaj krótko, precyzyjnie i wyłącznie w języku polskim."}, + {"role": "user", "content": "Jakie mamy pory roku w Polsce?"}, + {"role": "assistant", "content": "W Polsce mamy 4 pory roku: wiosna, lato, jesień i zima."}, + {"role": "user", "content": "Która jest najcieplejsza?"} +] + +input_ids = tokenizer.apply_chat_template(messages, return_tensors="pt") + +model_inputs = input_ids.to(device) +model.to(device) + +generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True) +decoded = tokenizer.batch_decode(generated_ids) +print(decoded[0]) +``` + +Fully formated input conversation by apply_chat_template from previous example: + +``` +<|im_start|> system +Odpowiadaj krótko, precyzyjnie i wyłącznie w języku polskim.<|im_end|> +<|im_start|> user +Jakie mamy pory roku w Polsce?<|im_end|> +<|im_start|> assistant +W Polsce mamy 4 pory roku: wiosna, lato, jesień i zima.<|im_end|> +<|im_start|> user +Która jest najcieplejsza?<|im_end|> +``` + +## Limitations and Biases + +Bielik-Minitron-7B-v3.0-Instruct is a quick demonstration that the base model can be easily fine-tuned to achieve compelling and promising performance. It does not have any moderation mechanisms. We're looking forward to engaging with the community in ways to make the model respect guardrails, allowing for deployment in environments requiring moderated outputs. + +Bielik-Minitron-7B-v3.0-Instruct can produce factually incorrect output, and should not be relied on to produce factually accurate data. Bielik-Minitron-7B-v3.0-Instruct was trained on various public datasets. While great efforts have been taken to clear the training data, it is possible that this model can generate lewd, false, biased or otherwise offensive outputs. + +## Responsible for training the model + +* [Remigiusz Kinas](https://www.linkedin.com/in/remigiusz-kinas/)Bielik.AI - conceptualizing, coordinating RL trainings, data preparation, benchmarking and quantizations +* [Paweł Kiszczak](https://www.linkedin.com/in/paveu-kiszczak/)Bielik.AI - pruning and distillation supervision +* [Sergio P. Perez](https://www.linkedin.com/in/sergiopp/)NVIDIA - conceptualizing, benchmarking and quantizations +* [Krzysztof Ociepa](https://www.linkedin.com/in/krzysztof-ociepa-44886550/)Bielik.AI - team leadership, conceptualizing, data preparation, process optimization and oversight of training +* [Łukasz Flis](https://www.linkedin.com/in/lukasz-flis-0a39631/)Cyfronet AGH - coordinating and supervising the training +* [Adrian Gwoździej](https://www.linkedin.com/in/adrgwo/)Bielik.AI - data preparation and ensuring data quality +* [Krzysztof Wróbel](https://www.linkedin.com/in/wrobelkrzysztof/)Bielik.AI - benchmarks + + +The model could not have been created without the commitment and work of the entire SpeakLeash team, whose contribution is invaluable. Thanks to the hard work of many individuals, it was possible to gather a large amount of content in Polish and establish collaboration between the open-science SpeakLeash project and the HPC center: ACK Cyfronet AGH. Individuals who contributed to the creation of the model: +[Sebastian Kondracki](https://www.linkedin.com/in/sebastian-kondracki/), +[Marek Magryś](https://www.linkedin.com/in/magrys/), +[Igor Ciuciura](https://www.linkedin.com/in/igor-ciuciura-1763b52a6/), +[Szymon Baczyński](https://www.linkedin.com/in/szymon-baczynski/), +[Dominika Basaj](https://www.linkedin.com/in/dominika-basaj/), +[Kuba Sołtys](https://www.linkedin.com/in/qooba/), +[Karol Jezierski](https://www.linkedin.com/in/karol-jezierski/), +[Jan Sowa](https://www.linkedin.com/in/janpiotrsowa/), +[Anna Przybył](https://www.linkedin.com/in/annaprzybyl/), +[Agnieszka Ratajska](https://www.linkedin.com/in/agnieszka-ratajska/), +[Witold Wydmański](https://www.linkedin.com/in/witold-wydmanski/). + +We gratefully acknowledge Polish high-performance computing infrastructure PLGrid (HPC Center: ACK Cyfronet AGH) for providing computer facilities and support within computational grant no. PLG/2024/016951. + +## Legal Aspects + +EU AI Act Transparency Documentation: [Bielik 11B v3 EU Public Summary.pdf](https://bit.ly/4qDcE81) + +## Data Protection and Copyright Requests + +For removal requests of personally identifiable information (PII) or of copyrighted content, please contact the respective dataset owners or us directly: [biuro@speakleash.org.pl](mailto:biuro@speakleash.org.pl). + +## Citation +Please cite this model using the following format: + +``` +@misc{kinas2026bielikminitron7bcompressinglargelanguage, + title={Bielik-Minitron-7B: Compressing Large Language Models via Structured Pruning and Knowledge Distillation for the Polish Language}, + author={Remigiusz Kinas and Paweł Kiszczak and Sergio P. Perez and Krzysztof Ociepa and Łukasz Flis and Krzysztof Wróbel and Adrian Gwoździej}, + year={2026}, + eprint={2603.11881}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2603.11881}, +} +@misc{BielikMinitron7Bv3i, + title = {Bielik-Minitron-7B-v3.0-Instruct model card}, + author = {Kinas, Remigiusz and Paweł, Kiszczak and Perez, Sergio and Ociepa, Krzysztof and Flis, Łukasz and Gwoździej, Adrian and Wróbel, Krzysztof and {Bielik.AI Team} and {Cyfronet Team} and {NVIDIA Team}}, + year = {2026}, + url = {https://huggingface.co/speakleash/Bielik-Minitron-7B-v3.0-Instruct}, + note = {Accessed: 2026-03-19}, % change this date + urldate = {2026-03-19} % change this date +} +``` + +## Contact Us + +If you have any questions or suggestions, please use the discussion tab. If you want to contact us directly, join our [Discord SpeakLeash](https://discord.gg/pv4brQMDTy). \ No newline at end of file diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..9c948b9 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,130 @@ +{ + "": 32008, + "": 32006, + "": 32007, + "": 32005, + "<|control_100|>": 32099, + "<|control_101|>": 32100, + "<|control_102|>": 32101, + "<|control_103|>": 32102, + "<|control_104|>": 32103, + "<|control_105|>": 32104, + "<|control_106|>": 32105, + "<|control_107|>": 32106, + "<|control_108|>": 32107, + "<|control_109|>": 32108, + "<|control_10|>": 32009, + "<|control_110|>": 32109, + "<|control_111|>": 32110, + "<|control_112|>": 32111, + "<|control_113|>": 32112, + "<|control_114|>": 32113, + "<|control_115|>": 32114, + "<|control_116|>": 32115, + "<|control_117|>": 32116, + "<|control_118|>": 32117, + "<|control_119|>": 32118, + "<|control_11|>": 32010, + "<|control_120|>": 32119, + "<|control_121|>": 32120, + "<|control_122|>": 32121, + "<|control_123|>": 32122, + "<|control_124|>": 32123, + "<|control_125|>": 32124, + "<|control_126|>": 32125, + "<|control_127|>": 32126, + "<|control_128|>": 32127, + "<|control_12|>": 32011, + "<|control_13|>": 32012, + "<|control_14|>": 32013, + "<|control_15|>": 32014, + "<|control_16|>": 32015, + "<|control_17|>": 32016, + "<|control_18|>": 32017, + "<|control_19|>": 32018, + "<|control_20|>": 32019, + "<|control_21|>": 32020, + "<|control_22|>": 32021, + "<|control_23|>": 32022, + "<|control_24|>": 32023, + "<|control_25|>": 32024, + "<|control_26|>": 32025, + "<|control_27|>": 32026, + "<|control_28|>": 32027, + "<|control_29|>": 32028, + "<|control_30|>": 32029, + "<|control_31|>": 32030, + "<|control_32|>": 32031, + "<|control_33|>": 32032, + "<|control_34|>": 32033, + "<|control_35|>": 32034, + "<|control_36|>": 32035, + "<|control_37|>": 32036, + "<|control_38|>": 32037, + "<|control_39|>": 32038, + "<|control_40|>": 32039, + "<|control_41|>": 32040, + "<|control_42|>": 32041, + "<|control_43|>": 32042, + "<|control_44|>": 32043, + "<|control_45|>": 32044, + "<|control_46|>": 32045, + "<|control_47|>": 32046, + "<|control_48|>": 32047, + "<|control_49|>": 32048, + "<|control_50|>": 32049, + "<|control_51|>": 32050, + "<|control_52|>": 32051, + "<|control_53|>": 32052, + "<|control_54|>": 32053, + "<|control_55|>": 32054, + "<|control_56|>": 32055, + "<|control_57|>": 32056, + "<|control_58|>": 32057, + "<|control_59|>": 32058, + "<|control_60|>": 32059, + "<|control_61|>": 32060, + "<|control_62|>": 32061, + "<|control_63|>": 32062, + "<|control_64|>": 32063, + "<|control_65|>": 32064, + "<|control_66|>": 32065, + "<|control_67|>": 32066, + "<|control_68|>": 32067, + "<|control_69|>": 32068, + "<|control_70|>": 32069, + "<|control_71|>": 32070, + "<|control_72|>": 32071, + "<|control_73|>": 32072, + "<|control_74|>": 32073, + "<|control_75|>": 32074, + "<|control_76|>": 32075, + "<|control_77|>": 32076, + "<|control_78|>": 32077, + "<|control_79|>": 32078, + "<|control_80|>": 32079, + "<|control_81|>": 32080, + "<|control_82|>": 32081, + "<|control_83|>": 32082, + "<|control_84|>": 32083, + "<|control_85|>": 32084, + "<|control_86|>": 32085, + "<|control_87|>": 32086, + "<|control_88|>": 32087, + "<|control_89|>": 32088, + "<|control_90|>": 32089, + "<|control_91|>": 32090, + "<|control_92|>": 32091, + "<|control_93|>": 32092, + "<|control_94|>": 32093, + "<|control_95|>": 32094, + "<|control_96|>": 32095, + "<|control_97|>": 32096, + "<|control_98|>": 32097, + "<|control_99|>": 32098, + "<|function_call|>": 32004, + "<|function_list|>": 32002, + "<|function_output|>": 32003, + "<|im_end|>": 32001, + "<|im_start|>": 32000 +} diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..463f9fd --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,4 @@ +{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..d3125b0 --- /dev/null +++ b/config.json @@ -0,0 +1,33 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": [ + 32001, + 2 + ], + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11264, + "max_position_embeddings": 32768, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 2, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 1000000, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.3", + "use_cache": true, + "vocab_size": 32128 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..eccfdc9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,10 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": [ + 32001, + 2 + ], + "pad_token_id": 2, + "transformers_version": "4.57.3" +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..dcb2ff6 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1b792f6c4dcb5f1433fb3b280627b69b2b99cf3642bfe44452f45ec7d30af9 +size 4922267168 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..f267fa1 --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca40989b32f5b06cf6c5f1aea4b66de2cb415e35d7548a5fd60a1d367b0a13af +size 4957869840 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..c74c4b3 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:392295974b913c8cf05244f563423232c5e3b6078f549afc9655765952f08e57 +size 4949530304 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..15e465d --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e03298c5c50350ca8b7e2285256d41cfa1fd9f0fa61aac6fca39722db8f7a598 +size 125829384 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..b3b5eed --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,370 @@ +{ + "metadata": { + "total_size": 14955454464 + }, + "weight_map": { + "lm_head.weight": "model-00001-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.norm.weight": "model-00002-of-00004.safetensors" + } +} diff --git a/params.json b/params.json new file mode 100644 index 0000000..5d59499 --- /dev/null +++ b/params.json @@ -0,0 +1,11 @@ +{ + "dim": 4096, + "n_layers": 40, + "head_dim": 128, + "hidden_dim": 11264, + "n_heads": 32, + "n_kv_heads": 8, + "norm_eps": 1e-05, + "vocab_size": 32128, + "rope_theta": 1000000.0 +} diff --git a/speakleash_cyfronet.png b/speakleash_cyfronet.png new file mode 100644 index 0000000..482f06d Binary files /dev/null and b/speakleash_cyfronet.png differ diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..5e7e2a1 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,160 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|function_list|>", + "<|function_output|>", + "<|function_call|>", + "", + "", + "", + "", + "<|control_10|>", + "<|control_11|>", + "<|control_12|>", + "<|control_13|>", + "<|control_14|>", + "<|control_15|>", + "<|control_16|>", + "<|control_17|>", + "<|control_18|>", + "<|control_19|>", + "<|control_20|>", + "<|control_21|>", + "<|control_22|>", + "<|control_23|>", + "<|control_24|>", + "<|control_25|>", + "<|control_26|>", + "<|control_27|>", + "<|control_28|>", + "<|control_29|>", + "<|control_30|>", + "<|control_31|>", + "<|control_32|>", + "<|control_33|>", + "<|control_34|>", + "<|control_35|>", + "<|control_36|>", + "<|control_37|>", + "<|control_38|>", + "<|control_39|>", + "<|control_40|>", + "<|control_41|>", + "<|control_42|>", + "<|control_43|>", + "<|control_44|>", + "<|control_45|>", + "<|control_46|>", + "<|control_47|>", + "<|control_48|>", + "<|control_49|>", + "<|control_50|>", + "<|control_51|>", + "<|control_52|>", + "<|control_53|>", + "<|control_54|>", + "<|control_55|>", + "<|control_56|>", + "<|control_57|>", + "<|control_58|>", + "<|control_59|>", + "<|control_60|>", + "<|control_61|>", + "<|control_62|>", + "<|control_63|>", + "<|control_64|>", + "<|control_65|>", + "<|control_66|>", + "<|control_67|>", + "<|control_68|>", + "<|control_69|>", + "<|control_70|>", + "<|control_71|>", + "<|control_72|>", + "<|control_73|>", + "<|control_74|>", + "<|control_75|>", + "<|control_76|>", + "<|control_77|>", + "<|control_78|>", + "<|control_79|>", + "<|control_80|>", + "<|control_81|>", + "<|control_82|>", + "<|control_83|>", + "<|control_84|>", + "<|control_85|>", + "<|control_86|>", + "<|control_87|>", + "<|control_88|>", + "<|control_89|>", + "<|control_90|>", + "<|control_91|>", + "<|control_92|>", + "<|control_93|>", + "<|control_94|>", + "<|control_95|>", + "<|control_96|>", + "<|control_97|>", + "<|control_98|>", + "<|control_99|>", + "<|control_100|>", + "<|control_101|>", + "<|control_102|>", + "<|control_103|>", + "<|control_104|>", + "<|control_105|>", + "<|control_106|>", + "<|control_107|>", + "<|control_108|>", + "<|control_109|>", + "<|control_110|>", + "<|control_111|>", + "<|control_112|>", + "<|control_113|>", + "<|control_114|>", + "<|control_115|>", + "<|control_116|>", + "<|control_117|>", + "<|control_118|>", + "<|control_119|>", + "<|control_120|>", + "<|control_121|>", + "<|control_122|>", + "<|control_123|>", + "<|control_124|>", + "<|control_125|>", + "<|control_126|>", + "<|control_127|>", + "<|control_128|>" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..8755ad1 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3574b108114217bdfc00cb1de99bd84a71b218a360dc30671669c1e50c53604 +size 3529655 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..8b443ef --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055 +size 493443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..df4d7fd --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,1198 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": true, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32000": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32001": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32002": { + "content": "<|function_list|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32003": { + "content": "<|function_output|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32004": { + "content": "<|function_call|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32005": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32006": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32007": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32008": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32009": { + "content": "<|control_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32010": { + "content": "<|control_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32011": { + "content": "<|control_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32012": { + "content": "<|control_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32013": { + "content": "<|control_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32014": { + "content": "<|control_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32015": { + "content": "<|control_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32016": { + "content": "<|control_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32017": { + "content": "<|control_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32018": { + "content": "<|control_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32019": { + "content": "<|control_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32020": { + "content": "<|control_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32021": { + "content": "<|control_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32022": { + "content": "<|control_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32023": { + "content": "<|control_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32024": { + "content": "<|control_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32025": { + "content": "<|control_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32026": { + "content": "<|control_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32027": { + "content": "<|control_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32028": { + "content": "<|control_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32029": { + "content": "<|control_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32030": { + "content": "<|control_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32031": { + "content": "<|control_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32032": { + "content": "<|control_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32033": { + "content": "<|control_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32034": { + "content": "<|control_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32035": { + "content": "<|control_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32036": { + "content": "<|control_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32037": { + "content": "<|control_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32038": { + "content": "<|control_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32039": { + "content": "<|control_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32040": { + "content": "<|control_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32041": { + "content": "<|control_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32042": { + "content": "<|control_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32043": { + "content": "<|control_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32044": { + "content": "<|control_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32045": { + "content": "<|control_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32046": { + "content": "<|control_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32047": { + "content": "<|control_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32048": { + "content": "<|control_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32049": { + "content": "<|control_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32050": { + "content": "<|control_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32051": { + "content": "<|control_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32052": { + "content": "<|control_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32053": { + "content": "<|control_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32054": { + "content": "<|control_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32055": { + "content": "<|control_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32056": { + "content": "<|control_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32057": { + "content": "<|control_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32058": { + "content": "<|control_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32059": { + "content": "<|control_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32060": { + "content": "<|control_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32061": { + "content": "<|control_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32062": { + "content": "<|control_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32063": { + "content": "<|control_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32064": { + "content": "<|control_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32065": { + "content": "<|control_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32066": { + "content": "<|control_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32067": { + "content": "<|control_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32068": { + "content": "<|control_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32069": { + "content": "<|control_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32070": { + "content": "<|control_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32071": { + "content": "<|control_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32072": { + "content": "<|control_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32073": { + "content": "<|control_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32074": { + "content": "<|control_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32075": { + "content": "<|control_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32076": { + "content": "<|control_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32077": { + "content": "<|control_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32078": { + "content": "<|control_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32079": { + "content": "<|control_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32080": { + "content": "<|control_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32081": { + "content": "<|control_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32082": { + "content": "<|control_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32083": { + "content": "<|control_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32084": { + "content": "<|control_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32085": { + "content": "<|control_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32086": { + "content": "<|control_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32087": { + "content": "<|control_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32088": { + "content": "<|control_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32089": { + "content": "<|control_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32090": { + "content": "<|control_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32091": { + "content": "<|control_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32092": { + "content": "<|control_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32093": { + "content": "<|control_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32094": { + "content": "<|control_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32095": { + "content": "<|control_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32096": { + "content": "<|control_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32097": { + "content": "<|control_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32098": { + "content": "<|control_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32099": { + "content": "<|control_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32100": { + "content": "<|control_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32101": { + "content": "<|control_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32102": { + "content": "<|control_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32103": { + "content": "<|control_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32104": { + "content": "<|control_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32105": { + "content": "<|control_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32106": { + "content": "<|control_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32107": { + "content": "<|control_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32108": { + "content": "<|control_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32109": { + "content": "<|control_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32110": { + "content": "<|control_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32111": { + "content": "<|control_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32112": { + "content": "<|control_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32113": { + "content": "<|control_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32114": { + "content": "<|control_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32115": { + "content": "<|control_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32116": { + "content": "<|control_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32117": { + "content": "<|control_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32118": { + "content": "<|control_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32119": { + "content": "<|control_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32120": { + "content": "<|control_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32121": { + "content": "<|control_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32122": { + "content": "<|control_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32123": { + "content": "<|control_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32124": { + "content": "<|control_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32125": { + "content": "<|control_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32126": { + "content": "<|control_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32127": { + "content": "<|control_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|function_list|>", + "<|function_output|>", + "<|function_call|>", + "", + "", + "", + "", + "<|control_10|>", + "<|control_11|>", + "<|control_12|>", + "<|control_13|>", + "<|control_14|>", + "<|control_15|>", + "<|control_16|>", + "<|control_17|>", + "<|control_18|>", + "<|control_19|>", + "<|control_20|>", + "<|control_21|>", + "<|control_22|>", + "<|control_23|>", + "<|control_24|>", + "<|control_25|>", + "<|control_26|>", + "<|control_27|>", + "<|control_28|>", + "<|control_29|>", + "<|control_30|>", + "<|control_31|>", + "<|control_32|>", + "<|control_33|>", + "<|control_34|>", + "<|control_35|>", + "<|control_36|>", + "<|control_37|>", + "<|control_38|>", + "<|control_39|>", + "<|control_40|>", + "<|control_41|>", + "<|control_42|>", + "<|control_43|>", + "<|control_44|>", + "<|control_45|>", + "<|control_46|>", + "<|control_47|>", + "<|control_48|>", + "<|control_49|>", + "<|control_50|>", + "<|control_51|>", + "<|control_52|>", + "<|control_53|>", + "<|control_54|>", + "<|control_55|>", + "<|control_56|>", + "<|control_57|>", + "<|control_58|>", + "<|control_59|>", + "<|control_60|>", + "<|control_61|>", + "<|control_62|>", + "<|control_63|>", + "<|control_64|>", + "<|control_65|>", + "<|control_66|>", + "<|control_67|>", + "<|control_68|>", + "<|control_69|>", + "<|control_70|>", + "<|control_71|>", + "<|control_72|>", + "<|control_73|>", + "<|control_74|>", + "<|control_75|>", + "<|control_76|>", + "<|control_77|>", + "<|control_78|>", + "<|control_79|>", + "<|control_80|>", + "<|control_81|>", + "<|control_82|>", + "<|control_83|>", + "<|control_84|>", + "<|control_85|>", + "<|control_86|>", + "<|control_87|>", + "<|control_88|>", + "<|control_89|>", + "<|control_90|>", + "<|control_91|>", + "<|control_92|>", + "<|control_93|>", + "<|control_94|>", + "<|control_95|>", + "<|control_96|>", + "<|control_97|>", + "<|control_98|>", + "<|control_99|>", + "<|control_100|>", + "<|control_101|>", + "<|control_102|>", + "<|control_103|>", + "<|control_104|>", + "<|control_105|>", + "<|control_106|>", + "<|control_107|>", + "<|control_108|>", + "<|control_109|>", + "<|control_110|>", + "<|control_111|>", + "<|control_112|>", + "<|control_113|>", + "<|control_114|>", + "<|control_115|>", + "<|control_116|>", + "<|control_117|>", + "<|control_118|>", + "<|control_119|>", + "<|control_120|>", + "<|control_121|>", + "<|control_122|>", + "<|control_123|>", + "<|control_124|>", + "<|control_125|>", + "<|control_126|>", + "<|control_127|>", + "<|control_128|>" + ], + "bos_token": "", + "chat_template": "{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|im_end|>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}