From dba18605602862f5473ebf348f7ab89579ae1835 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 23 May 2026 11:39:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: alpindale/gemma-7b-it Source: Original Platform --- .gitattributes | 37 +++ README.md | 495 +++++++++++++++++++++++++++++++ config.json | 28 ++ generation_config.json | 7 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 261 ++++++++++++++++ special_tokens_map.json | 46 +++ tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 70 +++++ 13 files changed, 962 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..216a002 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +gemma-7b-it.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d78bcc --- /dev/null +++ b/README.md @@ -0,0 +1,495 @@ +--- +library_name: transformers +tags: [] +extra_gated_heading: "Access Gemma on Hugging Face" +extra_gated_prompt: "To access Gemma on Hugging Face, you’re required to review and agree to Google’s usage license. To do this, please ensure you’re logged-in to Hugging Face and click below. Requests are processed immediately." +extra_gated_button_content: "Acknowledge license" +--- + +# Gemma Model Card + +**Model Page**: [Gemma](https://ai.google.dev/gemma/docs) + +This model card corresponds to the 7B instruct version of the Gemma model. You can also visit the model card of the [2B base model](https://huggingface.co/google/gemma-2b), [7B base model](https://huggingface.co/google/gemma-7b), and [2B instruct model](https://huggingface.co/google/gemma-2b-it). + +**Resources and Technical Documentation**: + +* [Responsible Generative AI Toolkit](https://ai.google.dev/responsible) +* [Gemma on Kaggle](https://www.kaggle.com/models/google/gemma) +* [Gemma on Vertex Model Garden](https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/335) + +**Terms of Use**: [Terms](https://www.kaggle.com/models/google/gemma/license/consent) + +**Authors**: Google + +## Model Information + +Summary description and brief definition of inputs and outputs. + +### Description + +Gemma is a family of lightweight, state-of-the-art open models from Google, +built from the same research and technology used to create the Gemini models. +They are text-to-text, decoder-only large language models, available in English, +with open weights, pre-trained variants, and instruction-tuned variants. Gemma +models are well-suited for a variety of text generation tasks, including +question answering, summarization, and reasoning. Their relatively small size +makes it possible to deploy them in environments with limited resources such as +a laptop, desktop or your own cloud infrastructure, democratizing access to +state of the art AI models and helping foster innovation for everyone. + +### Usage + +Below we share some code snippets on how to get quickly started with running the model. First make sure to `pip install -U transformers`, then copy the snippet from the section that is relevant for your usecase. + +#### Fine-tuning the model + +You can find fine-tuning scripts and notebook under the [`examples/` directory](https://huggingface.co/google/gemma-7b/tree/main/examples) of [`google/gemma-7b`](https://huggingface.co/google/gemma-7b) repository. To adapt it to this model, simply change the model-id to `google/gemma-7b-it`. +In that repository, we provide: + +* A script to perform Supervised Fine-Tuning (SFT) on UltraChat dataset using QLoRA +* A script to perform SFT using FSDP on TPU devices +* A notebook that you can run on a free-tier Google Colab instance to perform SFT on English quotes dataset + + +#### Running the model on a CPU + + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b-it") +model = AutoModelForCausalLM.from_pretrained("google/gemma-7b-it") + +input_text = "Write me a poem about Machine Learning." +input_ids = tokenizer(**input_text, return_tensors="pt") + +outputs = model.generate(input_ids) +print(tokenizer.decode(outputs[0])) +``` + + +#### Running the model on a single / multi GPU + + +```python +# pip install accelerate +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b-it") +model = AutoModelForCausalLM.from_pretrained("google/gemma-7b-it", device_map="auto") + +input_text = "Write me a poem about Machine Learning." +input_ids = tokenizer(input_text, return_tensors="pt").to("cuda") + +outputs = model.generate(**input_ids) +print(tokenizer.decode(outputs[0])) +``` + + +#### Running the model on a GPU using different precisions + +* _Using `torch.float16`_ + +```python +# pip install accelerate +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b-it") +model = AutoModelForCausalLM.from_pretrained("google/gemma-7b-it", device_map="auto", torch_dtype=torch.float16) + +input_text = "Write me a poem about Machine Learning." +input_ids = tokenizer(input_text, return_tensors="pt").to("cuda") + +outputs = model.generate(**input_ids) +print(tokenizer.decode(outputs[0])) +``` + +* _Using `torch.bfloat16`_ + +```python +# pip install accelerate +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b-it") +model = AutoModelForCausalLM.from_pretrained("google/gemma-7b-it", device_map="auto", torch_dtype=torch.bfloat16) + +input_text = "Write me a poem about Machine Learning." +input_ids = tokenizer(input_text, return_tensors="pt").to("cuda") + +outputs = model.generate(**input_ids) +print(tokenizer.decode(outputs[0])) +``` + +#### Quantized Versions through `bitsandbytes` + +* _Using 8-bit precision (int8)_ + +```python +# pip install bitsandbytes accelerate +from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig + +quantization_config = BitsAndBytesConfig(load_in_8bit=True) + +tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b-it") +model = AutoModelForCausalLM.from_pretrained("google/gemma-7b-it", quantization_config=quantization_config) + +input_text = "Write me a poem about Machine Learning." +input_ids = tokenizer(input_text, return_tensors="pt").to("cuda") + +outputs = model.generate(**input_ids) +print(tokenizer.decode(outputs[0])) +``` + +* _Using 4-bit precision_ + +```python +# pip install bitsandbytes accelerate +from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig + +quantization_config = BitsAndBytesConfig(load_in_4bit=True) + +tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b-it") +model = AutoModelForCausalLM.from_pretrained("google/gemma-7b-it", quantization_config=quantization_config) + +input_text = "Write me a poem about Machine Learning." +input_ids = tokenizer(input_text, return_tensors="pt").to("cuda") + +outputs = model.generate(**input_ids) +print(tokenizer.decode(outputs[0])) +``` + + +#### Other optimizations + +* _Flash Attention 2_ + +First make sure to install `flash-attn` in your environment `pip install flash-attn` + +```diff +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.float16, ++ attn_implementation="flash_attention_2" +).to(0) +``` + +### Chat Template + +The instruction-tuned models use a chat template that must be adhered to for conversational use. +The easiest way to apply it is using the tokenizer's built-in chat template, as shown in the following snippet. + +Let's load the model and apply the chat template to a conversation. In this example, we'll start with a single user interaction: + +```py +from transformers import AutoTokenizer, AutoModelForCausalLM +import transformers +import torch + +model_id = "gg-hf/gemma-7b-it" +dtype = torch.bfloat16 + +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained( + model_id, + device_map="cuda", + torch_dtype=dtype, +) + +chat = [ + { "role": "user", "content": "Write a hello world program" }, +] +prompt = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True) +``` + +At this point, the prompt contains the following text: + +``` +user +Write a hello world program +model +``` + +As you can see, each turn is preceeded by a `` delimiter and then the role of the entity +(either `user`, for content supplied by the user, or `model` for LLM responses). Turns finish with +the `` token. + +You can follow this format to build the prompt manually, if you need to do it without the tokenizer's +chat template. + +After the prompt is ready, generation can be performed like this: + +```py +inputs = tokenizer.encode(prompt, add_special_tokens=True, return_tensors="pt") +outputs = model.generate(input_ids=inputs.to(model.device), max_new_tokens=150) +``` + +### Inputs and outputs + +* **Input:** Text string, such as a question, a prompt, or a document to be + summarized. +* **Output:** Generated English-language text in response to the input, such + as an answer to a question, or a summary of a document. + +## Model Data + +Data used for model training and how the data was processed. + +### Training Dataset + +These models were trained on a dataset of text data that includes a wide variety +of sources, totaling 6 trillion tokens. Here are the key components: + +* Web Documents: A diverse collection of web text ensures the model is exposed + to a broad range of linguistic styles, topics, and vocabulary. Primarily + English-language content. +* Code: Exposing the model to code helps it to learn the syntax and patterns of + programming languages, which improves its ability to generate code or + understand code-related questions. +* Mathematics: Training on mathematical text helps the model learn logical + reasoning, symbolic representation, and to address mathematical queries. + +The combination of these diverse data sources is crucial for training a powerful +language model that can handle a wide variety of different tasks and text +formats. + +### Data Preprocessing + +Here are the key data cleaning and filtering methods applied to the training +data: + +* CSAM Filtering: Rigorous CSAM (Child Sexual Abuse Material) filtering was + applied at multiple stages in the data preparation process to ensure the + exclusion of harmful and illegal content +* Sensitive Data Filtering: As part of making Gemma pre-trained models safe and + reliable, automated techniques were used to filter out certain personal + information and other sensitive data from training sets. +* Additional methods: Filtering based on content quality and safely in line with + [our policies](https://storage.googleapis.com/gweb-uniblog-publish-prod/documents/2023_Google_AI_Principles_Progress_Update.pdf#page=11). + +## Implementation Information + +Details about the model internals. + +### Hardware + +Gemma was trained using the latest generation of +[Tensor Processing Unit (TPU)](https://cloud.google.com/tpu/docs/intro-to-tpu) hardware (TPUv5e). + +Training large language models requires significant computational power. TPUs, +designed specifically for matrix operations common in machine learning, offer +several advantages in this domain: + +* Performance: TPUs are specifically designed to handle the massive computations + involved in training LLMs. They can speed up training considerably compared to + CPUs. +* Memory: TPUs often come with large amounts of high-bandwidth memory, allowing + for the handling of large models and batch sizes during training. This can + lead to better model quality. +* Scalability: TPU Pods (large clusters of TPUs) provide a scalable solution for + handling the growing complexity of large foundation models. You can distribute + training across multiple TPU devices for faster and more efficient processing. +* Cost-effectiveness: In many scenarios, TPUs can provide a more cost-effective + solution for training large models compared to CPU-based infrastructure, + especially when considering the time and resources saved due to faster + training. +* These advantages are aligned with + [Google's commitments to operate sustainably](https://sustainability.google/operating-sustainably/). + +### Software + +Training was done using [JAX](https://github.com/google/jax) and [ML Pathways](https://blog.google/technology/ai/introducing-pathways-next-generation-ai-architecture/ml-pathways). + +JAX allows researchers to take advantage of the latest generation of hardware, +including TPUs, for faster and more efficient training of large models. + +ML Pathways is Google's latest effort to build artificially intelligent systems +capable of generalizing across multiple tasks. This is specially suitable for +[foundation models](https://ai.google/discover/foundation-models/), including large language models like +these ones. + +Together, JAX and ML Pathways are used as described in the +[paper about the Gemini family of models](https://arxiv.org/abs/2312.11805); "the 'single +controller' programming model of Jax and Pathways allows a single Python +process to orchestrate the entire training run, dramatically simplifying the +development workflow." + +## Evaluation + +Model evaluation metrics and results. + +### Benchmark Results + +These models were evaluated against a large collection of different datasets and +metrics to cover different aspects of text generation: + +| Benchmark | Metric | 2B Params | 7B Params | +| ------------------------------ | ------------- | ----------- | --------- | +| [MMLU](https://arxiv.org/abs/2009.03300) | 5-shot, top-1 | 42.3 | 64.3 | +| [HellaSwag](https://arxiv.org/abs/1905.07830) | 0-shot |71.4 | 81.2 | +| [PIQA](https://arxiv.org/abs/1911.11641) | 0-shot | 77.3 | 81.2 | +| [SocialIQA](https://arxiv.org/abs/1904.09728) | 0-shot | 59.7 | 51.8 | +| [BooIQ](https://arxiv.org/abs/1905.10044) | 0-shot | 69.4 | 83.2 | +| [WinoGrande](https://arxiv.org/abs/1907.10641) | partial score | 65.4 | 72.3 | +| [CommonsenseQA](https://arxiv.org/abs/1811.00937) | 7-shot | 65.3 | 71.3 | +| [OpenBookQA](https://arxiv.org/abs/1809.02789) | | 47.8 | 52.8 | +| [ARC-e](https://arxiv.org/abs/1911.01547) | | 73.2 | 81.5 | +| [ARC-c](https://arxiv.org/abs/1911.01547) | | 42.1 | 53.2 | +| [TriviaQA](https://arxiv.org/abs/1705.03551) | 5-shot | 53.2 | 63.4 | +| [Natural Questions](https://github.com/google-research-datasets/natural-questions) | 5-shot | - | 23 | +| [HumanEval](https://arxiv.org/abs/2107.03374) | pass@1 | 22.0 | 32.3 | +| [MBPP](https://arxiv.org/abs/2108.07732) | 3-shot | 29.2 | 44.4 | +| [GSM8K](https://arxiv.org/abs/2110.14168) | maj@1 | 17.7 | 46.4 | +| [MATH](https://arxiv.org/abs/2108.07732) | 4-shot | 11.8 | 24.3 | +| [AGIEval](https://arxiv.org/abs/2304.06364) | | 24.2 | 41.7 | +| [BIG-Bench](https://arxiv.org/abs/2206.04615) | | 35.2 | 55.1 | +| ------------------------------ | ------------- | ----------- | --------- | +| **Average** | | **54.0** | **56.4** | + +## Ethics and Safety + +Ethics and safety evaluation approach and results. + +### Evaluation Approach + +Our evaluation methods include structured evaluations and internal red-teaming +testing of relevant content policies. Red-teaming was conducted by a number of +different teams, each with different goals and human evaluation metrics. These +models were evaluated against a number of different categories relevant to +ethics and safety, including: + +* Text-to-Text Content Safety: Human evaluation on prompts covering safety + policies including child sexual abuse and exploitation, harassment, violence + and gore, and hate speech. +* Text-to-Text Representational Harms: Benchmark against relevant academic + datasets such as [WinoBias](https://arxiv.org/abs/1804.06876) and [BBQ Dataset](https://arxiv.org/abs/2110.08193v2). +* Memorization: Automated evaluation of memorization of training data, including + the risk of personally identifiable information exposure. +* Large-scale harm: Tests for "dangerous capabilities," such as chemical, + biological, radiological, and nuclear (CBRN) risks. + +### Evaluation Results + +The results of ethics and safety evaluations are within acceptable thresholds +for meeting [internal policies](https://storage.googleapis.com/gweb-uniblog-publish-prod/documents/2023_Google_AI_Principles_Progress_Update.pdf#page=11) for categories such as child +safety, content safety, representational harms, memorization, large-scale harms. +On top of robust internal evaluations, the results of well known safety +benchmarks like BBQ, BOLD, Winogender, Winobias, RealToxicity, and TruthfulQA +are shown here. + +| Benchmark | Metric | 2B Params | 7B Params | +| ------------------------------ | ------------- | ----------- | --------- | +| [RealToxicity](https://arxiv.org/abs/2009.11462) | average | 6.86 | 7.90 | +| [BOLD](https://arxiv.org/abs/2101.11718) | | 45.57 | 49.08 | +| [CrowS-Pairs](https://aclanthology.org/2020.emnlp-main.154/) | top-1 | 45.82 | 51.33 | +| [BBQ Ambig](https://arxiv.org/abs/2110.08193v2) | 1-shot, top-1 | 62.58 | 92.54 | +| [BBQ Disambig](https://arxiv.org/abs/2110.08193v2) | top-1 | 54.62 | 71.99 | +| [Winogender](https://arxiv.org/abs/1804.09301) | top-1 | 51.25 | 54.17 | +| [TruthfulQA](https://arxiv.org/abs/2109.07958) | | 44.84 | 31.81 | +| [Winobias 1_2](https://arxiv.org/abs/1804.06876) | | 56.12 | 59.09 | +| [Winobias 2_2](https://arxiv.org/abs/1804.06876) | | 91.10 | 92.23 | +| [Toxigen](https://arxiv.org/abs/2203.09509) | | 29.77 | 39.59 | +| ------------------------------ | ------------- | ----------- | --------- | + + +## Usage and Limitations + +These models have certain limitations that users should be aware of. + +### Intended Usage + +Open Large Language Models (LLMs) have a wide range of applications across +various industries and domains. The following list of potential uses is not +comprehensive. The purpose of this list is to provide contextual information +about the possible use-cases that the model creators considered as part of model +training and development. + +* Content Creation and Communication + * Text Generation: These models can be used to generate creative text formats + such as poems, scripts, code, marketing copy, and email drafts. + * Chatbots and Conversational AI: Power conversational interfaces for customer + service, virtual assistants, or interactive applications. + * Text Summarization: Generate concise summaries of a text corpus, research + papers, or reports. +* Research and Education + * Natural Language Processing (NLP) Research: These models can serve as a + foundation for researchers to experiment with NLP techniques, develop + algorithms, and contribute to the advancement of the field. + * Language Learning Tools: Support interactive language learning experiences, + aiding in grammar correction or providing writing practice. + * Knowledge Exploration: Assist researchers in exploring large bodies of text + by generating summaries or answering questions about specific topics. + +### Limitations + +* Training Data + * The quality and diversity of the training data significantly influence the + model's capabilities. Biases or gaps in the training data can lead to + limitations in the model's responses. + * The scope of the training dataset determines the subject areas the model can + handle effectively. +* Context and Task Complexity + * LLMs are better at tasks that can be framed with clear prompts and + instructions. Open-ended or highly complex tasks might be challenging. + * A model's performance can be influenced by the amount of context provided + (longer context generally leads to better outputs, up to a certain point). +* Language Ambiguity and Nuance + * Natural language is inherently complex. LLMs might struggle to grasp subtle + nuances, sarcasm, or figurative language. +* Factual Accuracy + * LLMs generate responses based on information they learned from their + training datasets, but they are not knowledge bases. They may generate + incorrect or outdated factual statements. +* Common Sense + * LLMs rely on statistical patterns in language. They might lack the ability + to apply common sense reasoning in certain situations. + +### Ethical Considerations and Risks + +The development of large language models (LLMs) raises several ethical concerns. +In creating an open model, we have carefully considered the following: + +* Bias and Fairness + * LLMs trained on large-scale, real-world text data can reflect socio-cultural + biases embedded in the training material. These models underwent careful + scrutiny, input data pre-processing described and posterior evaluations + reported in this card. +* Misinformation and Misuse + * LLMs can be misused to generate text that is false, misleading, or harmful. + * Guidelines are provided for responsible use with the model, see the + [Responsible Generative AI Toolkit](http://ai.google.dev/gemma/responsible). +* Transparency and Accountability: + * This model card summarizes details on the models' architecture, + capabilities, limitations, and evaluation processes. + * A responsibly developed open model offers the opportunity to share + innovation by making LLM technology accessible to developers and researchers + across the AI ecosystem. + +Risks identified and mitigations: + +* Perpetuation of biases: It's encouraged to perform continuous monitoring + (using evaluation metrics, human review) and the exploration of de-biasing + techniques during model training, fine-tuning, and other use cases. +* Generation of harmful content: Mechanisms and guidelines for content safety + are essential. Developers are encouraged to exercise caution and implement + appropriate content safety safeguards based on their specific product policies + and application use cases. +* Misuse for malicious purposes: Technical limitations and developer and + end-user education can help mitigate against malicious applications of LLMs. + Educational resources and reporting mechanisms for users to flag misuse are + provided. Prohibited uses of Gemma models are outlined in the + [Gemma Prohibited Use Policy](https://ai.google.dev/gemma/prohibited_use_policy). +* Privacy violations: Models were trained on data filtered for removal of PII + (Personally Identifiable Information). Developers are encouraged to adhere to + privacy regulations with privacy-preserving techniques. + +### Benefits + +At the time of release, this family of models provides high-performance open +large language model implementations designed from the ground up for Responsible +AI development compared to similarly sized models. + +Using the benchmark evaluation metrics described in this document, these models +have shown to provide superior performance to other, comparably-sized open model +alternatives. + diff --git a/config.json b/config.json new file mode 100644 index 0000000..31fe13c --- /dev/null +++ b/config.json @@ -0,0 +1,28 @@ +{ + "_name_or_path": "/raid/younes/gg-converted-ckpt/gemma-7b-it", + "architectures": [ + "GemmaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 2, + "eos_token_id": 1, + "head_dim": 256, + "hidden_act": "gelu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 24576, + "max_position_embeddings": 8192, + "model_type": "gemma", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 16, + "pad_token_id": 0, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000.0, + "torch_dtype": "bfloat16", + "transformers_version": "4.38.0.dev0", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..fb47c6d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": 1, + "pad_token_id": 0, + "transformers_version": "4.38.0.dev0" +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..8c944f3 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ba358c3ef24e72169e09a5d39c5ddf0a1113d646680964f1efe3c6f5b55d16 +size 4995496656 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..5a2d01c --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b13b6710374a0b7023415525b89542dfeae66942f8dbf882036cdc7db2848aea +size 4982953168 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..404f7da --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d2f076a3bd913313c98f229a1ad250afdcd36977adb97875b4b07bd2e415a98 +size 4982953200 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..ae72705 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a125907b39544cb387c145c37043a756e1ec38e5b2ea6dfabe3a789b31ce723 +size 2113988336 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..ea169ac --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,261 @@ +{ + "metadata": { + "total_size": 17075361792 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..64cb2a9 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,46 @@ +{ + "additional_special_tokens": [ + { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..10d1ca0 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05e97791a5e007260de1db7e1692e53150e08cea481e2bf25435553380c147ee +size 17477929 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..0fb99fa --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6969e64047744a44bb3abfb5c50f8de0f7ed8b571d5444426ef931f651d1a0ef +size 4241111 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..f66eacb --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,70 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\n' + message['content'] | trim + '\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\n'}}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": null, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}