From a36924575f5b876ea73e6b20ce5f1d4f6ef2ed12 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 9 May 2026 01:23:08 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Multilingual-Multimodal-NLP/IndustrialCoder-Base Source: Original Platform --- .gitattributes | 35 + README.md | 194 ++++++ config.json | 33 + configuration_iquestcoder.py | 182 +++++ generation_config.json | 6 + model-00001-of-00013.safetensors | 3 + model-00002-of-00013.safetensors | 3 + model-00003-of-00013.safetensors | 3 + model-00004-of-00013.safetensors | 3 + model-00005-of-00013.safetensors | 3 + model-00006-of-00013.safetensors | 3 + model-00007-of-00013.safetensors | 3 + model-00008-of-00013.safetensors | 3 + model-00009-of-00013.safetensors | 3 + model-00010-of-00013.safetensors | 3 + model-00011-of-00013.safetensors | 3 + model-00012-of-00013.safetensors | 3 + model-00013-of-00013.safetensors | 3 + model.safetensors.index.json | 586 ++++++++++++++++ modeling_iquestcoder.py | 1063 ++++++++++++++++++++++++++++++ tokenization_iquestcoder.py | 552 ++++++++++++++++ tokenizer.model | 3 + tokenizer_config.json | 240 +++++++ 23 files changed, 2933 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration_iquestcoder.py create mode 100644 generation_config.json create mode 100644 model-00001-of-00013.safetensors create mode 100644 model-00002-of-00013.safetensors create mode 100644 model-00003-of-00013.safetensors create mode 100644 model-00004-of-00013.safetensors create mode 100644 model-00005-of-00013.safetensors create mode 100644 model-00006-of-00013.safetensors create mode 100644 model-00007-of-00013.safetensors create mode 100644 model-00008-of-00013.safetensors create mode 100644 model-00009-of-00013.safetensors create mode 100644 model-00010-of-00013.safetensors create mode 100644 model-00011-of-00013.safetensors create mode 100644 model-00012-of-00013.safetensors create mode 100644 model-00013-of-00013.safetensors create mode 100644 model.safetensors.index.json create mode 100644 modeling_iquestcoder.py create mode 100644 tokenization_iquestcoder.py create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..6e86367 --- /dev/null +++ b/README.md @@ -0,0 +1,194 @@ +--- +license: apache-2.0 +library_name: transformers +pipeline_tag: text-generation +tags: +- code +- industrial-code +- pretrained +- base-model +- verilog +- cuda +- triton +- chip-design +- cad +--- + +# InCoder-32B-Base: Code Foundation Model for Industrial Scenarios + +
+ +[![HuggingFace](https://img.shields.io/badge/🤗-Model%20Hub-yellow)](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder-Base) +[![GitHub](https://img.shields.io/badge/GitHub-Industrial--Coder-blue)](https://github.com/CSJianYang/Industrial-Coder) +[![arXiv](https://img.shields.io/badge/arXiv-2603.16790-red)](https://huggingface.co/papers/2603.16790) +[![License](https://img.shields.io/badge/License-Apache%202.0-green)](LICENSE) + +
+ +## Model Summary + +**InCoder-32B-Base** is the pre-trained base model of the InCoder family — the first 32B-parameter code foundation model purpose-built for industrial code intelligence. This is the base (non-instruction-tuned) checkpoint, suitable for code completion, fill-in-the-middle (FIM), and further fine-tuning. + +For the instruction-tuned variant, see [IndustrialCoder](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder). For the reasoning variant, see [IndustrialCoder-Thinking](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder-Thinking). + +Presented in the paper [InCoder-32B: Code Foundation Model for Industrial Scenarios](https://huggingface.co/papers/2603.16790), InCoder-32B unifies code intelligence across five industrial domains: + +| Domain | Languages & Frameworks | +|---|---| +| 🔧 **Chip Design** | Verilog, SystemVerilog, RTL | +| ⚡ **GPU Kernel Optimization** | CUDA, Triton | +| 🖥️ **Embedded Systems** | C/C++, ARM Cortex-M4, STM32 | +| 🔨 **Compiler Optimization** | x86-64 ASM, C/C++, LLVM-IR | +| 📐 **3D Modeling / CAD** | CadQuery, OpenCascade, Python | + +--- + +## Model Architecture + +InCoder-32B-Base adopts a standard decoder-only Transformer architecture: + +| Hyperparameter | Value | +|---|---| +| Parameters | ~32B | +| Layers | 64 | +| Hidden Size | 5,120 | +| Attention Heads | 40 (8 KV heads, GQA) | +| Max Context Length | 131,072 (128K) | +| Positional Encoding | RoPE (θ = 500,000) | +| Precision | BFloat16 | +| Vocabulary Size | 76,800 | + +--- + +## Training Pipeline: Code-Flow + +InCoder-32B-Base is trained through a two-stage **Code-Flow** pipeline: + +### Stage 1 — Pre-training & Annealing +- **Industrial Recall**: Data pipeline using rule-based filtering, FastText classifiers, and semantic retrieval for Verilog, CUDA, firmware C, and CadQuery. +- **Refinement**: OCR extraction from technical manuals, multi-level deduplication, and repository-level fork consolidation. +- **Training**: 15T total tokens using Autoregressive LM + Fill-in-the-Middle (FIM) objectives on 4,096 GPUs. + +### Stage 2 — Mid-Training (Context Extension) +Context window extended progressively from 8K to 128K tokens: +- **8K → 32K**: Targets file-level tasks like completing RTL modules or kernel functions. +- **32K → 128K**: Unlocks long-context capabilities for extended debugging and cross-module projects. + +--- + +## Usage + +### Installation + +```bash +pip install transformers accelerate +``` + +### Code Completion + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM +import torch + +model_id = "Multilingual-Multimodal-NLP/IndustrialCoder-Base" + +tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", + trust_remote_code=True, +) + +prompt = """// Synthesizable Verilog: UART transmitter (8N1 protocol) +module uart_tx ( + input wire clk, + input wire rst_n, + input wire [7:0] data_in, + input wire tx_start, + output reg tx, + output reg tx_busy +); +""" + +inputs = tokenizer(prompt, return_tensors="pt").to(model.device) +outputs = model.generate( + **inputs, + max_new_tokens=512, + temperature=0.2, + do_sample=True, +) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +### Fill-in-the-Middle (FIM) + +InCoder-32B-Base supports FIM completion for code infilling tasks: + +```python +prefix = """// CUDA kernel for RMS Normalization +__global__ void rms_norm_kernel(float* output, const float* input, + const float* weight, int N, float eps) { + int idx = blockIdx.x; +""" +suffix = """ + output[idx * N + tid] = normalized * weight[tid]; +}""" + +fim_prompt = f"<|fim_prefix|>{prefix}<|fim_suffix|>{suffix}<|fim_middle|>" +inputs = tokenizer(fim_prompt, return_tensors="pt").to(model.device) +outputs = model.generate(**inputs, max_new_tokens=256) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +### Deployment with vLLM + +```bash +vllm serve Multilingual-Multimodal-NLP/IndustrialCoder-Base \ + --tensor-parallel-size 4 --max-model-len 32768 --trust-remote-code +``` + +--- + +## Fine-tuning + +We provide an SFT framework in the [GitHub repository](https://github.com/CSJianYang/Industrial-Coder/tree/main/sft). See the README for data preparation and training instructions. + +--- + +## Model Family + +| Model | Type | HuggingFace | +|---|---|---| +| InCoder-32B-Base | Pre-trained | [🤗 IndustrialCoder-Base](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder-Base) | +| InCoder-32B | Instruct | [🤗 IndustrialCoder](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder) | +| InCoder-32B-Thinking | Reasoning | [🤗 IndustrialCoder-Thinking](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder-Thinking) | +| InCoder-32B-FP8 | FP8 Quantized | [🤗 IndustrialCoder-32B-FP8](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder-32B-FP8) | +| InCoder-32B-AWQ-INT4 | AWQ INT4 | [🤗 IndustrialCoder-32B-AWQ-INT4](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder-32B-AWQ-INT4) | +| InCoder-32B-GPTQ-INT4 | GPTQ INT4 | [🤗 IndustrialCoder-32B-GPTQ-INT4](https://huggingface.co/Multilingual-Multimodal-NLP/IndustrialCoder-32B-GPTQ-INT4) | + +--- + +## Limitations & Disclaimers + +This is a **base model** — it has not been instruction-tuned and does not follow conversational instructions. It is best suited for: +- Code completion and generation +- Fill-in-the-middle (FIM) tasks +- Further fine-tuning for downstream applications + +Always review and test generated code in a sandboxed environment. Industrial code (RTL, embedded firmware, GPU kernels) requires expert review before deployment. + +--- + +## Citation + +```bibtex +@article{yang2026incoder, + title={InCoder-32B: Code Foundation Model for Industrial Scenarios}, + author={Yang, Jian and Zhang, Wei and Wu, Jiajun and Cheng, Junhang and Guo, Shawn + and Wang, Haowen and Gu, Weicheng and Du, Yaxin and Li, Joseph and Xu, Fanglin + and others}, + journal={arXiv preprint arXiv:2603.16790}, + year={2026} +} +``` diff --git a/config.json b/config.json new file mode 100644 index 0000000..4f2576b --- /dev/null +++ b/config.json @@ -0,0 +1,33 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": [ + 2, + 75864, + 75869 + ], + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 27648, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 40, + "num_hidden_layers": 64, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.55.4", + "use_cache": true, + "vocab_size": 76800 +} diff --git a/configuration_iquestcoder.py b/configuration_iquestcoder.py new file mode 100644 index 0000000..8aac9f4 --- /dev/null +++ b/configuration_iquestcoder.py @@ -0,0 +1,182 @@ +"""IQuestCoder model configuration.""" + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + + +class IQuestCoderConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`IQuestCoderModel`]. It is used to instantiate + an IQuestCoder model according to the specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 76800): + Vocabulary size of the IQuestCoder model. Defines the number of different tokens that can be represented + by the `inputs_ids` passed when calling [`IQuestCoderModel`]. + hidden_size (`int`, *optional*, defaults to 5120): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 27648): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 80): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 40): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*, defaults to 8): + This is the number of key_value heads that should be used to implement Grouped Query Attention (GQA). + If `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA). + If `num_key_value_heads=1`, the model will use Multi Query Attention (MQA). + head_dim (`int`, *optional*, defaults to 128): + The dimension of each attention head. If not specified, defaults to `hidden_size // num_attention_heads`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 16384): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + pad_token_id (`int`, *optional*): + Padding token id. + bos_token_id (`int`, *optional*, defaults to 1): + Beginning of stream token id. + eos_token_id (`int`, *optional*, defaults to 2): + End of stream token id. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings. + rope_theta (`float`, *optional*, defaults to 500000.0): + The base period of the RoPE embeddings. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. Supports various RoPE scaling + types including "linear", "dynamic", "yarn", "longrope", etc. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + mlp_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers. + clip_qkv (`float`, *optional*): + If set, clip the query, key, and value tensors to this value. Borrowed from OLMo for training stability. + use_sliding_window (`bool`, *optional*, defaults to `False`): + Whether to use sliding window attention. Borrowed from Qwen2. + sliding_window (`int`, *optional*): + The sliding window size. Only effective when `use_sliding_window=True`. + max_window_layers (`int`, *optional*, defaults to 0): + The number of layers that don't use sliding window attention. Borrowed from Qwen2. + + Example: + ```python + >>> from configuration_iquestcoder import IQuestCoderConfig + >>> from modeling_iquestcoder import IQuestCoderModel + + >>> # Initializing a IQuestCoder configuration + >>> configuration = IQuestCoderConfig() + + >>> # Initializing a model from the configuration + >>> model = IQuestCoderModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` + """ + + model_type = "iquestcoder" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=76800, + hidden_size=5120, + intermediate_size=27648, + num_hidden_layers=80, + num_attention_heads=40, + num_key_value_heads=8, + head_dim=128, + hidden_act="silu", + max_position_embeddings=16384, + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + rope_theta=500000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + mlp_bias=False, + # IQuestCoder specific (borrowed from OLMo) + clip_qkv=None, + # IQuestCoder specific (borrowed from Qwen2) + use_sliding_window=False, + sliding_window=None, + max_window_layers=0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.mlp_bias = mlp_bias + # IQuestCoder specific + self.clip_qkv = clip_qkv + self.use_sliding_window = use_sliding_window + self.sliding_window = sliding_window + self.max_window_layers = max_window_layers + + # Validate rope_scaling configuration + self._rope_scaling_validation() + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_validation(self): + """Validate the `rope_scaling` configuration.""" + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) < 1: + raise ValueError( + "`rope_scaling` must be a dictionary with a minimum of one field, `type` or `rope_type`." + ) + + rope_scaling_type = self.rope_scaling.get("type", None) or self.rope_scaling.get("rope_type", None) + if rope_scaling_type is None: + raise ValueError( + "`rope_scaling` must have a `type` or `rope_type` field." + ) + + valid_rope_types = ["linear", "dynamic", "yarn", "longrope", "llama3"] + if rope_scaling_type not in valid_rope_types: + raise ValueError( + f"`rope_scaling`'s type field must be one of {valid_rope_types}, got {rope_scaling_type}" + ) + + +__all__ = ["IQuestCoderConfig"] + diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..45ee998 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": [2, 75864, 75869], + "transformers_version": "4.55.4" +} diff --git a/model-00001-of-00013.safetensors b/model-00001-of-00013.safetensors new file mode 100644 index 0000000..36b3182 --- /dev/null +++ b/model-00001-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69006dee33f4335aaa693e12e92ecc051f16c5b498188a022515145533beaa58 +size 5358309312 diff --git a/model-00002-of-00013.safetensors b/model-00002-of-00013.safetensors new file mode 100644 index 0000000..18e5453 --- /dev/null +++ b/model-00002-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:115b53f7ad7d75a01d364f1cf6b431a41372d2ee0982fa0de87c655e5f37248d +size 5274455744 diff --git a/model-00003-of-00013.safetensors b/model-00003-of-00013.safetensors new file mode 100644 index 0000000..9813495 --- /dev/null +++ b/model-00003-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48a960c70fe7416cea7407d3786fe2be847e8f5ec3249e7c4b9a787987104536 +size 5159101664 diff --git a/model-00004-of-00013.safetensors b/model-00004-of-00013.safetensors new file mode 100644 index 0000000..85691bb --- /dev/null +++ b/model-00004-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba8ab6116ea38144ebf68f2f53a1ed770536ccf3a4dd0304664560eda726e4e +size 5284951976 diff --git a/model-00005-of-00013.safetensors b/model-00005-of-00013.safetensors new file mode 100644 index 0000000..b837e38 --- /dev/null +++ b/model-00005-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f0700362502e719d00bef14db06ca58e1ca3c52e532a06f58e26f4f32b0a31a +size 5159101664 diff --git a/model-00006-of-00013.safetensors b/model-00006-of-00013.safetensors new file mode 100644 index 0000000..c1aa65b --- /dev/null +++ b/model-00006-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7616544c846b71d90fc493182b42bb3dd5c1c02ae6bbfa1da777e040e2dfe34d +size 5159101680 diff --git a/model-00007-of-00013.safetensors b/model-00007-of-00013.safetensors new file mode 100644 index 0000000..17c87c0 --- /dev/null +++ b/model-00007-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19f166df6e0c2c83f042cb0604d15b5f9cfc8cd3ea1fb4a5d2d524cdd43d955f +size 5284951968 diff --git a/model-00008-of-00013.safetensors b/model-00008-of-00013.safetensors new file mode 100644 index 0000000..8bf8053 --- /dev/null +++ b/model-00008-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f71158f6fcca3945bbd54040a400477b6b834e7468faa7ec9454eb690f2c1839 +size 5159101680 diff --git a/model-00009-of-00013.safetensors b/model-00009-of-00013.safetensors new file mode 100644 index 0000000..72c1016 --- /dev/null +++ b/model-00009-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d1e34385a2ec4fdff956bb4ad1ae6f68da6f426efc023e017339a1b37d3a16b +size 5159101664 diff --git a/model-00010-of-00013.safetensors b/model-00010-of-00013.safetensors new file mode 100644 index 0000000..1978c16 --- /dev/null +++ b/model-00010-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fdc7a3909dd1a3eb314fb5b49d0bfe390f19f6c269640d048036d0ee20421c9 +size 5284951976 diff --git a/model-00011-of-00013.safetensors b/model-00011-of-00013.safetensors new file mode 100644 index 0000000..b809730 --- /dev/null +++ b/model-00011-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c2ab1f5030af66fc9c525b48a10ae2b91ac5d62c0991c1262f1f0ae110a4995 +size 5159101664 diff --git a/model-00012-of-00013.safetensors b/model-00012-of-00013.safetensors new file mode 100644 index 0000000..ad8be55 --- /dev/null +++ b/model-00012-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:489f0a22a4aa4df6ee6d32d81cc963d7188fbb13ed79306de02ebdad7a2f19ce +size 5159101664 diff --git a/model-00013-of-00013.safetensors b/model-00013-of-00013.safetensors new file mode 100644 index 0000000..a2c4c9d --- /dev/null +++ b/model-00013-of-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea6fdce4a0b497d54f1aaa7d33057c0c73448fb90f28fcb9b33b53ddea3b17d1 +size 1384163096 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..4b616a6 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,586 @@ +{ + "metadata": { + "total_size": 63985428480 + }, + "weight_map": { + "lm_head.weight": "model-00001-of-00013.safetensors", + "model.embed_tokens.weight": "model-00001-of-00013.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00013.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00013.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00013.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00013.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.2.input_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00013.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00013.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.22.input_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.23.input_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.24.input_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.25.input_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.26.input_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00013.safetensors", + "model.layers.27.input_layernorm.weight": "model-00004-of-00013.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.28.input_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.29.input_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.3.input_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.30.input_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.31.input_layernorm.weight": "model-00005-of-00013.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00005-of-00013.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.32.input_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.33.input_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.34.input_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.35.input_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.36.input_layernorm.weight": "model-00006-of-00013.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00006-of-00013.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.37.input_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.38.input_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.39.input_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.4.input_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.40.input_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00007-of-00013.safetensors", + "model.layers.41.input_layernorm.weight": "model-00007-of-00013.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.42.input_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.42.mlp.down_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.42.mlp.gate_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.42.mlp.up_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.42.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.42.self_attn.k_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.42.self_attn.o_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.42.self_attn.q_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.42.self_attn.v_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.43.input_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.43.mlp.down_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.43.mlp.gate_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.43.mlp.up_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.43.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.43.self_attn.k_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.43.self_attn.o_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.43.self_attn.q_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.43.self_attn.v_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.44.input_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.44.mlp.down_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.44.mlp.gate_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.44.mlp.up_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.44.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.44.self_attn.k_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.44.self_attn.o_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.44.self_attn.q_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.44.self_attn.v_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.45.input_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.45.mlp.down_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.45.mlp.gate_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.45.mlp.up_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.45.post_attention_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.45.self_attn.k_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.45.self_attn.o_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.45.self_attn.q_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.45.self_attn.v_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.46.input_layernorm.weight": "model-00008-of-00013.safetensors", + "model.layers.46.mlp.down_proj.weight": "model-00008-of-00013.safetensors", + "model.layers.46.mlp.gate_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.46.mlp.up_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.46.post_attention_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.46.self_attn.k_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.46.self_attn.o_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.46.self_attn.q_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.46.self_attn.v_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.47.input_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.47.mlp.down_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.47.mlp.gate_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.47.mlp.up_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.47.post_attention_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.47.self_attn.k_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.47.self_attn.o_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.47.self_attn.q_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.47.self_attn.v_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.48.input_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.48.mlp.down_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.48.mlp.gate_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.48.mlp.up_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.48.post_attention_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.48.self_attn.k_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.48.self_attn.o_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.48.self_attn.q_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.48.self_attn.v_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.49.input_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.49.mlp.down_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.49.mlp.gate_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.49.mlp.up_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.49.post_attention_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.49.self_attn.k_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.49.self_attn.o_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.49.self_attn.q_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.49.self_attn.v_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.5.input_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.50.input_layernorm.weight": "model-00009-of-00013.safetensors", + "model.layers.50.mlp.down_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.50.mlp.gate_proj.weight": "model-00009-of-00013.safetensors", + "model.layers.50.mlp.up_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.50.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.50.self_attn.k_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.50.self_attn.o_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.50.self_attn.q_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.50.self_attn.v_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.51.input_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.51.mlp.down_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.51.mlp.gate_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.51.mlp.up_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.51.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.51.self_attn.k_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.51.self_attn.o_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.51.self_attn.q_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.51.self_attn.v_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.52.input_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.52.mlp.down_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.52.mlp.gate_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.52.mlp.up_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.52.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.52.self_attn.k_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.52.self_attn.o_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.52.self_attn.q_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.52.self_attn.v_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.53.input_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.53.mlp.down_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.53.mlp.gate_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.53.mlp.up_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.53.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.53.self_attn.k_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.53.self_attn.o_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.53.self_attn.q_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.53.self_attn.v_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.54.input_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.54.mlp.down_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.54.mlp.gate_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.54.mlp.up_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.54.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.54.self_attn.k_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.54.self_attn.o_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.54.self_attn.q_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.54.self_attn.v_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.55.input_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.55.mlp.down_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.55.mlp.gate_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.55.mlp.up_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.55.post_attention_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.55.self_attn.k_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.55.self_attn.o_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.55.self_attn.q_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.55.self_attn.v_proj.weight": "model-00010-of-00013.safetensors", + "model.layers.56.input_layernorm.weight": "model-00010-of-00013.safetensors", + "model.layers.56.mlp.down_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.56.mlp.gate_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.56.mlp.up_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.56.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.56.self_attn.k_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.56.self_attn.o_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.56.self_attn.q_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.56.self_attn.v_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.57.input_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.57.mlp.down_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.57.mlp.gate_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.57.mlp.up_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.57.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.57.self_attn.k_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.57.self_attn.o_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.57.self_attn.q_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.57.self_attn.v_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.58.input_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.58.mlp.down_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.58.mlp.gate_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.58.mlp.up_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.58.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.58.self_attn.k_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.58.self_attn.o_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.58.self_attn.q_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.58.self_attn.v_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.59.input_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.59.mlp.down_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.59.mlp.gate_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.59.mlp.up_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.59.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.59.self_attn.k_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.59.self_attn.o_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.59.self_attn.q_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.59.self_attn.v_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.6.input_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.60.input_layernorm.weight": "model-00011-of-00013.safetensors", + "model.layers.60.mlp.down_proj.weight": "model-00011-of-00013.safetensors", + "model.layers.60.mlp.gate_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.60.mlp.up_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.60.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.60.self_attn.k_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.60.self_attn.o_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.60.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.60.self_attn.v_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.61.input_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.61.mlp.down_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.61.mlp.gate_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.61.mlp.up_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.61.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.61.self_attn.k_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.61.self_attn.o_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.61.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.61.self_attn.v_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.62.input_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.62.mlp.down_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.62.mlp.gate_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.62.mlp.up_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.62.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.62.self_attn.k_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.62.self_attn.o_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.62.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.62.self_attn.v_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.63.input_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.63.mlp.down_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.63.mlp.gate_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.63.mlp.up_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.63.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.63.self_attn.k_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.63.self_attn.o_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.63.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.63.self_attn.v_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.7.input_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.8.input_layernorm.weight": "model-00012-of-00013.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00012-of-00013.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00013-of-00013.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.9.input_layernorm.weight": "model-00013-of-00013.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00013-of-00013.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", + "model.norm.weight": "model-00013-of-00013.safetensors" + } +} \ No newline at end of file diff --git a/modeling_iquestcoder.py b/modeling_iquestcoder.py new file mode 100644 index 0000000..800a462 --- /dev/null +++ b/modeling_iquestcoder.py @@ -0,0 +1,1063 @@ +""" +Modified MIT License + +Software Copyright© 2025 IQuest Research + +Our only modification is that, if the Software (or any derivative works +thereof) is used for any of your commercial products or services, you shall +prominently display "IQuest Coder" on the user interface of such product or +service. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +from typing import Callable, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache, DynamicCache, SlidingWindowCache, StaticCache +from transformers.generation import GenerationMixin +from transformers.modeling_attn_mask_utils import AttentionMaskConverter +from transformers.modeling_flash_attention_utils import FlashAttentionKwargs +from transformers.modeling_layers import GradientCheckpointingLayer +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, + QuestionAnsweringModelOutput, + SequenceClassifierOutputWithPast, + TokenClassifierOutput, +) +from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from transformers.processing_utils import Unpack +from transformers.utils import ( + auto_docstring, + can_return_tuple, + is_torch_flex_attn_available, + logging, +) + +from .configuration_iquestcoder import IQuestCoderConfig + + +if is_torch_flex_attn_available(): + from torch.nn.attention.flex_attention import BlockMask + from transformers.integrations.flex_attention import make_flex_block_causal_mask + + +logger = logging.get_logger(__name__) + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def rotate_half(x: torch.Tensor) -> torch.Tensor: + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: Optional[torch.Tensor] = None, + unsqueeze_dim: int = 1, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q: The query tensor. + k: The key tensor. + cos: The cosine part of the rotary embedding. + sin: The sine part of the rotary embedding. + position_ids: Deprecated and unused. + unsqueeze_dim: The dimension along which to unsqueeze cos and sin. + + Returns: + Tuple of query and key tensors rotated using the Rotary Position Embedding. + """ + # Borrowed from OLMo: preserve original dtypes for numerical stability + q_dtype, k_dtype = q.dtype, k.dtype + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed.to(q_dtype), k_embed.to(k_dtype) + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + Expands key/value heads for Grouped Query Attention. + + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). + The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to + (batch, num_attention_heads, seqlen, head_dim). + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Standard eager attention implementation.""" + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = F.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +# ============================================================================= +# Model Components +# ============================================================================= + +class IQuestCoderRMSNorm(nn.Module): + """Root Mean Square Layer Normalization. + + RMSNorm is computationally simpler than LayerNorm while achieving similar + performance. It normalizes the input by its RMS value. + """ + + def __init__(self, hidden_size: int, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self) -> str: + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class IQuestCoderRotaryEmbedding(nn.Module): + """Rotary Position Embedding (RoPE). + + Implements rotary positional embeddings as described in the RoFormer paper. + Supports various RoPE scaling methods for extended context lengths. + """ + + def __init__(self, config: IQuestCoderConfig, device=None): + super().__init__() + # BC: "rope_type" was originally "type" + if hasattr(config, "rope_scaling") and config.rope_scaling is not None: + self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type")) + else: + self.rope_type = "default" + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + + inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = self.inv_freq + + @torch.no_grad() + @dynamic_rope_update + def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class IQuestCoderMLP(nn.Module): + """Feed-forward network with SwiGLU activation. + + Uses the gated linear unit variant with SiLU activation for improved + performance compared to standard FFN. + """ + + def __init__(self, config: IQuestCoderConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # SwiGLU: down_proj(act_fn(gate_proj(x)) * up_proj(x)) + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +class IQuestCoderAttention(nn.Module): + """Multi-headed attention with support for Grouped Query Attention (GQA). + + Features: + - Grouped Query Attention for memory efficiency + - Optional QKV clipping for training stability (from OLMo) + - Optional sliding window attention (from Qwen2) + - Rotary Position Embeddings + """ + + def __init__(self, config: IQuestCoderConfig, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim ** -0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + + # Projection layers + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: Tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + # Compute Q, K, V projections + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + # [OLMo Feature] Optional QKV clipping for training stability + if self.config.clip_qkv is not None: + query_states = query_states.clamp(min=-self.config.clip_qkv, max=self.config.clip_qkv) + key_states = key_states.clamp(min=-self.config.clip_qkv, max=self.config.clip_qkv) + value_states = value_states.clamp(min=-self.config.clip_qkv, max=self.config.clip_qkv) + + # Reshape to (batch, heads, seq_len, head_dim) + query_states = query_states.view(hidden_shape).transpose(1, 2) + key_states = key_states.view(hidden_shape).transpose(1, 2) + value_states = value_states.view(hidden_shape).transpose(1, 2) + + # Apply rotary position embeddings + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + # Update KV cache if provided + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # [Qwen2 Feature] Sliding window attention + sliding_window = None + if ( + self.config.use_sliding_window + and getattr(self.config, "sliding_window", None) is not None + and self.layer_idx >= self.config.max_window_layers + ): + sliding_window = self.config.sliding_window + + # Select attention implementation + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False): + logger.warning_once( + "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. " + 'Falling back to eager attention. This warning can be removed using the argument ' + '`attn_implementation="eager"` when loading the model.' + ) + else: + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + # Compute attention + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=sliding_window, + **kwargs, + ) + + # Reshape and project output + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + + return attn_output, attn_weights + + +class IQuestCoderDecoderLayer(GradientCheckpointingLayer): + """Transformer decoder layer with pre-normalization. + + Architecture: Pre-RMSNorm -> Attention -> Residual -> Pre-RMSNorm -> MLP -> Residual + """ + + def __init__(self, config: IQuestCoderConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = IQuestCoderAttention(config=config, layer_idx=layer_idx) + self.mlp = IQuestCoderMLP(config) + self.input_layernorm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + # Warn if sliding window is enabled but not properly supported + if config.use_sliding_window and config._attn_implementation != "flash_attention_2": + logger.warning_once( + f"Sliding Window Attention is enabled but not implemented for `{config._attn_implementation}`; " + "unexpected results may be encountered." + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + # Pre-norm + Self Attention + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + hidden_states, self_attn_weights = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Pre-norm + MLP + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + if output_attentions: + outputs += (self_attn_weights,) + + return outputs + + +# ============================================================================= +# Base Model +# ============================================================================= + +@auto_docstring +class IQuestCoderPreTrainedModel(PreTrainedModel): + """Base class for IQuestCoder models.""" + + config_class = IQuestCoderConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["IQuestCoderDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_flex_attn = True + _supports_cache_class = True + _supports_quantized_cache = True + _supports_static_cache = True + _supports_attention_backend = True + + def _init_weights(self, module: nn.Module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, IQuestCoderRMSNorm): + module.weight.data.fill_(1.0) + + +@auto_docstring +class IQuestCoderModel(IQuestCoderPreTrainedModel): + """ + IQuestCoder Model outputting raw hidden-states without any specific head on top. + + This model is compatible with LLaMA weights while incorporating features from OLMo and Qwen2. + """ + + def __init__(self, config: IQuestCoderConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [IQuestCoderDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = IQuestCoderRotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Embedding: + return self.embed_tokens + + def set_input_embeddings(self, value: nn.Embedding): + self.embed_tokens = value + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **flash_attn_kwargs: Unpack[FlashAttentionKwargs], + ) -> BaseModelOutputWithPast: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if self.gradient_checkpointing and self.training and use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." + ) + use_cache = False + + if not isinstance(past_key_values, (type(None), Cache)): + raise ValueError("The `past_key_values` should be either a `Cache` object or `None`.") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache() + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = self._update_causal_mask( + attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions + ) + + hidden_states = inputs_embeds + + # Create position embeddings to be shared across the decoder layers + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + # Decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **flash_attn_kwargs, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # Add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values if use_cache else None, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + def _update_causal_mask( + self, + attention_mask: Union[torch.Tensor, "BlockMask"], + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_key_values: Cache, + output_attentions: bool = False, + ): + if self.config._attn_implementation == "flash_attention_2": + if attention_mask is not None and past_key_values is not None: + is_padding_right = attention_mask[:, -1].sum().item() != input_tensor.size()[0] + if is_padding_right: + raise ValueError( + "You are attempting to perform batched generation with padding_side='right'. " + "This may lead to unexpected behaviour for Flash Attention version of IQuestCoder. " + "Make sure to call `tokenizer.padding_side = 'left'` before tokenizing the input." + ) + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + + if self.config._attn_implementation == "flex_attention": + if isinstance(attention_mask, torch.Tensor): + attention_mask = make_flex_block_causal_mask(attention_mask) + return attention_mask + + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + using_static_cache = isinstance(past_key_values, StaticCache) + using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache) + + if ( + self.config._attn_implementation == "sdpa" + and not (using_static_cache or using_sliding_window_cache) + and not output_attentions + ): + if AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask, + inputs_embeds=input_tensor, + past_key_values_length=past_seen_tokens, + sliding_window=self.config.sliding_window if self.config.use_sliding_window else None, + is_training=self.training, + ): + return None + + dtype = input_tensor.dtype + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + + if using_sliding_window_cache or using_static_cache: + target_length = past_key_values.get_max_cache_shape() + else: + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position( + attention_mask, + sequence_length=sequence_length, + target_length=target_length, + dtype=dtype, + cache_position=cache_position, + batch_size=input_tensor.shape[0], + config=self.config, + past_key_values=past_key_values, + ) + + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type in ["cuda", "xpu", "npu"] + and not output_attentions + ): + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + @staticmethod + def _prepare_4d_causal_attention_mask_with_cache_position( + attention_mask: torch.Tensor, + sequence_length: int, + target_length: int, + dtype: torch.dtype, + cache_position: torch.Tensor, + batch_size: int, + config: IQuestCoderConfig, + past_key_values: Cache, + ): + """Creates a causal 4D mask from a 2D mask, or returns the 4D mask if already provided.""" + if attention_mask is not None and attention_mask.dim() == 4: + causal_mask = attention_mask + else: + min_dtype = torch.finfo(dtype).min + causal_mask = torch.full( + (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=cache_position.device + ) + diagonal_attend_mask = torch.arange(target_length, device=cache_position.device) > cache_position.reshape( + -1, 1 + ) + + # [Qwen2 Feature] Handle sliding window mask + if getattr(config, "use_sliding_window", False) and config.sliding_window is not None: + if not isinstance(past_key_values, SlidingWindowCache) or sequence_length > target_length: + sliding_attend_mask = torch.arange(target_length, device=cache_position.device) <= ( + cache_position.reshape(-1, 1) - config.sliding_window + ) + diagonal_attend_mask.bitwise_or_(sliding_attend_mask) + + causal_mask *= diagonal_attend_mask + causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1) + + if attention_mask is not None: + causal_mask = causal_mask.clone() + if attention_mask.shape[-1] > target_length: + attention_mask = attention_mask[:, :target_length] + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to( + causal_mask.device + ) + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + + return causal_mask + + +# ============================================================================= +# Model Heads +# ============================================================================= + +@auto_docstring +class IQuestCoderForCausalLM(IQuestCoderPreTrainedModel, GenerationMixin): + """IQuestCoder Model with a language modeling head on top for causal LM.""" + + _tied_weights_keys = ["lm_head.weight"] + _tp_plan = {"lm_head": "colwise_rep"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + + def __init__(self, config: IQuestCoderConfig): + super().__init__(config) + self.model = IQuestCoderModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Embedding: + return self.model.embed_tokens + + def set_input_embeddings(self, value: nn.Embedding): + self.model.embed_tokens = value + + def get_output_embeddings(self) -> nn.Linear: + return self.lm_head + + def set_output_embeddings(self, new_embeddings: nn.Linear): + self.lm_head = new_embeddings + + def set_decoder(self, decoder: IQuestCoderModel): + self.model = decoder + + def get_decoder(self) -> IQuestCoderModel: + return self.model + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs + ) -> CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Example: + ```python + >>> from transformers import AutoTokenizer + >>> from modeling_iquestcoder import IQuestCoderForCausalLM + + >>> model = IQuestCoderForCausalLM.from_pretrained("path/to/IQuestCoder") + >>> tokenizer = AutoTokenizer.from_pretrained("path/to/IQuestCoder") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\\nI'm not conscious, but I can talk to you." + ``` + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + # Decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@auto_docstring( + custom_intro=""" + The IQuestCoder Model transformer with a sequence classification head on top (linear layer). + + [`IQuestCoderForSequenceClassification`] uses the last token in order to do the classification, as other causal + models (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. + If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. + """ +) +class IQuestCoderForSequenceClassification(IQuestCoderPreTrainedModel): + """IQuestCoder Model with a sequence classification head.""" + + def __init__(self, config: IQuestCoderConfig): + super().__init__(config) + self.num_labels = config.num_labels + self.model = IQuestCoderModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Embedding: + return self.model.embed_tokens + + def set_input_embeddings(self, value: nn.Embedding): + self.model.embed_tokens = value + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ) -> SequenceClassifierOutputWithPast: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + transformer_outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + hidden_states = transformer_outputs.last_hidden_state + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + last_non_pad_token = -1 + elif input_ids is not None: + non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32) + token_indices = torch.arange(input_ids.shape[-1], device=logits.device, dtype=torch.int32) + last_non_pad_token = (token_indices * non_pad_mask).argmax(-1) + else: + last_non_pad_token = -1 + logger.warning_once( + f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be " + "unexpected if using padding tokens in conjunction with `inputs_embeds.`" + ) + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token] + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config) + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + +@auto_docstring +class IQuestCoderForTokenClassification(IQuestCoderPreTrainedModel): + """IQuestCoder Model with a token classification head.""" + + def __init__(self, config: IQuestCoderConfig): + super().__init__(config) + self.num_labels = config.num_labels + self.model = IQuestCoderModel(config) + if getattr(config, "classifier_dropout", None) is not None: + classifier_dropout = config.classifier_dropout + elif getattr(config, "hidden_dropout", None) is not None: + classifier_dropout = config.hidden_dropout + else: + classifier_dropout = 0.1 + self.dropout = nn.Dropout(classifier_dropout) + self.score = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Embedding: + return self.model.embed_tokens + + def set_input_embeddings(self, value: nn.Embedding): + self.model.embed_tokens = value + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ) -> TokenClassifierOutput: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + sequence_output = outputs.last_hidden_state + sequence_output = self.dropout(sequence_output) + logits = self.score(sequence_output) + + loss = None + if labels is not None: + loss = self.loss_function(logits, labels, self.config) + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@auto_docstring +class IQuestCoderForQuestionAnswering(IQuestCoderPreTrainedModel): + """IQuestCoder Model with a span classification head for extractive question-answering.""" + + base_model_prefix = "transformer" + + def __init__(self, config: IQuestCoderConfig): + super().__init__(config) + self.transformer = IQuestCoderModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, 2) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Embedding: + return self.transformer.embed_tokens + + def set_input_embeddings(self, value: nn.Embedding): + self.transformer.embed_tokens = value + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + start_positions: Optional[torch.LongTensor] = None, + end_positions: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + **kwargs, + ) -> QuestionAnsweringModelOutput: + outputs: BaseModelOutputWithPast = self.transformer( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + + sequence_output = outputs.last_hidden_state + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1).contiguous() + end_logits = end_logits.squeeze(-1).contiguous() + + loss = None + if start_positions is not None and end_positions is not None: + loss = self.loss_function(start_logits, end_logits, start_positions, end_positions, **kwargs) + + return QuestionAnsweringModelOutput( + loss=loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = [ + "IQuestCoderPreTrainedModel", + "IQuestCoderModel", + "IQuestCoderForCausalLM", + "IQuestCoderForSequenceClassification", + "IQuestCoderForTokenClassification", + "IQuestCoderForQuestionAnswering", +] + diff --git a/tokenization_iquestcoder.py b/tokenization_iquestcoder.py new file mode 100644 index 0000000..9863d29 --- /dev/null +++ b/tokenization_iquestcoder.py @@ -0,0 +1,552 @@ +"""Tokenization classes for IQuestCoder.""" + +import os +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple, Union + +import sentencepiece as spm + +from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": {}, + "tokenizer_file": {}, +} +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} + + + +class IQuestCoderTokenizer(PreTrainedTokenizer): + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + unk_token="", + bos_token="", + eos_token="", + pad_token=None, + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + clean_up_tokenization_spaces=False, + add_prefix_space=False, + legacy=None, + use_default_system_prompt=False, + chat_template=None, + **kwargs, + ): + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token + eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token + unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token + pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token + + # Legacy behavior handling + if legacy is None: + logger.warning_once( + f"You are using the default legacy behaviour of the {self.__class__.__name__}. This is" + " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you." + " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it" + " means, and thoroughly read the reason why this was added as explained in" + " https://github.com/huggingface/transformers/pull/24565" + ) + legacy = True + + self.legacy = legacy + self.vocab_file = vocab_file + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + self.add_prefix_space = add_prefix_space + self.use_default_system_prompt = use_default_system_prompt + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(vocab_file) + + + + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + add_bos_token=add_bos_token, + add_eos_token=add_eos_token, + sp_model_kwargs=self.sp_model_kwargs, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + add_prefix_space=add_prefix_space, + legacy=legacy, + use_default_system_prompt=use_default_system_prompt, + chat_template=chat_template, + **kwargs, + ) + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(self.vocab_file) + + @property + def vocab_size(self) -> int: + """Returns the vocabulary size.""" + return self.sp_model.get_piece_size() + + def get_vocab(self) -> Dict[str, int]: + """Returns the vocabulary as a dictionary of token to index.""" + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text: str) -> List[str]: + """ + Tokenize a string. + + Args: + text (`str`): The text to tokenize. + + Returns: + `List[str]`: The list of tokens. + """ + if self.add_prefix_space: + text = " " + text + + if self.legacy: + return self.sp_model.encode(text, out_type=str) + + # Non-legacy behavior: handle special tokens properly + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token: str) -> int: + """Converts a token (str) to an id using the vocab.""" + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index: int) -> str: + """Converts an index (integer) to a token (str) using the vocab.""" + token = self.sp_model.IdToPiece(index) + return token + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + """ + Converts a sequence of tokens (strings) to a single string. + + This method handles special tokens separately to ensure they are not + decoded using the SentencePiece model. + + Args: + tokens (`List[str]`): The list of tokens to convert. + + Returns: + `str`: The decoded string. + """ + current_sub_tokens = [] + out_string = "" + prev_is_special = False + for i, token in enumerate(tokens): + # make sure that special tokens are not decoded using sentencepiece model + if token in self.all_special_tokens: + if not prev_is_special and i != 0: + out_string += " " + out_string += self.sp_model.decode(current_sub_tokens) + token + prev_is_special = True + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + prev_is_special = False + out_string += self.sp_model.decode(current_sub_tokens) + return out_string + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + filename_prefix (`str`, *optional*): + An optional prefix to add to the named of the saved files. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + out_vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, "wb") as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequences for sequence classification tasks by concatenating + and adding special tokens. + + An IQuestCoder sequence has the following format: + + - single sequence: ` X ` (if add_eos_token is True) or ` X` (default) + - pair of sequences: ` A B ` (if add_eos_token is True) or ` A B` (default) + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of input IDs with the appropriate special tokens. + """ + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = bos_token_id + token_ids_0 + eos_token_id + + if token_ids_1 is not None: + output = output + bos_token_id + token_ids_1 + eos_token_id + + return output + + def get_special_tokens_mask( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + ) + + bos_token_id = [1] if self.add_bos_token else [] + eos_token_id = [1] if self.add_eos_token else [] + + if token_ids_1 is None: + return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id + return ( + bos_token_id + + ([0] * len(token_ids_0)) + + eos_token_id + + bos_token_id + + ([0] * len(token_ids_1)) + + eos_token_id + ) + + def create_token_type_ids_from_sequences( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. + + An IQuestCoder sequence pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of token type IDs according to the given sequence(s). + """ + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = [0] * len(bos_token_id + token_ids_0 + eos_token_id) + + if token_ids_1 is not None: + output += [1] * len(bos_token_id + token_ids_1 + eos_token_id) + + return output + + @property + def default_chat_template(self) -> str: + """ + Returns the default chat template for IQuestCoder. + + This template formats conversations with system, user, and assistant roles. + """ + return DEFAULT_CHAT_TEMPLATE + + def apply_chat_template( + self, + conversation: Union[List[Dict[str, str]], "Conversation"], + chat_template: Optional[str] = None, + add_generation_prompt: bool = False, + tokenize: bool = True, + padding: bool = False, + truncation: bool = False, + max_length: Optional[int] = None, + return_tensors: Optional[str] = None, + return_dict: bool = False, + **tokenizer_kwargs, + ): + """ + Apply a chat template to format a conversation. + + Args: + conversation (`List[Dict[str, str]]` or `Conversation`): + A list of dicts with "role" and "content" keys, representing the conversation history. + chat_template (`str`, *optional*): + A Jinja template to use for formatting. If not provided, the tokenizer's default will be used. + add_generation_prompt (`bool`, *optional*, defaults to `False`): + Whether to add a generation prompt at the end for the assistant to continue. + tokenize (`bool`, *optional*, defaults to `True`): + Whether to tokenize the output. If `False`, returns a string. + padding (`bool`, *optional*, defaults to `False`): + Whether to pad sequences. + truncation (`bool`, *optional*, defaults to `False`): + Whether to truncate sequences. + max_length (`int`, *optional*): + Maximum length of the output. + return_tensors (`str`, *optional*): + The type of tensors to return ("pt", "tf", "np", or None). + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a dictionary with additional information. + **tokenizer_kwargs: + Additional keyword arguments passed to the tokenizer. + + Returns: + `Union[str, List[int], BatchEncoding]`: The formatted (and optionally tokenized) conversation. + + Example: + ```python + >>> tokenizer = IQuestCoderTokenizer.from_pretrained("path/to/model") + >>> conversation = [ + ... {"role": "system", "content": "You are a helpful assistant."}, + ... {"role": "user", "content": "Hello!"}, + ... {"role": "assistant", "content": "Hi there! How can I help you today?"}, + ... {"role": "user", "content": "What's the weather like?"}, + ... ] + >>> tokenizer.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False) + '<|system|>\\nYou are a helpful assistant.\\n<|user|>\\nHello!\\n...' + ``` + """ + # Use parent class implementation with our template + return super().apply_chat_template( + conversation, + chat_template=chat_template, + add_generation_prompt=add_generation_prompt, + tokenize=tokenize, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + return_dict=return_dict, + **tokenizer_kwargs, + ) + + +# Try to import and create Fast tokenizer version +try: + from transformers import PreTrainedTokenizerFast + from tokenizers import Tokenizer, decoders, models, normalizers, pre_tokenizers, processors + + class IQuestCoderTokenizerFast(PreTrainedTokenizerFast): + """ + Construct a "fast" IQuestCoder tokenizer (backed by HuggingFace's *tokenizers* library). + + This is a fast implementation of [`IQuestCoderTokenizer`] using the 🤗 Tokenizers library. + + Args: + vocab_file (`str`, *optional*): + Path to the vocabulary file (SentencePiece model). + tokenizer_file (`str`, *optional*): + Path to a tokenizer JSON file. + unk_token (`str`, *optional*, defaults to `""`): + The unknown token. + bos_token (`str`, *optional*, defaults to `""`): + The beginning of sequence token. + eos_token (`str`, *optional*, defaults to `""`): + The end of sequence token. + pad_token (`str`, *optional*): + The token used for padding. + add_bos_token (`bool`, *optional*, defaults to `True`): + Whether to add a BOS token at the start of sequences. + add_eos_token (`bool`, *optional*, defaults to `False`): + Whether to add an EOS token at the end of sequences. + add_prefix_space (`bool`, *optional*, defaults to `False`): + Whether to add an initial space to the input. + use_default_system_prompt (`bool`, *optional*, defaults to `False`): + Whether to use the default system prompt. + chat_template (`str`, *optional*): + A Jinja template for formatting conversations. + + Example: + ```python + >>> from tokenization_iquestcoder import IQuestCoderTokenizerFast + + >>> tokenizer = IQuestCoderTokenizerFast.from_pretrained("path/to/model") + >>> tokenizer.encode("Hello, world!") + [1, 15043, 29892, 3186, 29991] + ``` + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask"] + slow_tokenizer_class = IQuestCoderTokenizer + + def __init__( + self, + vocab_file=None, + tokenizer_file=None, + unk_token="", + bos_token="", + eos_token="", + pad_token=None, + add_bos_token=True, + add_eos_token=False, + add_prefix_space=False, + use_default_system_prompt=False, + chat_template=None, + **kwargs, + ): + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + self.add_prefix_space = add_prefix_space + self.use_default_system_prompt = use_default_system_prompt + + if chat_template is None: + chat_template = DEFAULT_CHAT_TEMPLATE + + super().__init__( + vocab_file=vocab_file, + tokenizer_file=tokenizer_file, + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + add_bos_token=add_bos_token, + add_eos_token=add_eos_token, + add_prefix_space=add_prefix_space, + use_default_system_prompt=use_default_system_prompt, + chat_template=chat_template, + **kwargs, + ) + + @property + def can_save_slow_tokenizer(self) -> bool: + return os.path.isfile(self.vocab_file) if self.vocab_file else False + + @property + def default_chat_template(self) -> str: + """Returns the default chat template.""" + return DEFAULT_CHAT_TEMPLATE + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """Build model inputs with special tokens.""" + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = bos_token_id + token_ids_0 + eos_token_id + + if token_ids_1 is not None: + output = output + bos_token_id + token_ids_1 + eos_token_id + + return output + + def get_special_tokens_mask( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False + ) -> List[int]: + """Retrieve special tokens mask.""" + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + ) + + bos_token_id = [1] if self.add_bos_token else [] + eos_token_id = [1] if self.add_eos_token else [] + + if token_ids_1 is None: + return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id + return ( + bos_token_id + + ([0] * len(token_ids_0)) + + eos_token_id + + bos_token_id + + ([0] * len(token_ids_1)) + + eos_token_id + ) + + def create_token_type_ids_from_sequences( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """Create token type IDs from sequences.""" + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = [0] * len(bos_token_id + token_ids_0 + eos_token_id) + + if token_ids_1 is not None: + output += [1] * len(bos_token_id + token_ids_1 + eos_token_id) + + return output + +except ImportError: + # tokenizers library not available, Fast tokenizer not supported + IQuestCoderTokenizerFast = None + logger.info( + "The `tokenizers` library is not installed. " + "IQuestCoderTokenizerFast will not be available. " + "Install it with `pip install tokenizers`." + ) + diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..c5da580 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3be68e090a927f31e0e378d7599b15c206dd47e4a73933775a746cc9c1cd91 +size 1345108 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..af0e3b8 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": true, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": true, + "special": true + }, + "75858": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75859": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75860": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75861": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75862": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75863": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75864": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75865": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75866": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75867": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75868": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75869": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75870": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75871": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75872": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75873": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75874": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75875": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75876": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75877": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75878": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75879": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|CLS|>", + "<|SEP|>", + "<|EOD|>", + "<|MASK|>", + "<|PAD|>", + "<|fim_prefix|>", + "<|fim_middle|>", + "<|fim_suffix|>", + "<|im_start|>", + "<|im_end|>", + "<|fim_pad|>", + "<|endoftext|>", + "<|repo_name|>", + "<|file_sep|>" + ], + "auto_map": { + "AutoTokenizer": [ + "tokenization_iquestcoder.IQuestCoderTokenizer", + null + ] + }, + "bos_token": "", + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- else %}\n {{- 'You are IndustrialCoder, a helpful assistant developed by Beihang University.' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {%- for tool in tools %}\n {%- if tool.type == 'function' and tool.function %}\n {%- set func = tool.function %}\n {%- else %}\n {%- set func = tool %}\n {%- endif %}\n {{- \"\\n\\n\" + func.name + \"\" }}\n {%- if func.description %}\n {{- \"\\n\" + func.description + \"\" }}\n {%- endif %}\n {{- \"\\n\" }}\n {%- if func.parameters and func.parameters.properties %}\n {%- for param_name, param_fields in func.parameters.properties.items() %}\n {{- \"\\n\" }}\n {{- \"\\n\" + param_name + \"\" }}\n {%- if param_fields.type %}\n {{- \"\\n\" + param_fields.type + \"\" }}\n {%- endif %}\n {%- if param_fields.description %}\n {{- \"\\n\" + param_fields.description + \"\" }}\n {%- endif %}\n {{- \"\\n\" }}\n {%- endfor %}\n {%- endif %}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- \"\\n\\n\\nIf you choose to call a function ONLY reply in the following format:\\n\\n\\n\\n\\nvalue_1\\n\\n\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are IndustrialCoder, a helpful assistant developed by Beihang University.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tc = tool_call.function %}\n {%- else %}\n {%- set tc = tool_call %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- if tc.arguments is string %}\n {%- set args = tc.arguments | fromjson %}\n {%- else %}\n {%- set args = tc.arguments %}\n {%- endif %}\n {%- for arg_name, arg_value in args.items() %}\n {{- '\\n' }}\n {%- if arg_value is string %}\n {{- arg_value }}\n {%- else %}\n {{- arg_value | tojson }}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endfor %}\n {{- '\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "sp_model_kwargs": {}, + "split_special_tokens": false, + "tokenizer_class": "IQuestCoderTokenizer", + "unk_token": "", + "use_fast": false +}