From db932fe0b11b5ea4c4c9554e3ee32aa3a446d1e1 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 19 May 2026 20:10:08 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: BAAI/OpenSeek-Mid-v1 Source: Original Platform --- .gitattributes | 66 ++ README.md | 244 ++++++ config.json | 32 + configuration.json | 1 + configuration_qwen3.py | 208 ++++++ generation_config.json | 6 + model-00001-of-00005.safetensors | 3 + model-00002-of-00005.safetensors | 3 + model-00003-of-00005.safetensors | 3 + model-00004-of-00005.safetensors | 3 + model-00005-of-00005.safetensors | 3 + model.safetensors.index.json | 626 ++++++++++++++++ modeling_qwen3.py | 1204 ++++++++++++++++++++++++++++++ qwen.tiktoken | 3 + qwen_generation_utils.py | 416 +++++++++++ special_tokens_map.json | 6 + tokenization_qwen.py | 276 +++++++ tokenizer_config.json | 10 + 18 files changed, 3113 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 configuration_qwen3.py create mode 100644 generation_config.json create mode 100644 model-00001-of-00005.safetensors create mode 100644 model-00002-of-00005.safetensors create mode 100644 model-00003-of-00005.safetensors create mode 100644 model-00004-of-00005.safetensors create mode 100644 model-00005-of-00005.safetensors create mode 100644 model.safetensors.index.json create mode 100644 modeling_qwen3.py create mode 100644 qwen.tiktoken create mode 100644 qwen_generation_utils.py create mode 100644 special_tokens_map.json create mode 100644 tokenization_qwen.py create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ec666cb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,66 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +iter_0023860_hf/model-00001-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text +iter_0023860_hf/model-00002-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text +iter_0023860_hf/model-00003-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text +iter_0023860_hf/model-00004-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text +iter_0023860_hf/model-00005-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text +iter_0023860_hf/model-00006-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text +iter_0023860_hf/model-00007-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text + +model-00001-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text + +model-00002-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text + +model-00003-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text + +model-00004-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text + +model-00005-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text + +qwen.tiktoken filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..20cdeb2 --- /dev/null +++ b/README.md @@ -0,0 +1,244 @@ +--- +extra_gated_prompt: >- + You agree to not use the dataset to conduct experiments that cause harm to + human subjects. +extra_gated_fields: + Company/Organization: text + Country: country +pipeline_tag: text-generation +library_name: transformers +--- +--- +license: apache-2.0 +--- +# OpenSeek-Mid-v1 + +**OpenSeek-Mid-v1** is a 10.61-billion-parameter language model grown from [Qwen3-4B-Base](https://huggingface.co/Qwen/Qwen3-4B-Base) through a two-stage model expansion pipeline and trained on only **2 trillion tokens** of fully open-source data. + +Despite having **25% fewer parameters** and using **18x less training data**, OpenSeek-Mid-v1 matches or surpasses Qwen3-14B-Base across multiple benchmarks. + + + +results_all + + +--- + +## Highlights + +- **Model Growth, Not From-Scratch Training**: Grown from Qwen3-4B via width expansion + partial depth stacking, inheriting the seed model's learned representations. +- **Extreme Data Efficiency**: Matches Qwen3-14B-Base (~36T tokens) with only 2T tokens of training — an 18x reduction in data requirement. +- **Muon Optimizer**: Spectral whitening ensures expanded dimensions are effectively utilized, delivering significant gains over AdamW in the model growth setting. +- **Fully Open-Source Data**: All training data comes from publicly available datasets (NemotronCC-v2, Stack-Edu, Dolmino, CCI, etc.). + +--- + +## Architecture + +| Specification | Value | +|---|---| +| Parameters | 10.61B | +| Layers | 56 | +| Hidden Size (d_model) | 2560 | +| FFN Intermediate Size (d_FFN) | 19456 | +| Attention Heads | 32 | +| KV Heads | 8 | +| Sequence Length | 8192 | +| Vocabulary Size | same as Qwen3-4B | + +### Growth Pipeline + +``` +Qwen3-4B (4.02B, 36L) + │ Width expansion (d_FFN: 9728 → 19456, SNR=10dB) + ▼ +Width-Expanded (7.10B, 36L) + │ Partial depth stacking (layers 14–34 × 2) + ▼ +OpenSeek-Mid-v1 (10.61B, 56L) + │ Continual pretraining with Muon (2T tokens) + ▼ +Final Model +``` + +--- + +## Training + +### Training Configuration + +| Parameter | Value | +|---|---| +| Optimizer | Muon | +| Sequence Length | 8192 | +| Global Batch Size | 2048 sequences | +| Peak Learning Rate | 1e-4 | +| LR Schedule | Cosine with linear warmup | +| Warmup Steps | 1000 | +| Weight Decay | 0.1 | +| Training Framework | FlagScale (FlagOS) | +| Total Training Tokens | ~2.06T | + +### Stage 1: Broad Knowledge Acquisition (1.36T tokens) + +#### Stage 1 Data Mixture + +| Category | Proportion | Tokens (B) | +|---|---|---| +| Web | 42% | ~571B | +| Math | 20% | ~272B | +| Code | 20% | ~272B | +| STEM | 15% | ~204B | +| Multilingual | 3% | ~41B | + +--- + +### Stage 2: Capability Specialization (0.70T tokens) + +#### Stage 2 Data Mixture + +| Category | Proportion | Tokens (B) | Delta vs. Stage 1 | +|---|---|---|---| +| Web | 35% | ~245B | -7% | +| Math | 20% | ~140B | — | +| Code | 24% | ~168B | +4% | +| STEM | 18% | ~126B | +3% | +| Multilingual | 3% | ~21B | — | + +--- + +### Detailed Dataset Composition + +Stage 1 (%) and Stage 2 (%) denote each dataset's sampling weight within the respective stage. "—" indicates the dataset is not used in that stage. + +**Web** + +| Dataset | Tokens (B) | Stage 1 (%) | Stage 2 (%) | +|---|---|---|---| +| Nemotron-CC-v2-HQ-Syn | 798.41 | 23.24 | 19.36 | +| Nemotron-CC-v2-Diverse-QA (×5 shards) | 340.81 | 9.92 | 8.26 | +| Nemotron-CC-v2-HQ (×5 shards) | 303.82 | 8.84 | 7.36 | +| dolmino-mix-1124-wiki | 3.82 | 0.15 | 0.18 | +| dolmino-mix-1124-stackexchange | 1.30 | 0.05 | 0.06 | + +**Math** + +| Dataset | Tokens (B) | Stage 1 (%) | Stage 2 (%) | +|---|---|---|---| +| Nemotron-SFT-MATH | 207.46 | 11.70 | 11.70 | +| Nemotron-CC-Math-v1-4plus-MIND | 74.34 | 4.19 | 4.19 | +| Nemotron-CC-Math-v1-4plus | 53.37 | 3.01 | 3.01 | +| Dolmino-math | 11.17 | 0.63 | 0.63 | +| OpenMathInstruct-2 | 5.30 | 0.30 | 0.30 | +| OpenMathReasoning-4k | 2.48 | 0.14 | 0.14 | +| NuminaMath-1.5 | 0.38 | 0.02 | 0.02 | + +**Code** + +| Dataset | Tokens (B) | Stage 1 (%) | Stage 2 (%) | +|---|---|---|---| +| Nemotron-Pretraining-Code-v1-Syn | 171.53 | 9.05 | 10.86 | +| Nemotron-SFT-Code | 57.47 | 3.03 | 3.64 | +| stack-edu-Java | 31.70 | 1.06 | 1.27 | +| stack-edu-Markdown | 26.64 | 0.38 | 0.45 | +| stack-edu-Python | 18.27 | 1.54 | 1.85 | +| stack-edu-Cpp | 12.62 | 1.11 | 1.33 | +| stack-edu-JavaScript | 8.99 | 1.00 | 1.20 | +| stack-edu-SQL | 8.23 | 0.37 | 0.44 | +| github-issue | 8.46 | 0.25 | 0.30 | +| stack-edu-PHP | 7.43 | 0.25 | 0.30 | +| stack-edu-CSharp | 7.26 | 0.37 | 0.44 | +| stack-edu-C | 4.80 | 0.43 | 0.52 | +| stack-edu-Shell | 2.60 | 0.01 | 0.01 | +| stack-edu-TypeScript | 2.51 | 0.18 | 0.22 | +| OpenCodeInstruct | 1.59 | — | 0.10 | +| stack-edu-Swift | 1.53 | 0.06 | 0.07 | +| stack-edu-Rust | 1.45 | 0.05 | 0.06 | +| stack-edu-Go | 1.42 | 0.03 | 0.04 | +| kaggle-notebooks | 1.42 | 0.65 | 0.78 | +| stack-edu-Ruby | 1.36 | 0.01 | 0.01 | +| OpenCodeReasoning-2-cpp-4k | 0.76 | 0.04 | 0.05 | +| OpenCodeReasoning-2-python-4k | 0.58 | 0.03 | 0.04 | +| github-code-review | 0.32 | — | 0.02 | + +**STEM & Science** + +| Dataset | Tokens (B) | Stage 1 (%) | Stage 2 (%) | +|---|---|---|---| +| Nemotron-Pretraining-Specialized-v1 (×4 shards) | 276.83 | 10.55 | 12.73 | +| Nemotron-Pretraining-SFT-v1-General | 86.93 | 3.31 | 4.00 | +| dolmino-mix-1124-pes2o | 60.19 | 0.50 | 0.50 | +| Nemotron-Pretraining-Specialized-v1.1 | 9.04 | — | 0.42 | +| OpenScienceReasoning-2-4k | 1.72 | 0.07 | 0.08 | +| MegaScience | 0.98 | 0.04 | 0.04 | + +**Multilingual** + +| Dataset | Tokens (B) | Stage 1 (%) | Stage 2 (%) | +|---|---|---|---| +| Nemotron-CC-v2-Translated-Diverse-QA | 135.80 | 1.74 | 1.74 | +| CCI4_0-Zh-High | 98.76 | 1.26 | 1.26 | + +--- + +### Checkpoint Merging + +The final model is a weighted average of 5 complementary checkpoints, each selected for a unique strength: + +| Checkpoint | Weight | Role | Key Metric | +|---|---|---|---| +| iter 169984 | 0.30 | Code anchor | MBPP **78.84** | +| iter 219136 | 0.25 | Reasoning lead | GPQA-d **44.39** | +| iter 174080 | 0.15 | Code peak | EvalPlus **68.88** | +| iter 190464 | 0.15 | Math bridge | GPQA-d **42.86** | +| iter 217088 | 0.15 | General boost | BBH **82.84** | + +--- + +## Evaluation Results + +All evaluations conducted via `lm-eval-harness` with consistent settings. + +| Benchmark | Qwen3-4B | Qwen3-8B | Qwen3.5-9B | Nemotron-12B | Gemma3-12B | Qwen3-14B | **OpenSeek-Mid-v1** | +|---|---|---|---|---|---|---|---| +| *Training tokens* | *36T* | *36T* | *36T* | *20T* | *12T* | *36T* | ***2T*** | +| MMLU (5-shot) | 72.72 | 76.57 | 78.64 | 78.07 | 73.28 | **80.57** | 79.31 | +| MMLU-Pro (5-shot CoT) | 49.31 | 52.35 | 58.48 | 57.57 | 41.16 | 56.00 | **66.57** | +| AGIEval-en (0-shot) | 45.92 | 49.09 | 45.15 | 49.20 | 44.89 | **52.83** | 52.18 | +| BBH (3-shot CoT) | 71.20 | 77.75 | 82.23 | 69.65 | 73.78 | 78.71 | **82.55** | +| HellaSwag (5-shot) | 75.36 | 79.47 | 81.04 | 83.13 | **83.45** | 82.05 | 81.81 | +| Winogrande (5-shot) | 71.90 | 77.51 | 76.80 | 79.24 | **80.35** | 79.40 | 79.24 | +| PIQA (5-shot) | 78.89 | 81.39 | 81.61 | 82.97 | 81.80 | **83.30** | 83.19 | +| OpenBookQA (5-shot) | 45.00 | 49.00 | 50.00 | 50.20 | 49.60 | **50.80** | 49.80 | +| ARC-C (0-shot) | 51.19 | 56.91 | 56.83 | 60.58 | **64.68** | 59.30 | 62.12 | +| GSM8K (4-shot CoT) | 84.31 | 86.73 | 85.60 | 81.43 | 72.02 | **90.07** | 89.16 | +| MATH (4-shot CoT) | 50.16 | 52.48 | 56.16 | 57.30 | 43.30 | 59.70 | **65.88** | +| GPQA-diamond (3-shot CoT) | 32.65 | 35.71 | 37.76 | 31.12 | 23.47 | 37.76 | **45.41** | +| MBPP (0-shot) | 73.81 | 75.66 | 77.51 | 73.81 | 73.28 | **84.92** | 76.19 | +| EvalPlus Avg (0-shot) | 63.96 | 67.95 | 59.54 | 61.20 | 53.48 | **73.41** | 66.45 | +| | | | | | | | | +| **Avg General** | 62.39 | 66.67 | 67.86 | 65.04 | 60.98 | 69.22 | **70.75** | +| **Avg All** | 61.88 | 65.61 | 66.24 | 65.39 | 61.32 | 69.20 | **69.99** | + +- **Avg General**: average of knowledge, reasoning, and commonsense benchmarks (MMLU, MMLU-Pro, AGIEval-en, BBH, HellaSwag, Winogrande, PIQA, OpenBookQA, ARC-C). +- **Avg All**: average of all benchmarks above, including math, STEM, and code (+ GSM8K, MATH, GPQA-diamond, MBPP, EvalPlus Avg). + +--- + +## Citation + +If you find this work useful, please cite: + +```bibtex +@misc{openseek-mid-v1, + title={OpenSeek-Mid-v1: Efficient Language Model Scaling via Seed Model Expansion}, + year={2026}, + note={Technical report coming soon} +} +``` + +--- + +## Acknowledgements + +This project was built using open-source data and tools, including NemotronCC-v2, Stack-Edu, Dolmino, CCI, OpenMathInstruct, OpenCodeReasoning, and FlagOS. \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..0768e48 --- /dev/null +++ b/config.json @@ -0,0 +1,32 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_qwen3.Qwen3Config", + "AutoModelForCausalLM": "modeling_qwen3.Qwen3ForCausalLM" + }, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.006, + "intermediate_size": 19456, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 56, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.3", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151851 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..159097f --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "others", "allow_remote": true} \ No newline at end of file diff --git a/configuration_qwen3.py b/configuration_qwen3.py new file mode 100644 index 0000000..7a29840 --- /dev/null +++ b/configuration_qwen3.py @@ -0,0 +1,208 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qwen3 model configuration""" + +from transformers.configuration_utils import PretrainedConfig +from transformers.modeling_rope_utils import rope_config_validation +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +class Qwen3Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Qwen3Model`]. It is used to instantiate a + Qwen3 model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of + Qwen3-8B [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 151936): + Vocabulary size of the Qwen3 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Qwen3Model`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 22016): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*, defaults to 32): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`. + head_dim (`int`, *optional*, defaults to 128): + The attention head dimension. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 32768): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (`float`, *optional*): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (`int`, *optional*): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (`float`, *optional*): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + use_sliding_window (`bool`, *optional*, defaults to `False`): + Whether to use sliding window attention. + sliding_window (`int`, *optional*, defaults to 4096): + Sliding window attention (SWA) window size. If not specified, will default to `4096`. + max_window_layers (`int`, *optional*, defaults to 28): + The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + + ```python + >>> from transformers import Qwen3Model, Qwen3Config + + >>> # Initializing a Qwen3 style configuration + >>> configuration = Qwen3Config() + + >>> # Initializing a model from the Qwen3-8B style configuration + >>> model = Qwen3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "qwen3" + keys_to_ignore_at_inference = ["past_key_values"] + + # Default tensor parallel plan for base model `Qwen3` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + def __init__( + self, + vocab_size=151936, + hidden_size=4096, + intermediate_size=22016, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=32, + head_dim=128, + hidden_act="silu", + max_position_embeddings=32768, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + use_sliding_window=False, + sliding_window=4096, + max_window_layers=28, + attention_dropout=0.0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.use_sliding_window = use_sliding_window + self.sliding_window = sliding_window # we check `use_sliding_window` in the modeling code + self.max_window_layers = max_window_layers + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + # Validate the correctness of rotary position embeddings parameters + # BC: if there is a 'type' field, move it to 'rope_type'. + if self.rope_scaling is not None and "type" in self.rope_scaling: + self.rope_scaling["rope_type"] = self.rope_scaling["type"] + rope_config_validation(self) + + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) + + +__all__ = ["Qwen3Config"] diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..139d911 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 151849, + "eos_token_id": 151850, + "transformers_version": "4.41.2" +} diff --git a/model-00001-of-00005.safetensors b/model-00001-of-00005.safetensors new file mode 100644 index 0000000..a23733b --- /dev/null +++ b/model-00001-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dabc704327965bdac65cfecb620d55881d0f6e7d0353badd87de75cc9d3115c7 +size 4992896952 diff --git a/model-00002-of-00005.safetensors b/model-00002-of-00005.safetensors new file mode 100644 index 0000000..cd335cc --- /dev/null +++ b/model-00002-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d5fb1d13829606228ecd380df10cb650363f822636013033a3517a3bbe6e01b +size 4970419768 diff --git a/model-00003-of-00005.safetensors b/model-00003-of-00005.safetensors new file mode 100644 index 0000000..cea1138 --- /dev/null +++ b/model-00003-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ea891c5f8e618c07b575a93d7ff6cef8242929536df7204091f5e90ffdab37 +size 4917989760 diff --git a/model-00004-of-00005.safetensors b/model-00004-of-00005.safetensors new file mode 100644 index 0000000..aa21236 --- /dev/null +++ b/model-00004-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2cb71ea4200692279d02e3b81e4378de68cfaed8816ebe806737638fa2b544b +size 4917989760 diff --git a/model-00005-of-00005.safetensors b/model-00005-of-00005.safetensors new file mode 100644 index 0000000..757263a --- /dev/null +++ b/model-00005-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a1fbcf10aee4f71b4bc8f715990c3a710f42ca5a09862e15a8250268f871fb9 +size 1427622384 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..73b1836 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,626 @@ +{ + "metadata": { + "total_size": 21226847232 + }, + "weight_map": { + "lm_head.weight": "model-00005-of-00005.safetensors", + "model.embed_tokens.weight": "model-00001-of-00005.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00002-of-00005.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.32.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.33.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.34.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.35.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.36.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.36.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.36.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.37.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.37.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.37.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.38.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.38.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.38.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.39.input_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", + "model.layers.39.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.39.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.40.self_attn.k_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.40.self_attn.q_norm.weight": "model-00003-of-00005.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.41.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.41.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.42.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.42.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.42.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.42.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.42.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.42.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.42.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.42.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.42.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.42.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.42.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.43.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.43.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.43.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.43.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.43.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.43.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.43.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.43.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.43.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.43.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.43.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.44.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.44.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.44.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.44.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.44.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.44.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.44.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.44.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.44.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.44.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.44.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.45.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.45.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.45.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.45.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.45.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.45.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.45.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.45.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.45.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.45.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.45.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.46.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.46.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.46.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.46.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.46.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.46.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.46.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.46.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.46.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.46.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.46.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.47.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.47.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.47.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.47.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.47.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.47.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.47.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.47.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.47.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.47.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.47.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.48.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.48.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.48.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.48.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.48.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.48.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.48.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.48.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.48.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.48.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.48.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.49.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.49.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.49.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.49.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.49.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.49.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.49.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.49.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.49.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.49.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.49.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.50.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.50.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.50.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.50.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.50.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.50.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.50.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.50.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.50.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.50.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.50.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.51.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.51.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.51.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.51.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.51.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.51.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.51.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.51.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.51.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.51.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.51.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.52.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.52.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.52.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.52.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.52.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.52.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.52.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.52.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.52.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.52.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.52.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.53.input_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.53.mlp.down_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.53.mlp.gate_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.53.mlp.up_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.53.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", + "model.layers.53.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.53.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.53.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.53.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.53.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.53.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.54.input_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.54.mlp.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.54.mlp.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.54.mlp.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.54.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.54.self_attn.k_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.54.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.54.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.54.self_attn.q_norm.weight": "model-00004-of-00005.safetensors", + "model.layers.54.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.54.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", + "model.layers.55.input_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.55.mlp.down_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.55.mlp.gate_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.55.mlp.up_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.55.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", + "model.layers.55.self_attn.k_norm.weight": "model-00005-of-00005.safetensors", + "model.layers.55.self_attn.k_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.55.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.55.self_attn.q_norm.weight": "model-00005-of-00005.safetensors", + "model.layers.55.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.55.self_attn.v_proj.weight": "model-00005-of-00005.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00005.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", + "model.norm.weight": "model-00005-of-00005.safetensors" + } +} diff --git a/modeling_qwen3.py b/modeling_qwen3.py new file mode 100644 index 0000000..bc67c2a --- /dev/null +++ b/modeling_qwen3.py @@ -0,0 +1,1204 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨��🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨��🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/qwen3/modular_qwen3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_qwen3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨��🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨��🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partial +from typing import Callable, Optional, Tuple, Union + +import torch + +from torch import nn +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache, DynamicCache, SlidingWindowCache, StaticCache +from transformers.generation import GenerationMixin +from transformers.modeling_attn_mask_utils import AttentionMaskConverter +from transformers.modeling_flash_attention_utils import FlashAttentionKwargs +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, + QuestionAnsweringModelOutput, + SequenceClassifierOutputWithPast, + TokenClassifierOutput, +) +from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from transformers.processing_utils import Unpack +from transformers.utils import ( + LossKwargs, + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + can_return_tuple, + logging, + replace_return_docstrings, +) +from transformers.utils.deprecation import deprecate_kwarg + +from .configuration_qwen3 import Qwen3Config + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "Qwen/Qwen3-8B" +_CONFIG_FOR_DOC = "Qwen3Config" + + +class Qwen3RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Qwen3RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class Qwen3MLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand( + batch, num_key_value_heads, n_rep, slen, head_dim + ) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs, +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +class Qwen3Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: Qwen3Config, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr( + config, "head_dim", config.hidden_size // config.num_attention_heads + ) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + + self.q_proj = nn.Linear( + config.hidden_size, + config.num_attention_heads * self.head_dim, + bias=config.attention_bias, + ) + self.k_proj = nn.Linear( + config.hidden_size, + config.num_key_value_heads * self.head_dim, + bias=config.attention_bias, + ) + self.v_proj = nn.Linear( + config.hidden_size, + config.num_key_value_heads * self.head_dim, + bias=config.attention_bias, + ) + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, + config.hidden_size, + bias=config.attention_bias, + ) + self.q_norm = Qwen3RMSNorm( + self.head_dim, eps=config.rms_norm_eps + ) # unlike olmo, only on the head dim! + self.k_norm = Qwen3RMSNorm( + self.head_dim, eps=config.rms_norm_eps + ) # thus post q_norm does not need reshape + self.sliding_window = config.sliding_window + if not ( + self.config.use_sliding_window + and getattr(self.config, "sliding_window", None) is not None + and self.layer_idx >= self.config.max_window_layers + ): + self.sliding_window = None + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: Tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2) + key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + if self.config._attn_implementation == "sdpa" and kwargs.get( + "output_attentions", False + ): + logger.warning_once( + "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to " + 'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + else: + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=self.sliding_window, # diff with Llama + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class Qwen3DecoderLayer(nn.Module): + def __init__(self, config: Qwen3Config, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Qwen3Attention(config=config, layer_idx=layer_idx) + self.mlp = Qwen3MLP(config) + self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if ( + config.sliding_window and config._attn_implementation != "flash_attention_2" + ): # diff with Llama is this warning + logger.warning_once( + f"Sliding Window Attention is enabled but not implemented for `{config._attn_implementation}`; " + "unexpected results may be encountered." + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[ + Tuple[torch.Tensor, torch.Tensor] + ] = None, # necessary, but kept here for BC + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + if output_attentions: + outputs += (self_attn_weights,) + + return outputs + + +class Qwen3RotaryEmbedding(nn.Module): + def __init__(self, config: Qwen3Config, device=None): + super().__init__() + # BC: "rope_type" was originally "type" + if hasattr(config, "rope_scaling") and config.rope_scaling is not None: + self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type")) + else: + self.rope_type = "default" + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + + inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = self.inv_freq + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + inv_freq_expanded = ( + self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) + ) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = ( + x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + ) + with torch.autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +QWEN3_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`Qwen3Config`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Qwen3 Model outputting raw hidden-states without any specific head on top.", + QWEN3_START_DOCSTRING, +) +class Qwen3PreTrainedModel(PreTrainedModel): + config_class = Qwen3Config + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Qwen3DecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_flex_attn = True + _supports_cache_class = True + _supports_quantized_cache = True + _supports_static_cache = True + _supports_attention_backend = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +QWEN3_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`Cache`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` + returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, + this tensor is not affected by padding. It is used to update the cache in the correct position and to infer + the complete sequence length. +""" + + +@add_start_docstrings( + "The bare Qwen3 Model outputting raw hidden-states without any specific head on top.", + QWEN3_START_DOCSTRING, +) +class Qwen3Model(Qwen3PreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen3DecoderLayer`] + + Args: + config: Qwen3Config + """ + + def __init__(self, config: Qwen3Config): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Qwen3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Qwen3RotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + @can_return_tuple + @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **flash_attn_kwargs: Unpack[FlashAttentionKwargs], + ) -> BaseModelOutputWithPast: + output_attentions = ( + output_attentions if output_attentions is not None else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if self.gradient_checkpointing and self.training and use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." + ) + use_cache = False + + # TODO (joao): remove this exception in v4.56 -- it exists for users that try to pass a legacy cache + if not isinstance(past_key_values, (type(None), Cache)): + raise ValueError("The `past_key_values` should be either a `Cache` object or `None`.") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache() + + if cache_position is None: + past_seen_tokens = ( + past_key_values.get_seq_length() if past_key_values is not None else 0 + ) + cache_position = torch.arange( + past_seen_tokens, + past_seen_tokens + inputs_embeds.shape[1], + device=inputs_embeds.device, + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = self._update_causal_mask( + attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions + ) + + hidden_states = inputs_embeds + + # create position embeddings to be shared across the decoder layers + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + partial(decoder_layer.__call__, **flash_attn_kwargs), + hidden_states, + causal_mask, + position_ids, + past_key_values, + output_attentions, + use_cache, + cache_position, + position_embeddings, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **flash_attn_kwargs, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values if use_cache else None, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + def _update_causal_mask( + self, + attention_mask: torch.Tensor, + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_key_values: Cache, + output_attentions: bool = False, + ): + if self.config._attn_implementation == "flash_attention_2": + if attention_mask is not None and past_key_values is not None: + is_padding_right = attention_mask[:, -1].sum().item() != input_tensor.size()[0] + if is_padding_right: + raise ValueError( + "You are attempting to perform batched generation with padding_side='right'" + " this may lead to unexpected behaviour for Flash Attention version of Qwen3. Make sure to " + " call `tokenizer.padding_side = 'left'` before tokenizing the input. " + ) + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + + # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in + # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail + # to infer the attention mask. + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + using_static_cache = isinstance(past_key_values, StaticCache) + using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache) + + # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward + if ( + self.config._attn_implementation == "sdpa" + and not (using_static_cache or using_sliding_window_cache) + and not output_attentions + ): + if AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask, + inputs_embeds=input_tensor, + past_key_values_length=past_seen_tokens, + sliding_window=self.config.sliding_window, + is_training=self.training, + ): + return None + + dtype, device = input_tensor.dtype, input_tensor.device + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + # SlidingWindowCache or StaticCache + if using_sliding_window_cache or using_static_cache: + target_length = past_key_values.get_max_cache_shape() + # DynamicCache or no cache + else: + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + # In case the provided `attention` mask is 2D, we generate a causal mask here (4D). + causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position( + attention_mask, + sequence_length=sequence_length, + target_length=target_length, + dtype=dtype, + device=device, + cache_position=cache_position, + batch_size=input_tensor.shape[0], + config=self.config, + past_key_values=past_key_values, + ) + + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type in ["cuda", "xpu"] + and not output_attentions + ): + # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + @staticmethod + def _prepare_4d_causal_attention_mask_with_cache_position( + attention_mask: torch.Tensor, + sequence_length: int, + target_length: int, + dtype: torch.dtype, + device: torch.device, + cache_position: torch.Tensor, + batch_size: int, + config: Qwen3Config, + past_key_values: Cache, + ): + """ + Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape + `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. + + Args: + attention_mask (`torch.Tensor`): + A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. + sequence_length (`int`): + The sequence length being processed. + target_length (`int`): + The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. + dtype (`torch.dtype`): + The dtype to use for the 4D attention mask. + device (`torch.device`): + The device to place the 4D attention mask on. + cache_position (`torch.Tensor`): + Indices depicting the position of the input sequence tokens in the sequence. + batch_size (`torch.Tensor`): + Batch size. + config (`Qwen3Config`): + The model's configuration class + past_key_values (`Cache`): + The cache class that is being used currently to generate + """ + if attention_mask is not None and attention_mask.dim() == 4: + # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing. + causal_mask = attention_mask + else: + min_dtype = torch.finfo(dtype).min + causal_mask = torch.full( + (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device + ) + diagonal_attend_mask = torch.arange( + target_length, device=device + ) > cache_position.reshape(-1, 1) + if config.sliding_window is not None: + # if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also + # the check is needed to verify is current checkpoint was trained with sliding window or not + if ( + not isinstance(past_key_values, SlidingWindowCache) + or sequence_length > target_length + ): + sliding_attend_mask = torch.arange(target_length, device=device) <= ( + cache_position.reshape(-1, 1) - config.sliding_window + ) + diagonal_attend_mask.bitwise_or_(sliding_attend_mask) + causal_mask *= diagonal_attend_mask + causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + if attention_mask.shape[-1] > target_length: + attention_mask = attention_mask[:, :target_length] + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[ + :, None, None, : + ].to(causal_mask.device) + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + return causal_mask + + +class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... + + +class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): + _tied_weights_keys = ["lm_head.weight"] + _tp_plan = {"lm_head": "colwise_rep"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + + def __init__(self, config): + super().__init__(config) + self.model = Qwen3Model(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @can_return_tuple + @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep") + @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[KwargsForCausalLM], + ) -> CausalLMOutputWithPast: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + logits_to_keep (`int` or `torch.Tensor`, *optional*): + If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all + `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that + token can save memory, which becomes pretty significant for long sequences or large vocabulary size. + If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. + This is useful when using packed tensor format (single dimension for batch and sequence length). + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, Qwen3ForCausalLM + + >>> model = Qwen3ForCausalLM.from_pretrained("Qwen/Qwen3-8B") + >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + output_attentions = ( + output_attentions if output_attentions is not None else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = ( + slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + ) + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function( + logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs + ) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + The Qwen3 Model transformer with a sequence classification head on top (linear layer). + + [`Qwen3ForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + QWEN3_START_DOCSTRING, +) +class Qwen3ForSequenceClassification(Qwen3PreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = Qwen3Model(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @can_return_tuple + @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ) -> SequenceClassifierOutputWithPast: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + + transformer_outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + hidden_states = transformer_outputs.last_hidden_state + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + last_non_pad_token = -1 + elif input_ids is not None: + # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id + non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32) + token_indices = torch.arange( + input_ids.shape[-1], device=logits.device, dtype=torch.int32 + ) + last_non_pad_token = (token_indices * non_pad_mask).argmax(-1) + else: + last_non_pad_token = -1 + logger.warning_once( + f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be " + "unexpected if using padding tokens in conjunction with `inputs_embeds.`" + ) + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token] + + loss = None + if labels is not None: + loss = self.loss_function( + logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config + ) + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + +@add_start_docstrings( + """ + The Qwen3 Model transformer with a token classification head on top (a linear layer on top of the hidden-states + output) e.g. for Named-Entity-Recognition (NER) tasks. + """, + QWEN3_START_DOCSTRING, +) +class Qwen3ForTokenClassification(Qwen3PreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = Qwen3Model(config) + if getattr(config, "classifier_dropout", None) is not None: + classifier_dropout = config.classifier_dropout + elif getattr(config, "hidden_dropout", None) is not None: + classifier_dropout = config.hidden_dropout + else: + classifier_dropout = 0.1 + self.dropout = nn.Dropout(classifier_dropout) + self.score = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @can_return_tuple + @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TokenClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ) -> TokenClassifierOutput: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + + outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + sequence_output = outputs.last_hidden_state + sequence_output = self.dropout(sequence_output) + logits = self.score(sequence_output) + + loss = None + if labels is not None: + loss = self.loss_function(logits, labels, self.config) + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ +The Qwen3 Model transformer with a span classification head on top for extractive question-answering tasks like +SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). + """, + QWEN3_START_DOCSTRING, +) +class Qwen3ForQuestionAnswering(Qwen3PreTrainedModel): + base_model_prefix = "transformer" + + def __init__(self, config): + super().__init__(config) + self.transformer = Qwen3Model(config) + self.qa_outputs = nn.Linear(config.hidden_size, 2) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.transformer.embed_tokens + + def set_input_embeddings(self, value): + self.transformer.embed_tokens = value + + @can_return_tuple + @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + start_positions: Optional[torch.LongTensor] = None, + end_positions: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + **kwargs, + ) -> QuestionAnsweringModelOutput: + r""" + start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + """ + + outputs: BaseModelOutputWithPast = self.transformer( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + + sequence_output = outputs.last_hidden_state + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1).contiguous() + end_logits = end_logits.squeeze(-1).contiguous() + + loss = None + if start_positions is not None and end_positions is not None: + loss = self.loss_function( + start_logits, end_logits, start_positions, end_positions, **kwargs + ) + + return QuestionAnsweringModelOutput( + loss=loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = [ + "Qwen3ForCausalLM", + "Qwen3ForQuestionAnswering", + "Qwen3Model", + "Qwen3PreTrainedModel", + "Qwen3ForSequenceClassification", + "Qwen3ForTokenClassification", +] diff --git a/qwen.tiktoken b/qwen.tiktoken new file mode 100644 index 0000000..b59a29f --- /dev/null +++ b/qwen.tiktoken @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186 +size 2561218 diff --git a/qwen_generation_utils.py b/qwen_generation_utils.py new file mode 100644 index 0000000..4e8e1d8 --- /dev/null +++ b/qwen_generation_utils.py @@ -0,0 +1,416 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Generation support.""" + +from typing import Tuple, List, Union, Iterable + +import numpy as np +import torch +import torch.nn.functional as F +from transformers import PreTrainedTokenizer +from transformers import logging +from transformers.generation import LogitsProcessor + +logger = logging.get_logger(__name__) + +# Types. +HistoryType = List[Tuple[str, str]] +TokensType = List[int] +BatchTokensType = List[List[int]] + + +def pad_batch(batch: BatchTokensType, pad_id: int, seq_length: int) -> BatchTokensType: + for tokens in batch: + context_length = len(tokens) + if context_length < seq_length: + tokens.extend([pad_id] * (seq_length - context_length)) + return batch + + +def get_ltor_masks_and_position_ids( + data, + eod_token, + reset_position_ids, + reset_attention_mask, + eod_mask_loss, +): + """Build masks and position id for left to right model.""" + + # Extract batch size and sequence length. + micro_batch_size, seq_length = data.size() + + # Attention mask (lower triangular). + if reset_attention_mask: + att_mask_batch = micro_batch_size + else: + att_mask_batch = 1 + attention_mask = torch.tril( + torch.ones((att_mask_batch, seq_length, seq_length), device=data.device) + ).view(att_mask_batch, 1, seq_length, seq_length) + + # Loss mask. + loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device) + if eod_mask_loss: + loss_mask[data == eod_token] = 0.0 + + # Position ids. + position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device) + position_ids = position_ids.unsqueeze(0).expand_as(data) + # We need to clone as the ids will be modifed based on batch index. + if reset_position_ids: + position_ids = position_ids.clone() + + if reset_position_ids or reset_attention_mask: + # Loop through the batches: + for b in range(micro_batch_size): + + # Find indecies where EOD token is. + eod_index = position_ids[b, data[b] == eod_token] + # Detach indecies from positions if going to modify positions. + if reset_position_ids: + eod_index = eod_index.clone() + + # Loop through EOD indecies: + prev_index = 0 + for j in range(eod_index.size()[0]): + i = eod_index[j] + # Mask attention loss. + if reset_attention_mask: + attention_mask[b, 0, (i + 1) :, : (i + 1)] = 0 + # Reset positions. + if reset_position_ids: + position_ids[b, (i + 1) :] -= i + 1 - prev_index + prev_index = i + 1 + + # Convert attention mask to binary: + attention_mask = attention_mask < 0.5 + + return attention_mask, loss_mask, position_ids + + +def get_batch(context_tokens: torch.LongTensor, eod_id: int): + """Generate batch from context tokens.""" + # Move to GPU. + tokens = context_tokens.contiguous().to(context_tokens.device) + # Get the attention mask and postition ids. + attention_mask, _, position_ids = get_ltor_masks_and_position_ids( + tokens, + eod_id, + reset_position_ids=False, + reset_attention_mask=False, + eod_mask_loss=False, + ) + return tokens, attention_mask, position_ids + + +def get_stop_words_ids(chat_format, tokenizer): + if chat_format == "raw": + stop_words_ids = [tokenizer.encode("Human:"), [tokenizer.eod_id]] + elif chat_format == "chatml": + stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]] + else: + raise NotImplementedError(f"Unknown chat format {chat_format!r}") + return stop_words_ids + + +def make_context( + tokenizer: PreTrainedTokenizer, + query: str, + history: List[Tuple[str, str]] = None, + system: str = "", + max_window_size: int = 6144, + chat_format: str = "chatml", +): + if history is None: + history = [] + + if chat_format == "chatml": + im_start, im_end = "<|im_start|>", "<|im_end|>" + im_start_tokens = [tokenizer.im_start_id] + im_end_tokens = [tokenizer.im_end_id] + nl_tokens = tokenizer.encode("\n") + + def _tokenize_str(role, content): + return f"{role}\n{content}", tokenizer.encode( + role, allowed_special=set() + ) + nl_tokens + tokenizer.encode(content, allowed_special=set()) + + system_text, system_tokens_part = _tokenize_str("system", system) + system_tokens = im_start_tokens + system_tokens_part + im_end_tokens + + raw_text = "" + context_tokens = [] + + for turn_query, turn_response in reversed(history): + query_text, query_tokens_part = _tokenize_str("user", turn_query) + query_tokens = im_start_tokens + query_tokens_part + im_end_tokens + response_text, response_tokens_part = _tokenize_str( + "assistant", turn_response + ) + response_tokens = im_start_tokens + response_tokens_part + im_end_tokens + + next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens + prev_chat = ( + f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}" + ) + + current_context_size = ( + len(system_tokens) + len(next_context_tokens) + len(context_tokens) + ) + if current_context_size < max_window_size: + context_tokens = next_context_tokens + context_tokens + raw_text = prev_chat + raw_text + else: + break + + context_tokens = system_tokens + context_tokens + raw_text = f"{im_start}{system_text}{im_end}" + raw_text + context_tokens += ( + nl_tokens + + im_start_tokens + + _tokenize_str("user", query)[1] + + im_end_tokens + + nl_tokens + + im_start_tokens + + tokenizer.encode("assistant") + + nl_tokens + ) + raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n" + + elif chat_format == "raw": + raw_text = query + context_tokens = tokenizer.encode(raw_text) + else: + raise NotImplementedError(f"Unknown chat format {chat_format!r}") + + return raw_text, context_tokens + + +def _decode_default( + tokens: List[int], + *, + stop_words: List[str], + eod_words: List[str], + tokenizer: PreTrainedTokenizer, + raw_text_len: int, + verbose: bool = False, + return_end_reason: bool = False, + errors: str='replace', +): + trim_decode_tokens = tokenizer.decode(tokens, errors=errors)[raw_text_len:] + if verbose: + print("\nRaw Generate: ", trim_decode_tokens) + + end_reason = f"Gen length {len(tokens)}" + for stop_word in stop_words: + trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip() + for eod_word in eod_words: + if eod_word in trim_decode_tokens: + end_reason = f"Gen {eod_word!r}" + trim_decode_tokens = trim_decode_tokens.split(eod_word)[0] + trim_decode_tokens = trim_decode_tokens.strip() + if verbose: + print("\nEnd Reason:", end_reason) + print("\nGenerate: ", trim_decode_tokens) + + if return_end_reason: + return trim_decode_tokens, end_reason + else: + return trim_decode_tokens + + +def _decode_chatml( + tokens: List[int], + *, + stop_words: List[str], + eod_token_ids: List[int], + tokenizer: PreTrainedTokenizer, + raw_text_len: int, + context_length: int, + verbose: bool = False, + return_end_reason: bool = False, + errors: str='replace' +): + end_reason = f"Gen length {len(tokens)}" + eod_token_idx = context_length + for eod_token_idx in range(context_length, len(tokens)): + if tokens[eod_token_idx] in eod_token_ids: + end_reason = f"Gen {tokenizer.decode([tokens[eod_token_idx]])!r}" + break + + trim_decode_tokens = tokenizer.decode(tokens[:eod_token_idx], errors=errors)[raw_text_len:] + if verbose: + print("\nRaw Generate w/o EOD:", tokenizer.decode(tokens, errors=errors)[raw_text_len:]) + print("\nRaw Generate:", trim_decode_tokens) + print("\nEnd Reason:", end_reason) + for stop_word in stop_words: + trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip() + trim_decode_tokens = trim_decode_tokens.strip() + if verbose: + print("\nGenerate:", trim_decode_tokens) + + if return_end_reason: + return trim_decode_tokens, end_reason + else: + return trim_decode_tokens + + +def decode_tokens( + tokens: Union[torch.LongTensor, TokensType], + tokenizer: PreTrainedTokenizer, + raw_text_len: int, + context_length: int, + chat_format: str, + verbose: bool = False, + return_end_reason: bool = False, + errors: str="replace", +) -> str: + if torch.is_tensor(tokens): + tokens = tokens.cpu().numpy().tolist() + + if chat_format == "chatml": + return _decode_chatml( + tokens, + stop_words=[], + eod_token_ids=[tokenizer.im_start_id, tokenizer.im_end_id], + tokenizer=tokenizer, + raw_text_len=raw_text_len, + context_length=context_length, + verbose=verbose, + return_end_reason=return_end_reason, + errors=errors, + ) + elif chat_format == "raw": + return _decode_default( + tokens, + stop_words=["<|endoftext|>"], + eod_words=["<|endoftext|>"], + tokenizer=tokenizer, + raw_text_len=raw_text_len, + verbose=verbose, + return_end_reason=return_end_reason, + errors=errors, + ) + else: + raise NotImplementedError(f"Unknown chat format {chat_format!r}") + + +class StopWordsLogitsProcessor(LogitsProcessor): + """ + :class:`transformers.LogitsProcessor` that enforces that when specified sequences appear, stop geration. + + Args: + stop_words_ids (:obj:`List[List[int]]`): + List of list of token ids of stop ids. In order to get the tokens of the words + that should not appear in the generated text, use :obj:`tokenizer(bad_word, + add_prefix_space=True).input_ids`. + eos_token_id (:obj:`int`): + The id of the `end-of-sequence` token. + """ + + def __init__(self, stop_words_ids: Iterable[Iterable[int]], eos_token_id: int): + + if not isinstance(stop_words_ids, List) or len(stop_words_ids) == 0: + raise ValueError( + f"`stop_words_ids` has to be a non-emtpy list, but is {stop_words_ids}." + ) + if any(not isinstance(bad_word_ids, list) for bad_word_ids in stop_words_ids): + raise ValueError( + f"`stop_words_ids` has to be a list of lists, but is {stop_words_ids}." + ) + if any( + any( + (not isinstance(token_id, (int, np.integer)) or token_id < 0) + for token_id in stop_word_ids + ) + for stop_word_ids in stop_words_ids + ): + raise ValueError( + f"Each list in `stop_words_ids` has to be a list of positive integers, but is {stop_words_ids}." + ) + + self.stop_words_ids = list( + filter( + lambda bad_token_seq: bad_token_seq != [eos_token_id], stop_words_ids + ) + ) + self.eos_token_id = eos_token_id + for stop_token_seq in self.stop_words_ids: + assert ( + len(stop_token_seq) > 0 + ), "Stop words token sequences {} cannot have an empty list".format( + stop_words_ids + ) + + def __call__( + self, input_ids: torch.LongTensor, scores: torch.FloatTensor + ) -> torch.FloatTensor: + stopped_samples = self._calc_stopped_samples(input_ids) + for i, should_stop in enumerate(stopped_samples): + if should_stop: + scores[i, self.eos_token_id] = float(2**15) + return scores + + def _tokens_match(self, prev_tokens: torch.LongTensor, tokens: List[int]) -> bool: + if len(tokens) == 0: + # if bad word tokens is just one token always ban it + return True + elif len(tokens) > len(prev_tokens): + # if bad word tokens are longer then prev input_ids they can't be equal + return False + elif prev_tokens[-len(tokens) :].tolist() == tokens: + # if tokens match + return True + else: + return False + + def _calc_stopped_samples(self, prev_input_ids: Iterable[int]) -> Iterable[int]: + stopped_samples = [] + for prev_input_ids_slice in prev_input_ids: + match = False + for stop_token_seq in self.stop_words_ids: + if self._tokens_match(prev_input_ids_slice, stop_token_seq): + # if tokens do not match continue + match = True + break + stopped_samples.append(match) + + return stopped_samples + + +def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float("Inf")): + """This function has been mostly taken from huggingface conversational + ai code at + https://medium.com/huggingface/how-to-build-a-state-of-the-art- + conversational-ai-with-transfer-learning-2d818ac26313""" + + if top_k > 0: + # Remove all tokens with a probability less than the + # last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits[indices_to_remove] = filter_value + + if top_p > 0.0: + # Cconvert to 1D + sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1) + cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold + sorted_indices_to_remove = cumulative_probs > top_p + # Shift the indices to the right to keep also the first token + # above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + for i in range(sorted_indices.size(0)): + indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]] + logits[i][indices_to_remove] = filter_value + + return logits + + +def switch(val1, val2, boolean): + boolean = boolean.type_as(val1) + return (1 - boolean) * val1 + boolean * val2 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..eaf7472 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,6 @@ +{ + "bos_token": "<|extra_203|>", + "eos_token": "<|extra_204|>", + "unk_token": "<|endoftext|>", + "pad_token": "<|endoftext|>" +} diff --git a/tokenization_qwen.py b/tokenization_qwen.py new file mode 100644 index 0000000..2a526d6 --- /dev/null +++ b/tokenization_qwen.py @@ -0,0 +1,276 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Tokenization classes for QWen.""" + +import base64 +import logging +import os +import unicodedata +from typing import Collection, Dict, List, Set, Tuple, Union + +import tiktoken +from transformers import PreTrainedTokenizer, AddedToken + +logger = logging.getLogger(__name__) + + +VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"} + +PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""" +ENDOFTEXT = "<|endoftext|>" +IMSTART = "<|im_start|>" +IMEND = "<|im_end|>" +# as the default behavior is changed to allow special tokens in +# regular texts, the surface forms of special tokens need to be +# as different as possible to minimize the impact +EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205))) +# changed to use actual index to avoid misconfiguration with vocabulary expansion +SPECIAL_START_ID = 151643 +SPECIAL_TOKENS = tuple( + enumerate( + ( + ( + ENDOFTEXT, + IMSTART, + IMEND, + ) + + EXTRAS + ), + start=SPECIAL_START_ID, + ) +) +SPECIAL_TOKENS_SET = set(t for i, t in SPECIAL_TOKENS) + + +def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]: + with open(tiktoken_bpe_file, "rb") as f: + contents = f.read() + return { + base64.b64decode(token): int(rank) + for token, rank in (line.split() for line in contents.splitlines() if line) + } + + +class QWenTokenizer(PreTrainedTokenizer): + """QWen tokenizer.""" + + vocab_files_names = VOCAB_FILES_NAMES + + def __init__( + self, + vocab_file, + errors="replace", + extra_vocab_file=None, + **kwargs, + ): + super().__init__(**kwargs) + + # how to handle errors in decoding UTF-8 byte sequences + # use ignore if you are in streaming inference + self.errors = errors + + self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: Dict[bytes, int] + self.special_tokens = { + token: index + for index, token in SPECIAL_TOKENS + } + + # try load extra vocab from file + if extra_vocab_file is not None: + used_ids = set(self.mergeable_ranks.values()) | set(self.special_tokens.values()) + extra_mergeable_ranks = _load_tiktoken_bpe(extra_vocab_file) + for token, index in extra_mergeable_ranks.items(): + if token in self.mergeable_ranks: + logger.info(f"extra token {token} exists, skipping") + continue + if index in used_ids: + logger.info(f'the index {index} for extra token {token} exists, skipping') + continue + self.mergeable_ranks[token] = index + # the index may be sparse after this, but don't worry tiktoken.Encoding will handle this + + enc = tiktoken.Encoding( + "Qwen", + pat_str=PAT_STR, + mergeable_ranks=self.mergeable_ranks, + special_tokens=self.special_tokens, + ) + assert ( + len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab + ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding" + + self.decoder = { + v: k for k, v in self.mergeable_ranks.items() + } # type: dict[int, bytes|str] + self.decoder.update({v: k for k, v in self.special_tokens.items()}) + + self.tokenizer = enc # type: tiktoken.Encoding + + self.eod_id = self.tokenizer.eot_token + self.im_start_id = self.special_tokens[IMSTART] + self.im_end_id = self.special_tokens[IMEND] + + def __getstate__(self): + # for pickle lovers + state = self.__dict__.copy() + del state["tokenizer"] + return state + + def __setstate__(self, state): + # tokenizer is not python native; don't pass it; rebuild it + self.__dict__.update(state) + enc = tiktoken.Encoding( + "Qwen", + pat_str=PAT_STR, + mergeable_ranks=self.mergeable_ranks, + special_tokens=self.special_tokens, + ) + self.tokenizer = enc + + def __len__(self) -> int: + return self.tokenizer.n_vocab + + def get_vocab(self) -> Dict[bytes, int]: + return self.mergeable_ranks + + def convert_tokens_to_ids( + self, tokens: Union[bytes, str, List[Union[bytes, str]]] + ) -> List[int]: + ids = [] + if isinstance(tokens, (str, bytes)): + if tokens in self.special_tokens: + return self.special_tokens[tokens] + else: + return self.mergeable_ranks.get(tokens) + for token in tokens: + if token in self.special_tokens: + ids.append(self.special_tokens[token]) + else: + ids.append(self.mergeable_ranks.get(token)) + return ids + + def _add_tokens( + self, + new_tokens: Union[List[str], List[AddedToken]], + special_tokens: bool = False, + ) -> int: + if not special_tokens and new_tokens: + raise ValueError("Adding regular tokens is not supported") + for token in new_tokens: + surface_form = token.content if isinstance(token, AddedToken) else token + if surface_form not in SPECIAL_TOKENS_SET: + raise ValueError("Adding unknown special tokens is not supported") + return 0 + + def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]: + """ + Save only the vocabulary of the tokenizer (vocabulary). + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + file_path = os.path.join(save_directory, "qwen.tiktoken") + with open(file_path, "w", encoding="utf8") as w: + for k, v in self.mergeable_ranks.items(): + line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n" + w.write(line) + return (file_path,) + + def tokenize( + self, + text: str, + allowed_special: Union[Set, str] = "all", + disallowed_special: Union[Collection, str] = (), + **kwargs, + ) -> List[Union[bytes, str]]: + """ + Converts a string in a sequence of tokens. + + Args: + text (`str`): + The sequence to be encoded. + allowed_special (`Literal["all"]` or `set`): + The surface forms of the tokens to be encoded as special tokens in regular texts. + Default to "all". + disallowed_special (`Literal["all"]` or `Collection`): + The surface forms of the tokens that should not be in regular texts and trigger errors. + Default to an empty tuple. + + kwargs (additional keyword arguments, *optional*): + Will be passed to the underlying model specific encode method. + + Returns: + `List[bytes|str]`: The list of tokens. + """ + tokens = [] + text = unicodedata.normalize("NFC", text) + + # this implementation takes a detour: text -> token id -> token surface forms + for t in self.tokenizer.encode( + text, allowed_special=allowed_special, disallowed_special=disallowed_special + ): + tokens.append(self.decoder[t]) + return tokens + + def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str: + """ + Converts a sequence of tokens in a single string. + """ + text = "" + temp = b"" + for t in tokens: + if isinstance(t, str): + if temp: + text += temp.decode("utf-8", errors=self.errors) + temp = b"" + text += t + elif isinstance(t, bytes): + temp += t + else: + raise TypeError("token should only be of type types or str") + if temp: + text += temp.decode("utf-8", errors=self.errors) + return text + + @property + def vocab_size(self): + return self.tokenizer.n_vocab + + def _convert_id_to_token(self, index: int) -> Union[bytes, str]: + """Converts an id to a token, special tokens included""" + if index in self.decoder: + return self.decoder[index] + raise ValueError("unknown ids") + + def _convert_token_to_id(self, token: Union[bytes, str]) -> int: + """Converts a token to an id using the vocab, special tokens included""" + if token in self.special_tokens: + return self.special_tokens[token] + if token in self.mergeable_ranks: + return self.mergeable_ranks[token] + raise ValueError("unknown token") + + def _tokenize(self, text: str, **kwargs): + """ + Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based + vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). + + Do NOT take care of added tokens. + """ + raise NotImplementedError + + def _decode( + self, + token_ids: Union[int, List[int]], + skip_special_tokens: bool = False, + errors: str = None, + **kwargs, + ) -> str: + if isinstance(token_ids, int): + token_ids = [token_ids] + if skip_special_tokens: + token_ids = [i for i in token_ids if i < self.eod_id] + return self.tokenizer.decode(token_ids, errors=errors or self.errors) diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..9c37cac --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,10 @@ +{ + "model_max_length": 8192, + "tokenizer_class": "QWenTokenizer", + "auto_map": { + "AutoTokenizer": [ + "tokenization_qwen.QWenTokenizer", + null + ] + } +}