From 6392bbeaf13c75cef0ac5b9279500d18e8e68ab5 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 25 May 2026 16:32:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: FlyDutch/telechat2-7b-Cot Source: Original Platform --- .gitattributes | 47 ++ README.md | 66 +++ added_tokens.json | 3 + config.json | 45 ++ configuration.json | 4 + configuration_telechat2.py | 94 ++++ generation_config.json | 15 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 310 +++++++++++ modeling_telechat2.py | 855 +++++++++++++++++++++++++++++++ sft_args.json | 261 ++++++++++ special_tokens_map.json | 42 ++ tokenization_telechat2.py | 223 ++++++++ tokenizer.model | 3 + tokenizer_config.json | 124 +++++ 18 files changed, 2104 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 added_tokens.json create mode 100644 config.json create mode 100644 configuration.json create mode 100644 configuration_telechat2.py create mode 100644 generation_config.json create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 modeling_telechat2.py create mode 100644 sft_args.json create mode 100644 special_tokens_map.json create mode 100644 tokenization_telechat2.py create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..53d7257 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,47 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..609f394 --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +--- +frameworks: +- Pytorch +license: Apache License 2.0 +tasks: +- text-generation + +#model-type: +##如 gpt、phi、llama、chatglm、baichuan 等 +#- gpt + +#domain: +##如 nlp、cv、audio、multi-modal +#- nlp + +#language: +##语言代码列表 https://help.aliyun.com/document_detail/215387.html?spm=a2c4g.11186623.0.0.9f8d7467kni6Aa +#- cn + +#metrics: +##如 CIDEr、Blue、ROUGE 等 +#- CIDEr + +#tags: +##各种自定义，包括 pretrained、fine-tuned、instruction-tuned、RL-tuned 等训练方法和其他 +#- pretrained + +#tools: +##如 vllm、fastchat、llamacpp、AdaSeq 等 +#- vllm +language: + - zh + - en +base_model: + - TeleAI/TeleChat2-7B-32K +base_model_relation: finetune +datasets: + - liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT + - AI-ModelScope/Bespoke-Stratos-17k +--- + + +该模型是基于TeleChat2 7B基座模型（非32K版）。 + +经过满血版DeepSeek R1蒸馏中文数据集和英文数据集SFT微调而来，可以让星辰大模型TeleChat2 7B拥有思考能力。 + +可点击进入右下角魔搭创空间中体验效果。我们是来自中国电信上海分公司客服中心的魔方算法团队。 + +#### 您可以通过如下git clone命令，或者ModelScope SDK来下载模型 +SDK下载 +```bash +#安装ModelScope +pip install modelscope +``` +```python +#SDK模型下载 +from modelscope import snapshot_download +model_dir = snapshot_download('FlyDutch/telechat2-7b-Cot') +``` +Git下载 +``` +#Git模型下载 +git clone https://www.modelscope.cn/FlyDutch/telechat2-7b-Cot.git +``` + +

如果您是本模型的贡献者，我们邀请您根据模型贡献文档，及时完善模型卡片内容。

\ No newline at end of file diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..3f7ebe0 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,3 @@ +{ + "": 131072 +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..4544e4b --- /dev/null +++ b/config.json @@ -0,0 +1,45 @@ +{ + "_name_or_path": "/data/Telechat/TeleChat2/TeleChat2-7B", + "apply_residual_connection_post_layernorm": false, + "architectures": [ + "TeleChat2ForCausalLM" + ], + "attention_dropout": 0.0, + "attention_softmax_in_fp32": true, + "auto_map": { + "AutoConfig": "configuration_telechat2.Telechat2Config", + "AutoModelForCausalLM": "modeling_telechat2.Telechat2ForCausalLM" + }, + "base_seqlen": 8192, + "bias_dropout_fusion": true, + "bos_token_id": 1, + "embed_layernorm": false, + "eos_token_id": 2, + "ffn_hidden_size": 12288, + "flash_attn": false, + "hidden_dropout": 0.0, + "hidden_size": 4096, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "logn": true, + "masked_softmax_fusion": true, + "model_type": "telechat", + "n_head": 32, + "n_inner": null, + "n_layer": 30, + "num_key_value_heads": 32, + "offset_alibi": 100, + "pad_token_id": 3, + "pretraining_tp": 1, + "seq_length": 32768, + "skip_bias_add": true, + "skip_bias_add_qkv": false, + "slow_but_exact": false, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "training_seqlen": 8192, + "transformers_version": "4.47.0", + "unk_token_id": 0, + "use_cache": true, + "vocab_size": 131072 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..02cdb77 --- /dev/null +++ b/configuration.json @@ -0,0 +1,4 @@ +{ + "framework": "Pytorch", + "task": "text-generation" +} \ No newline at end of file diff --git a/configuration_telechat2.py b/configuration_telechat2.py new file mode 100644 index 0000000..9d215ad --- /dev/null +++ b/configuration_telechat2.py @@ -0,0 +1,94 @@ +# coding=utf-8 +# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Telechat configuration""" + +from packaging import version +from collections import OrderedDict +from transformers.utils import is_torch_available, logging +from transformers.configuration_utils import PretrainedConfig +from typing import TYPE_CHECKING, Any, List, Mapping, Optional + +logger = logging.get_logger(__name__) + +class Telechat2Config(PretrainedConfig): + """ + Args: + vocab_size (`int`, *optional*, defaults to 160256): Vocabulary size of the Telechat model. + hidden_size (`int`, *optional*, defaults to 4096): Dimensionality of the embeddings and hidden states. + ffn_hidden_size (`int`, *optional*, defaults to 12288): Dimensionality of the feed-forward hidden states. + n_layer (`int`, *optional*, defaults to 30): Number of hidden layers in the Transformer + n_head (`int`, *optional*, defaults to 32): Number of attention heads for each attention layer. + layer_norm_epsilon (`float`, *optional*, defaults to 1e-5): The epsilon to use in the layer normalization layers. + initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + apply_residual_connection_post_layernorm (`bool`, *optional*, defaults to `False`): If enabled, use the layer norm of the hidden states as the residual in the transformer blocks + hidden_dropout (`float`, *optional*, defaults to 0.0): Dropout rate of the dropout function on the bias dropout. + attention_dropout (`float`, *optional*, defaults to 0.0): Dropout rate applied to the attention probs + use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions. + training_seqlen (`int`, *optional*, defaults to 8192): Sequence length during last finetuning. + logn (`bool`, *optional*, defaults to `True`): Whether or not to use logN during extrapolation. + embed_layernorm (`bool`, *optional*, defaults to `True`): Whether or not to use embedding layernorm. + + """ + + model_type = "telechat" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = { + "num_hidden_layers": "n_layer", + "num_attention_heads": "n_head", + } + + def __init__( + self, + vocab_size=160256, + hidden_size=4096, + n_layer=30, + n_head=32, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + use_cache=True, + bos_token_id=1, + eos_token_id=2, + apply_residual_connection_post_layernorm=False, + hidden_dropout=0.0, + attention_dropout=0.0, + ffn_hidden_size=12288, + training_seqlen = 8192, + logn = True, + embed_layernorm = False, + **kwargs, + ): + self.vocab_size = vocab_size + n_embed = kwargs.pop("n_embed", None) + self.hidden_size = hidden_size if n_embed is None else n_embed + self.n_layer = n_layer + self.n_head = n_head + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.use_cache = use_cache + self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.logn = logn + self.ffn_hidden_size = ffn_hidden_size + self.training_seqlen = training_seqlen + self.embed_layernorm = embed_layernorm + self.num_key_value_heads= kwargs.pop("num_key_value_heads", None) + + + super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) + diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c11fbdb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,15 @@ +{ + "bos_token_id": 1, + "bot_token_id": 5, + "do_sample": true, + "eos_token_id": 2, + "max_new_tokens": 2048, + "pad_token_id": 3, + "repetition_penalty": 1.02, + "start_token_id": 1, + "temperature": 0.3, + "top_k": 5, + "top_p": 0.85, + "transformers_version": "4.47.0", + "user_token_id": 4 +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..05c5b9d --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a684761935648221e1a5f08455bee3afa3f4ef9944636235b011ccbe2a19f021 +size 4999924128 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..af291e5 --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516630ac142cfd2df041a10ab723341179534af5f4ae43ba4e2b55e6cfcc4fb2 +size 4932891792 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..d05355a --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:974047f758d906834212959326b79a9c476e6f99c72e731c5408759615c0d4ba +size 4228180816 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..a634868 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08ee6c489c7721f86aafa914441b5901cb987fd4abf5ad0b74079b09eeebf4c5 +size 1073741952 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..b31d7d5 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,310 @@ +{ + "metadata": { + "total_size": 15234703360 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "transformer.h.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.0.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.0.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.0.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.0.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.1.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.1.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.10.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.10.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.10.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.10.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.10.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.11.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.11.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.12.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.12.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.13.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.13.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.14.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.14.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.15.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.15.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.16.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.16.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.17.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.17.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.18.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.18.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.19.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.19.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.2.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.2.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.2.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.2.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.20.input_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.20.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.20.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.20.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.20.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.20.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.h.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.21.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.21.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.21.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.21.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.21.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.22.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.22.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.23.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.23.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.24.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.24.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.25.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.25.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.26.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.26.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.27.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.27.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.28.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.28.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "transformer.h.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.self_attention.dense.bias": "model-00003-of-00004.safetensors", + "transformer.h.29.self_attention.dense.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.self_attention.key_value.weight": "model-00003-of-00004.safetensors", + "transformer.h.29.self_attention.query.weight": "model-00003-of-00004.safetensors", + "transformer.h.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.3.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.3.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.3.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.3.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.4.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.4.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.5.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.5.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.6.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.6.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.7.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.7.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "transformer.h.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.self_attention.dense.bias": "model-00001-of-00004.safetensors", + "transformer.h.8.self_attention.dense.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.self_attention.key_value.weight": "model-00001-of-00004.safetensors", + "transformer.h.8.self_attention.query.weight": "model-00001-of-00004.safetensors", + "transformer.h.9.input_layernorm.weight": "model-00001-of-00004.safetensors", + "transformer.h.9.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "transformer.h.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "transformer.h.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "transformer.h.9.self_attention.dense.bias": "model-00002-of-00004.safetensors", + "transformer.h.9.self_attention.dense.weight": "model-00002-of-00004.safetensors", + "transformer.h.9.self_attention.key_value.weight": "model-00002-of-00004.safetensors", + "transformer.h.9.self_attention.query.weight": "model-00002-of-00004.safetensors", + "transformer.ln_f.weight": "model-00003-of-00004.safetensors", + "transformer.word_embeddings.weight": "model-00001-of-00004.safetensors" + } +} diff --git a/modeling_telechat2.py b/modeling_telechat2.py new file mode 100644 index 0000000..585df35 --- /dev/null +++ b/modeling_telechat2.py @@ -0,0 +1,855 @@ +# coding=utf-8 +# Copyright 2022 HuggingFace Inc. team and BigScience workshop. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. + +# Copyright (c) 2021 EleutherAI +# This file is based on code by the authors denoted below and has been modified from its original version. +# +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""PyTorch TELECHAT model.""" + +import warnings +from typing import Optional, Tuple, Union, List, Dict +from threading import Thread + +import torch +import math +import copy +from torch import nn +import torch.utils.checkpoint +from torch.nn import functional as F +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + CausalLMOutputWithCrossAttentions +) +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import logging +from transformers import GenerationConfig + +from .configuration_telechat2 import Telechat2Config + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "telechat" +_CONFIG_FOR_DOC = "Telechat2Config" + +TELECHAT_PRETRAINED_MODEL_ARCHIVE_LIST = [] + +try: + from einops import rearrange +except ImportError: + rearrange = None + +use_flash_attn = True +try: + from flash_attn.flash_attn_interface import flash_attn_unpadded_func +except ImportError: + try: + from flash_attn.flash_attn_interface import flash_attn_varlen_func as flash_attn_unpadded_func + except ImportError: + flash_attn_unpadded_func = None + + +class RotaryEmbedding(torch.nn.Module): + # Extracted from: https://github.com/EleutherAI/gpt-neox + def __init__(self, dim, config, base=10000, precision=torch.half): + super().__init__() + self.config = config + self.dim = dim + self.base = base + self.inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float().half() / dim)).cuda() + self.max_seq_len_cached = None + self.cos_cached = None + self.sin_cached = None + self.precision = precision + + def get_mscale(self, scale=1): + if scale <= 1: + return 1.0 + return 0.1 * math.log(scale) + 1.0 + + def get_ntk_alpha(self, true_seq_len): + context_value = math.log(true_seq_len / self.config.base_seqlen, 2) + 1 + # ntk_alpha = 2 ** context_value - 1 + ntk_alpha = 2 ** math.ceil(context_value) - 1 + ntk_alpha = max(ntk_alpha, 1) + return ntk_alpha + + def forward(self, x, seq_dim=0, seq_len=None): + if seq_len is None: + seq_len = x.shape[seq_dim] + seq_len = max(seq_len, self.config.training_seqlen) + ntk_alpha = self.get_ntk_alpha(seq_len) + self.mscale = float(self.get_mscale(seq_len / self.config.training_seqlen)) + if True: + base = self.base * ntk_alpha ** (self.dim / (self.dim - 2)) + self.inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2, device=x.device).float() / self.dim)) + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum('i,j->ij', t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1).to(x.device) + if self.precision == torch.bfloat16: + emb = emb.float() + # [sx, 1 (b * np), hn] + self.cos_cached = self.mscale * emb.cos()[:, None, :].half() + self.sin_cached = self.mscale * emb.sin()[:, None, :].half() + if self.precision == torch.bfloat16: + self.cos_cached = self.cos_cached.bfloat16() + self.sin_cached = self.sin_cached.bfloat16() + return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...] + + +# rotary pos emb helpers: +def rotate_half(x): + x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:] + return torch.cat((-x2, x1), dim=x1.ndim - 1) # dim=-1 triggers a bug in earlier torch versions + + +def apply_rotary_pos_emb_torch(q, k, cos, sin, offset: int = 0): # jitting fails with bf16 + cos, sin = cos[offset:q.shape[0] + offset, ...], sin[offset:q.shape[0] + offset, ...] + return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin) + + +class MixedFusedRMSNorm(nn.Module): + # Extracted from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py + def __init__(self, hidden_size, eps=1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +class FlashSelfAttention(torch.nn.Module): + # Extracted from https://github.com/microsoft/Megatron-DeepSpeed/blob/main/megatron/model/transformer.py + """Implement the scaled dot product attention with softmax. + Arguments + --------- + softmax_scale: The temperature to use for the softmax attention. + (default: 1/sqrt(d_keys) where d_keys is computed at + runtime) + attention_dropout: The dropout rate to apply to the attention + (default: 0.0) + """ + + def __init__(self, causal=False, softmax_scale=None, attention_dropout=0.0, + device=None, dtype=None): + super().__init__() + assert flash_attn_unpadded_func is not None, ('Please install FlashAttention first, ' + 'e.g., with pip install flash-attn') + assert rearrange is not None, 'Please install einops first, e.g., with pip install einops' + self.causal = causal + self.softmax_scale = softmax_scale + self.dropout_p = attention_dropout + + def forward(self, q, k, v): + """Implements the multihead softmax attention. + Arguments + --------- + q, k, v: The tensor containing the query, key, and value. (B, S, H, D) + """ + assert all((i.dtype in [torch.float16, torch.bfloat16] for i in (q, k, v))) + assert all((i.is_cuda for i in (q, k, v))) + + batch_size, seqlen_q = q.shape[0], q.shape[1] + seqlen_k = k.shape[1] + + q, k, v = [rearrange(x, 'b s ... -> (b s) ...') for x in [q, k, v]] + cu_seqlens_q = torch.arange(0, (batch_size + 1) * seqlen_q, step=seqlen_q, dtype=torch.int32, + device=q.device) + self.training = False + if self.training: + # during training q,k,v always have same seqlen + assert seqlen_k == seqlen_q + + is_causal = self.causal + cu_seqlens_k = cu_seqlens_q + dropout_p = self.dropout_p + else: + # turn off FA causal mask after first inference autoregressive iteration + # only on first autoregressive step q,k,v have same seqlen + is_causal = seqlen_q == seqlen_k + cu_seqlens_k = torch.arange(0, (batch_size + 1) * seqlen_k, step=seqlen_k, dtype=torch.int32, + device=q.device) + dropout_p = 0 + + output = flash_attn_unpadded_func( + q, k, v, cu_seqlens_q, cu_seqlens_k, seqlen_q, seqlen_k, + dropout_p=dropout_p, + softmax_scale=self.softmax_scale, causal=is_causal + ) + + output = rearrange(output, '(b s) ... -> b s ...', b=batch_size) + return output + + +def _make_causal_mask( + input_ids_shape: torch.Size, device: torch.device, past_key_values_length: int +) -> torch.BoolTensor: + """ + Make causal mask used for self-attention. + """ + batch_size, target_length = input_ids_shape + mask = torch.empty((target_length, target_length + past_key_values_length), dtype=torch.bool, device=device) + # ONNX doesn't support `torch.Tensor.triu` properly, thus we use this workaround + seq_ids = torch.arange(target_length, device=device) + mask[:, past_key_values_length:] = seq_ids[:, None] < seq_ids[None, :] + + if past_key_values_length > 0: + mask[:, :past_key_values_length] = False + + expanded_mask = mask[None, None, :, :].expand(batch_size, 1, target_length, target_length + past_key_values_length) + return expanded_mask + + +def _expand_mask(mask: torch.Tensor, tgt_length: int) -> torch.BoolTensor: + """ + Expands attention_mask from `[batch_size, src_length]` to `[batch_size, 1, tgt_length, src_length]`. + """ + batch_size, src_length = mask.shape + tgt_length = tgt_length if tgt_length is not None else src_length + + expanded_mask = ~(mask[:, None, None, :].to(torch.bool)) + return expanded_mask.expand(batch_size, 1, tgt_length, src_length) + + +def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor: + """ + Dropout add function + + Args: + x (`torch.tensor`, *required*): + input tensor + residual (`torch.tensor`, *required*): + residual tensor + prob (`float`, *required*): + dropout probability + training (`bool`, *required*): + training mode + """ + out = F.dropout(x, p=prob, training=training) + out = residual + out + return out + + +def telechat_gelu_forward(x: torch.Tensor) -> torch.Tensor: + """ + Custom bias GELU function. Adapted from Megatron-DeepSpeed code. Here we use a simple implementation (inference) to + make the model jitable. + + Args: + x (`torch.tensor`, *required*): + input hidden states + """ + return x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))) + + +def telechat_gelu_back(g: torch.Tensor, x: torch.Tensor) -> torch.Tensor: + """ + gradient of tanh approximation of gelu gradient of actual gelu is: 0.5 * (1. + torch.erf(x * 0.70710678)) + + 0.3989423 * x * torch.exp(-0.5 * x * x) + + Args: + g (`torch.tensor`, *required*): + gradient output tensor + x (`torch.tensor`, *required*): + input tensor + """ + x = x[0] # x is a tuple of 1 element, needs to unpack it first + tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)) + # sqrt(2/pi) * 3 * 0.044715 -> 0.1070322243 + ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (1 + tanh_out) + return ff * g + + +class GeLUFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input: torch.Tensor) -> torch.Tensor: + ctx.save_for_backward(input) + return telechat_gelu_forward(input) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor) -> torch.Tensor: + input = ctx.saved_tensors + tmp = telechat_gelu_back(grad_output, input) + return tmp + + +class TelechatGelu(nn.Module): + """ + TelechatBiasGelu wrapper function that make use of the simple function on inference mode to make the model + torchscriptable and use the autograd function in training mode to get the accurate results of the gradients Partly + copied from Megatron-DeepSpeed code and adapted for our needs + + See here why autograd functions are not torchscriptable: https://github.com/pytorch/pytorch/issues/22329 + """ + + def __init__(self): + super().__init__() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.training: + return GeLUFunction.apply(x) + else: + return telechat_gelu_forward(x) + + +class TelechatAttention(nn.Module): + def __init__(self, config: Telechat2Config, layer_idx): + super().__init__() + self.kv_cache = None + self.layer_idx = layer_idx + + self.hidden_size = config.hidden_size + self.num_heads = config.n_head + self.head_dim = self.hidden_size // self.num_heads + self.split_size = self.hidden_size + self.hidden_dropout = config.hidden_dropout + self.config = config + + if self.head_dim * self.num_heads != self.hidden_size: + raise ValueError( + f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:" + f" {self.num_heads})." + ) + + # Layer-wise attention scaling + self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) + self.beta = 1.0 + + self.num_key_value_heads = config.num_key_value_heads if config.num_key_value_heads else self.num_heads + self.kv_projection_size = self.head_dim * self.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.query = nn.Linear(self.hidden_size, self.hidden_size, bias=False) + self.key_value = nn.Linear(self.hidden_size, self.kv_projection_size * 2, bias=False) + self.dense = nn.Linear(self.hidden_size, self.hidden_size) + self.attention_dropout = nn.Dropout(config.attention_dropout) + self.rotary_emb = RotaryEmbedding(self.head_dim, config=config) + + if config.flash_attn: + self.core_attention_flash = FlashSelfAttention( + causal=True, attention_dropout=config.attention_dropout + ) + + self.last_key_layer = None + # logn_list = [math.log(i, 4096) if i > 4096 else 1 for i in range(1, 32768)] + # self.logn_tensor = torch.tensor(logn_list)[None, :, None, None].half().cuda() + + def repeat_kv(self, hidden_states, n_rep): + slen, batch, num_key_value_heads_per_partition, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, :, None, :].expand(slen, batch, num_key_value_heads_per_partition, n_rep, + head_dim) + return hidden_states.reshape(slen, batch, num_key_value_heads_per_partition * n_rep, head_dim) + + def split_tensor_along_last_dim(self, + tensor: torch.Tensor, + num_partitions: int, + contiguous_split_chunks: bool = False, + ): + + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = tensor.size()[last_dim] // num_partitions + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # Note: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: + batch_size_and_num_heads, seq_length, _ = x.shape + batch_size = batch_size_and_num_heads // self.num_heads + x = x.view(batch_size, self.num_heads, seq_length, self.head_dim) + x = x.permute(0, 2, 1, 3) + return x.reshape(batch_size, seq_length, self.num_heads * self.head_dim) + + def forward( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + attention_mask: torch.Tensor, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + hidden_states = hidden_states.transpose(1, 0) + query_layer = self.query(hidden_states) + new_tensor_shape = query_layer.size()[:-1] + \ + (self.num_heads, + self.head_dim) + query_layer = query_layer.view(*new_tensor_shape) + + mixed_kv_layer = self.key_value(hidden_states) + new_tensor_shape = mixed_kv_layer.size()[:-1] + \ + (self.num_key_value_heads, + 2 * self.head_dim) + mixed_kv_layer = mixed_kv_layer.view(*new_tensor_shape) + (key_layer, value_layer) = self.split_tensor_along_last_dim(mixed_kv_layer, 2) + + output_size = (query_layer.size(1), + query_layer.size(2), + query_layer.size(0), + key_layer.size(0), + key_layer.size(2) + ) + + query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1) + key_layer = key_layer.view(output_size[3], output_size[0] * output_size[4], -1) + + apply_rotary_fn = apply_rotary_pos_emb_torch + + seq_len = key_layer.shape[0] + offset = 0 + + if use_cache and layer_past != None: + past_key, past_value = layer_past + offset = past_key.shape[0] + seq_len += offset + + cos, sin = self.rotary_emb(value_layer, seq_len=seq_len) + + query_layer, key_layer = apply_rotary_fn(query_layer, key_layer, cos, sin, offset=offset) + if use_cache: + if layer_past != None: + past_key, past_value = layer_past + key_layer = torch.cat((past_key, key_layer[-1, ...].unsqueeze(0)), dim=0) + value_layer = torch.cat((past_value, value_layer[-1, ...].unsqueeze(0)), dim=0) + layer_past = key_layer, value_layer + + s_value, bz, kv_head, dim = value_layer.shape + s_key = key_layer.shape[0] + s_query = query_layer.shape[0] + q_head = output_size[1] + + query_layer = query_layer.reshape((s_query, bz, q_head, dim)) + key_layer = key_layer.reshape((s_key, bz, kv_head, dim)) + + key_layer = self.repeat_kv(key_layer, self.num_key_value_groups) + value_layer = self.repeat_kv(value_layer, self.num_key_value_groups) + + if self.config.flash_attn: + q, k, v = [rearrange(x, 's b ... -> b s ...').contiguous() for x in + (query_layer, key_layer, value_layer)] + context_layer = self.core_attention_flash(q, k, v) + context_layer = rearrange(context_layer, 'b s h d -> b s (h d)').contiguous() + else: + ##[sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.reshape(s_query, bz * self.num_heads, dim) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.reshape(s_key, bz * self.num_heads, dim) + matmul_result = self.inv_norm_factor * torch.einsum('bik,bkj->bij', query_layer.transpose(0, 1), + key_layer.transpose(0, 1).transpose(1, 2)) + + attention_scores = matmul_result.view(bz, self.num_heads, s_query, s_key) + + input_dtype = attention_scores.dtype + if input_dtype == torch.float16: + attention_scores = attention_scores.to(torch.float) + attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min) + attention_probs = F.softmax(attn_weights, dim=-1).to(input_dtype) ##dtype = torch.float32 + attention_probs = self.attention_dropout(attention_probs) + attention_probs_reshaped = attention_probs.view(bz * self.num_heads, s_query, s_key) + + value_layer = value_layer.reshape(s_key, bz * self.num_heads, dim) + context_layer = torch.bmm(attention_probs_reshaped, value_layer.transpose(0, 1)) + context_layer = self._merge_heads(context_layer) + output_tensor = self.dense(context_layer) + + output_tensor = dropout_add(output_tensor, residual, self.hidden_dropout, self.training) + present = None + outputs = (output_tensor, present) + if output_attentions: + outputs += (attention_probs,) + + return output_tensor, layer_past + + +class TelechatMLP(nn.Module): + def __init__(self, config: Telechat2Config): + super().__init__() + hidden_size = config.hidden_size + self.gate_proj = nn.Linear(hidden_size, config.ffn_hidden_size, bias=False) + self.up_proj = nn.Linear(hidden_size, config.ffn_hidden_size, bias=False) + self.down_proj = nn.Linear(config.ffn_hidden_size, hidden_size, bias=True) + self.hidden_dropout = config.hidden_dropout + + def forward(self, hidden_states: torch.Tensor, residual: torch.Tensor) -> torch.Tensor: + intermediate_output = self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states)) + output = dropout_add(intermediate_output, residual, self.hidden_dropout, self.training) + return output + + +class TelechatBlock(nn.Module): + def __init__(self, config: Telechat2Config, layer_idx): + super().__init__() + hidden_size = config.hidden_size + + self.input_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon) + self.num_heads = config.n_head + self.layer_idx = layer_idx + self.self_attention = TelechatAttention(config, layer_idx) + self.post_attention_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon) + + self.mlp = TelechatMLP(config) + + self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm + self.hidden_dropout = config.hidden_dropout + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + layernorm_output = self.input_layernorm(hidden_states) + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + attn_outputs = self.self_attention( + layernorm_output, + residual, + layer_past=layer_past, + attention_mask=attention_mask, + use_cache=use_cache, + output_attentions=output_attentions, + ) + + attention_output = attn_outputs[0] + outputs = attn_outputs[1:] + layernorm_output = self.post_attention_layernorm(attention_output) + + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = attention_output + output = self.mlp(layernorm_output, residual) + + if use_cache: + outputs = (output,) + outputs + else: + outputs = (output,) + outputs[1:] + + return outputs + + +class TelechatPreTrainedModel(PreTrainedModel): + config_class = Telechat2Config + base_model_prefix = "transformer" + supports_gradient_checkpointing = True + _no_split_modules = ["TelechatBlock"] + _skip_keys_device_placement = "past_key_values" + + def __init__(self, *inputs, **kwargs): + super().__init__(*inputs, **kwargs) + + def _init_weights(self, module: nn.Module): + """Initialize the weights.""" + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + elif isinstance(module, LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module: nn.Module, value: bool = False): + if isinstance(module, TelechatModel): + module.gradient_checkpointing = value + + +class TelechatModel(TelechatPreTrainedModel): + def __init__(self, config: Telechat2Config): + super().__init__(config) + + self.embed_dim = config.hidden_size + self.num_heads = config.n_head + self.config = config + self.word_embeddings = nn.Embedding(config.vocab_size, self.embed_dim) + if self.config.embed_layernorm: + self.word_embeddings_layernorm = MixedFusedRMSNorm(self.embed_dim, eps=config.layer_norm_epsilon) + + self.h = nn.ModuleList([TelechatBlock(config, _) for _ in range(config.num_hidden_layers)]) + self.ln_f = MixedFusedRMSNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.gradient_checkpointing = False + self.post_init() + + def get_input_embeddings(self): + return self.word_embeddings + + def _prepare_attn_mask( + self, attention_mask: torch.Tensor, input_shape: Tuple[int, int], past_key_values_length: int + ) -> torch.BoolTensor: + combined_attention_mask = None + device = attention_mask.device + _, src_length = input_shape + + if src_length > 1: + combined_attention_mask = _make_causal_mask( + input_shape, device=device, past_key_values_length=past_key_values_length + ) + expanded_attn_mask = _expand_mask(attention_mask, tgt_length=src_length) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask | combined_attention_mask + ) + + return combined_attention_mask + + def set_input_embeddings(self, new_embeddings: torch.Tensor): + self.word_embeddings = new_embeddings + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, + ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]: + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + + if past_key_values is None: + past_key_values = tuple([None] * len(self.h)) + # input_ids = torch.load("Megatron-LM-0624-3B/tensors/input_ids.pt").to(input_ids.device) + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + hidden_states = inputs_embeds + # print(f"[INFO_Telechat]: inputs_embeds={inputs_embeds}") + if self.config.embed_layernorm: + hidden_states = self.word_embeddings_layernorm(inputs_embeds) + + presents = () if use_cache else None + all_self_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + + if self.gradient_checkpointing and self.training: + if use_cache: + use_cache = False + + seq_length_with_past = seq_length + past_key_values_length = 0 + if past_key_values[0] is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + if attention_mask is None: + attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device) + else: + attention_mask = attention_mask.to(hidden_states.device) + causal_mask = self._prepare_attn_mask( + attention_mask, + input_shape=(batch_size, seq_length), + past_key_values_length=past_key_values_length, + ) + + # print(f"[INFO_Telechat]: word_embeddings_layernorm={hidden_states}") + for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + # None for past_key_value + return module(*inputs, use_cache=use_cache, output_attentions=output_attentions) + + return custom_forward + + outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + causal_mask, + layer_past, + ) + else: + outputs = block( + hidden_states, + layer_past=layer_past, + attention_mask=causal_mask, + use_cache=use_cache, + output_attentions=output_attentions, + ) + + # print(f"[INFO_Telechat]: outputs{i}={outputs}") + hidden_states = outputs[0] + if use_cache is True: + presents = presents + (outputs[1],) + + if output_attentions: + all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],) + hidden_states = self.ln_f(hidden_states) + # print(f"[INFO_Telechat]: hidden_states={hidden_states}") + # ref = torch.load("Megatron-LM-0624-3B/tensors/final_layernorm.pt") + # print(hidden_states.squeeze()[2048:]) + # print(ref.squeeze()) + # print(torch.max(hidden_states.squeeze()[2048:] - ref.squeeze().to(hidden_states.device))) + # exit() + # print(ref.shape,hidden_states.shape) + # print(hidden_states) + # exit() + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + if not return_dict: + return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=presents, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + +class Telechat2ForCausalLM(TelechatPreTrainedModel): + # _tied_weights_keys = ["lm_head.weight"] + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config: Telechat2Config): + super().__init__(config) + self.transformer = TelechatModel(config) + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.post_init() + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings: torch.Tensor): + self.lm_head = new_embeddings + + def prepare_inputs_for_generation( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs, + ) -> dict: + if past_key_values: + input_ids = input_ids[:, -1].unsqueeze(-1) + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, + ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.transformer( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + lm_logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + labels = labels.to(lm_logits.device) + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + batch_size, seq_length, vocab_size = shift_logits.shape + loss_fct = CrossEntropyLoss() + loss = loss_fct( + shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length) + ) + + if not return_dict: + output = (lm_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=loss, + logits=lm_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/sft_args.json b/sft_args.json new file mode 100644 index 0000000..0f8e764 --- /dev/null +++ b/sft_args.json @@ -0,0 +1,261 @@ +{ + "model_type": "telechat2-115b", + "model_id_or_path": "/data/Telechat/TeleChat2/TeleChat2-7B", + "model_revision": "master", + "full_determinism": false, + "sft_type": "full", + "freeze_parameters": [], + "freeze_vit": false, + "freeze_parameters_ratio": 0.0, + "additional_trainable_parameters": [], + "tuner_backend": "peft", + "template_type": "telechat2", + "output_dir": "/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240", + "add_output_dir_suffix": true, + "ddp_backend": "nccl", + "ddp_find_unused_parameters": null, + "ddp_broadcast_buffers": null, + "ddp_timeout": 1800, + "seed": 42, + "resume_from_checkpoint": null, + "resume_only_model": false, + "ignore_data_skip": false, + "dtype": "fp16", + "packing": false, + "train_backend": "transformers", + "tp": 1, + "pp": 1, + "min_lr": null, + "sequence_parallel": false, + "model_kwargs": {}, + "loss_name": null, + "dataset": [ + "/data/Telechat/shuyun_deepseek_Bespoke-Stratos-10k.jsonl#2000", + "/data/Telechat/Chinese-DeepSeek-R1-Distill-data-110k/distill_r1_10k_clean_no_system.jsonl#5500" + ], + "val_dataset": [], + "dataset_seed": 42, + "dataset_test_ratio": 0.01, + "use_loss_scale": false, + "loss_scale_config_path": "/root/anaconda3/envs/telechat2/lib/python3.9/site-packages/swift/llm/agent/default_loss_scale_config.json", + "system": null, + "tools_prompt": "react_en", + "max_length": 5120, + "truncation_strategy": "delete", + "check_dataset_strategy": "none", + "streaming": false, + "streaming_val_size": 0, + "streaming_buffer_size": 16384, + "model_name": [ + "小服", + "Xiao Fu" + ], + "model_author": [ + "客服中心", + "kefu_center" + ], + "quant_method": null, + "quantization_bit": 0, + "hqq_axis": 0, + "hqq_dynamic_config_path": null, + "bnb_4bit_comp_dtype": "fp16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "rescale_image": -1, + "target_modules": [ + "dense", + "key_value", + "up_proj", + "down_proj", + "query", + "gate_proj" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias_trainable": "none", + "lora_dtype": null, + "lora_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "init_lora_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "rope_scaling": null, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "ia3_feedforward_modules": [], + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "neftune_noise_alpha": null, + "neftune_backend": "transformers", + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "use_liger": false, + "gradient_checkpointing": true, + "vit_use_gc": true, + "deepspeed": null, + "batch_size": 1, + "eval_batch_size": 1, + "auto_find_batch_size": false, + "num_train_epochs": 7, + "max_steps": -1, + "optim": "adamw_torch", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "learning_rate": 0.0001, + "weight_decay": 0.1, + "gradient_accumulation_steps": 8, + "max_grad_norm": 1, + "predict_with_generate": false, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": {}, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "eval_steps": 50, + "save_steps": 20, + "save_only_model": false, + "save_total_limit": 15, + "logging_steps": 5, + "acc_steps": 1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_drop_last": false, + "push_to_hub": false, + "hub_model_id": null, + "hub_token": null, + "hub_private_repo": false, + "hub_strategy": "every_save", + "test_oom_error": false, + "disable_tqdm": false, + "lazy_tokenize": false, + "preprocess_num_proc": 1, + "use_flash_attn": null, + "ignore_args_error": false, + "check_model_is_latest": true, + "logging_dir": "/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240/runs", + "report_to": [ + "tensorboard" + ], + "acc_strategy": "token", + "save_on_each_node": false, + "evaluation_strategy": "steps", + "save_strategy": "steps", + "save_safetensors": true, + "gpu_memory_fraction": null, + "include_num_input_tokens_seen": false, + "local_repo_path": null, + "custom_register_path": null, + "custom_dataset_info": null, + "device_map_config": null, + "device_max_memory": [], + "max_new_tokens": 2048, + "do_sample": null, + "temperature": null, + "top_k": null, + "top_p": null, + "repetition_penalty": null, + "num_beams": 1, + "fsdp": "", + "fsdp_config": null, + "sequence_parallel_size": 1, + "model_layer_cls_name": null, + "metric_warmup_step": 0, + "fsdp_num": 1, + "per_device_train_batch_size": null, + "per_device_eval_batch_size": null, + "eval_strategy": null, + "self_cognition_sample": 0, + "train_dataset_mix_ratio": 0.0, + "train_dataset_mix_ds": [ + "ms-bench" + ], + "train_dataset_sample": -1, + "val_dataset_sample": null, + "safe_serialization": null, + "only_save_model": null, + "neftune_alpha": null, + "deepspeed_config_path": null, + "model_cache_dir": null, + "lora_dropout_p": null, + "lora_target_modules": [ + "dense", + "key_value", + "up_proj", + "down_proj", + "query", + "gate_proj" + ], + "lora_target_regex": null, + "lora_modules_to_save": [], + "boft_target_modules": [], + "boft_modules_to_save": [], + "vera_target_modules": [], + "vera_modules_to_save": [], + "ia3_target_modules": [], + "ia3_modules_to_save": [], + "custom_train_dataset_path": [], + "custom_val_dataset_path": [], + "device_map_config_path": null, + "push_hub_strategy": null, + "use_self_cognition": false, + "is_multimodal": false, + "is_vision": false, + "lora_use_embedding": false, + "lora_use_all": true, + "lora_m2s_use_embedding": false, + "lora_m2s_use_ln": false, + "torch_dtype": "torch.float16", + "fp16": true, + "bf16": false, + "rank": 0, + "local_rank": 0, + "world_size": 2, + "local_world_size": 2, + "bnb_4bit_compute_dtype": "torch.float16", + "load_in_4bit": false, + "load_in_8bit": false, + "train_sampler_random": true, + "train_type": "sft", + "training_args": "Seq2SeqTrainingArguments(output_dir='/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=0.0001, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1, num_train_epochs=7, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs={}, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=20, save_total_limit=15, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=False, fp16=True, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend='nccl', tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=50, dataloader_num_workers=1, dataloader_prefetch_factor=None, past_index=-1, run_name='/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed=None, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=False, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', evaluation_strategy=None, push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, split_batches=None, include_tokens_per_second=False, include_num_input_tokens_seen=False, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, eval_use_gather_object=False, average_tokens_across_devices=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=GenerationConfig {\n \"bos_token_id\": 1,\n \"bot_token_id\": 5,\n \"do_sample\": true,\n \"eos_token_id\": 2,\n \"max_new_tokens\": 2048,\n \"pad_token_id\": 3,\n \"repetition_penalty\": 1.02,\n \"start_token_id\": 1,\n \"temperature\": 0.3,\n \"top_k\": 5,\n \"top_p\": 0.85,\n \"user_token_id\": 4\n}\n, acc_strategy='token', loss_name=None, additional_saved_files=[], train_sampler_random=True, metric_warmup_step=0, train_dataset_sample=-1)" +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..935b138 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,42 @@ +{ + "additional_special_tokens": [ + "<_start>", + "<_end>", + "<_pad>", + "<_user>", + "<_bot>", + "<_system>", + "", + "", + "", + "" + ], + "bos_token": { + "content": "<_start>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<_end>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<_pad>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenization_telechat2.py b/tokenization_telechat2.py new file mode 100644 index 0000000..a061444 --- /dev/null +++ b/tokenization_telechat2.py @@ -0,0 +1,223 @@ +import os +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple +import sentencepiece as spm +from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"} + +# TODO: when we get download url from huggingface, refresh the map +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": {}, + "tokenizer_file": {}, +} + + +class Telechat2Tokenizer(PreTrainedTokenizer): + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + unk_token="", + bos_token="<_start>", + eos_token="<_end>", + pad_token="<_pad>", + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + clean_up_tokenization_spaces=False, + **kwargs, + ): + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token + eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token + unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token + pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(vocab_file) + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + add_bos_token=add_bos_token, + add_eos_token=add_eos_token, + sp_model_kwargs=self.sp_model_kwargs, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + self.vocab_file = vocab_file + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(self.vocab_file) + + @property + def vocab_size(self): + """Returns vocab size""" + return self.sp_model.get_piece_size() + + def get_vocab(self): + """Returns vocab as a dict""" + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + @property + def vocab(self): + return self.get_vocab() + + def _tokenize(self, text): + """Returns a tokenized string.""" + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + token = self.sp_model.IdToPiece(index) + return token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + current_sub_tokens = [] + out_string = "" + # prev_is_special = False + for i, token in enumerate(tokens): + # make sure that special tokens are not decoded using sentencepiece model + if token in self.all_special_tokens: + # if not prev_is_special and i != 0: + # out_string += " " + out_string += self.sp_model.decode(current_sub_tokens) + token + # prev_is_special = True + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + # prev_is_special = False + out_string += self.sp_model.decode(current_sub_tokens) + return out_string + + def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]: + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + out_vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, "wb") as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = bos_token_id + token_ids_0 + eos_token_id + + if token_ids_1 is not None: + output = output + bos_token_id + token_ids_1 + eos_token_id + + return output + + def get_special_tokens_mask( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + ) + + bos_token_id = [1] if self.add_bos_token else [] + eos_token_id = [1] if self.add_eos_token else [] + + if token_ids_1 is None: + return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id + return ( + bos_token_id + + ([0] * len(token_ids_0)) + + eos_token_id + + bos_token_id + + ([0] * len(token_ids_1)) + + eos_token_id + ) + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT + sequence pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + if token_ids_1 is None, only returns the first portion of the mask (0s). + + Args: + token_ids_0 (`List[int]`): + List of ids. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). + """ + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = [0] * len(bos_token_id + token_ids_0 + eos_token_id) + + if token_ids_1 is not None: + output += [1] * len(bos_token_id + token_ids_1 + eos_token_id) + + return output diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..86d585d --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7a5b465bbc9465b214e0962076c1170783a8ee88fb01454b0c33609bd3cf954 +size 2197499 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5772493 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,124 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "added_tokens_decoder": { + "1": { + "content": "<_start>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "<_end>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "<_pad>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "<_user>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "<_bot>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "<_system>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "131072": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<_start>", + "<_end>", + "<_pad>", + "<_user>", + "<_bot>", + "<_system>", + "", + "", + "", + "" + ], + "auto_map": { + "AutoTokenizer": [ + "tokenization_telechat2.Telechat2Tokenizer", + null + ] + }, + "bos_token": "<_start>", + "chat_template": "{%- if tools %}\n {%- if messages[0]['role'] == 'system' %}\n {{-'<_system>'+messages[0]['content'] }}\n {%- else %}\n {{- '<_system>'+'你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。' }}\n {%- endif %}\n {{- '\\n\\n# 可用工具\\n你可以调用标签中包含的一个或多个工具来辅助你回答问题,以下是可用工具详情：\\n\\n' }}\n {%- for tool in tools %}\n {{- tool | tojson }}\n {{-'\\n'}}\n {%- endfor %}\n {{- '\\n\\n# 调用方法\\n你需要遵循工具的要求，使用json格式返回工具名称及参数，并用包含。下方是一个调用模板：\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n\\n\\n' }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<_system>' + messages[0]['content'] + '\\n' }}\n {%- else %}\n {{- '<_system>'+'你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == 'user') %}\n {{- '<_user>' + message.content }}\n {%- elif message.role == 'bot' %}\n {{- '<_bot>' }}\n {%- if message.content %}\n {{- message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.index0 == 0 %}\n {{-''}}\n {%- else %}\n {{-'\\n'}}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}{{ tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n' }}\n {%- endfor %}\n {{- '<_end>\\n' }}\n {%- elif message.role == 'tool' %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != 'tool') %}\n {{- '<_user>'+'\\n' }}\n {%- else %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- message.content }}\n {{- '\\n' }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<_bot>' }}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<_end>", + "extra_special_tokens": {}, + "model_max_length": 100000000, + "pad_token": "<_pad>", + "sp_model_kwargs": {}, + "split_special_tokens": false, + "tokenizer_class": "Telechat2Tokenizer", + "unk_token": "", + "use_fast": false +}