FlyDutch/telechat2-7b-Cot
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: FlyDutch/telechat2-7b-Cot
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-05-25 16:32:12 +08:00
commit 6392bbeaf1
18 changed files with 2104 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
.gitattributes vendored Normal file
View File

@@ -0,0 +1,47 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bin.* filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

66
README.md Normal file
View File

@@ -0,0 +1,66 @@
---
frameworks:
- Pytorch
license: Apache License 2.0
tasks:
- text-generation
#model-type:
##如 gpt、phi、llama、chatglm、baichuan 等
#- gpt
#domain:
##如 nlp、cv、audio、multi-modal
#- nlp
#language:
##语言代码列表 https://help.aliyun.com/document_detail/215387.html?spm=a2c4g.11186623.0.0.9f8d7467kni6Aa
#- cn
#metrics:
##如 CIDEr、Blue、ROUGE 等
#- CIDEr
#tags:
##各种自定义，包括 pretrained、fine-tuned、instruction-tuned、RL-tuned 等训练方法和其他
#- pretrained
#tools:
##如 vllm、fastchat、llamacpp、AdaSeq 等
#- vllm
language:
- zh
- en
base_model:
- TeleAI/TeleChat2-7B-32K
base_model_relation: finetune
datasets:
- liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT
- AI-ModelScope/Bespoke-Stratos-17k
---
该模型是基于TeleChat2 7B基座模型（非32K版）。
经过满血版DeepSeek R1蒸馏中文数据集和英文数据集SFT微调而来，可以让星辰大模型TeleChat2 7B拥有思考能力。
可点击进入右下角魔搭创空间中体验效果。我们是来自中国电信上海分公司客服中心的魔方算法团队。
#### 您可以通过如下git clone命令，或者ModelScope SDK来下载模型
SDK下载
```bash
#安装ModelScope
pip install modelscope
```
```python
#SDK模型下载
from modelscope import snapshot_download
model_dir = snapshot_download('FlyDutch/telechat2-7b-Cot')
```
Git下载
```
#Git模型下载
git clone https://www.modelscope.cn/FlyDutch/telechat2-7b-Cot.git
```
<p style="color: lightgrey;">如果您是本模型的贡献者，我们邀请您根据<a href="https://modelscope.cn/docs/ModelScope%E6%A8%A1%E5%9E%8B%E6%8E%A5%E5%85%A5%E6%B5%81%E7%A8%8B%E6%A6%82%E8%A7%88" style="color: lightgrey; text-decoration: underline;">模型贡献文档</a>，及时完善模型卡片内容。</p>

3
added_tokens.json Normal file
View File

@@ -0,0 +1,3 @@
{
"<unk>": 131072
}

45
config.json Normal file
View File

@@ -0,0 +1,45 @@
{
"_name_or_path": "/data/Telechat/TeleChat2/TeleChat2-7B",
"apply_residual_connection_post_layernorm": false,
"architectures": [
"TeleChat2ForCausalLM"
],
"attention_dropout": 0.0,
"attention_softmax_in_fp32": true,
"auto_map": {
"AutoConfig": "configuration_telechat2.Telechat2Config",
"AutoModelForCausalLM": "modeling_telechat2.Telechat2ForCausalLM"
},
"base_seqlen": 8192,
"bias_dropout_fusion": true,
"bos_token_id": 1,
"embed_layernorm": false,
"eos_token_id": 2,
"ffn_hidden_size": 12288,
"flash_attn": false,
"hidden_dropout": 0.0,
"hidden_size": 4096,
"initializer_range": 0.02,
"layer_norm_epsilon": 1e-05,
"logn": true,
"masked_softmax_fusion": true,
"model_type": "telechat",
"n_head": 32,
"n_inner": null,
"n_layer": 30,
"num_key_value_heads": 32,
"offset_alibi": 100,
"pad_token_id": 3,
"pretraining_tp": 1,
"seq_length": 32768,
"skip_bias_add": true,
"skip_bias_add_qkv": false,
"slow_but_exact": false,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"training_seqlen": 8192,
"transformers_version": "4.47.0",
"unk_token_id": 0,
"use_cache": true,
"vocab_size": 131072
}

4
configuration.json Normal file
View File

@@ -0,0 +1,4 @@
{
"framework": "Pytorch",
"task": "text-generation"
}

94
configuration_telechat2.py Normal file
View File

@@ -0,0 +1,94 @@
# coding=utf-8
# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Telechat configuration"""
from packaging import version
from collections import OrderedDict
from transformers.utils import is_torch_available, logging
from transformers.configuration_utils import PretrainedConfig
from typing import TYPE_CHECKING, Any, List, Mapping, Optional
logger = logging.get_logger(__name__)
class Telechat2Config(PretrainedConfig):
"""
Args:
vocab_size (`int`, *optional*, defaults to 160256): Vocabulary size of the Telechat model.
hidden_size (`int`, *optional*, defaults to 4096): Dimensionality of the embeddings and hidden states.
ffn_hidden_size (`int`, *optional*, defaults to 12288): Dimensionality of the feed-forward hidden states.
n_layer (`int`, *optional*, defaults to 30): Number of hidden layers in the Transformer
n_head (`int`, *optional*, defaults to 32): Number of attention heads for each attention layer.
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5): The epsilon to use in the layer normalization layers.
initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
apply_residual_connection_post_layernorm (`bool`, *optional*, defaults to `False`): If enabled, use the layer norm of the hidden states as the residual in the transformer blocks
hidden_dropout (`float`, *optional*, defaults to 0.0): Dropout rate of the dropout function on the bias dropout.
attention_dropout (`float`, *optional*, defaults to 0.0): Dropout rate applied to the attention probs
use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions.
training_seqlen (`int`, *optional*, defaults to 8192): Sequence length during last finetuning.
logn (`bool`, *optional*, defaults to `True`): Whether or not to use logN during extrapolation.
embed_layernorm (`bool`, *optional*, defaults to `True`): Whether or not to use embedding layernorm.
"""
model_type = "telechat"
keys_to_ignore_at_inference = ["past_key_values"]
attribute_map = {
"num_hidden_layers": "n_layer",
"num_attention_heads": "n_head",
}
def __init__(
self,
vocab_size=160256,
hidden_size=4096,
n_layer=30,
n_head=32,
layer_norm_epsilon=1e-5,
initializer_range=0.02,
use_cache=True,
bos_token_id=1,
eos_token_id=2,
apply_residual_connection_post_layernorm=False,
hidden_dropout=0.0,
attention_dropout=0.0,
ffn_hidden_size=12288,
training_seqlen = 8192,
logn = True,
embed_layernorm = False,
**kwargs,
):
self.vocab_size = vocab_size
n_embed = kwargs.pop("n_embed", None)
self.hidden_size = hidden_size if n_embed is None else n_embed
self.n_layer = n_layer
self.n_head = n_head
self.layer_norm_epsilon = layer_norm_epsilon
self.initializer_range = initializer_range
self.use_cache = use_cache
self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm
self.hidden_dropout = hidden_dropout
self.attention_dropout = attention_dropout
self.bos_token_id = bos_token_id
self.eos_token_id = eos_token_id
self.logn = logn
self.ffn_hidden_size = ffn_hidden_size
self.training_seqlen = training_seqlen
self.embed_layernorm = embed_layernorm
self.num_key_value_heads= kwargs.pop("num_key_value_heads", None)
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)

15
generation_config.json Normal file
View File

@@ -0,0 +1,15 @@
{
"bos_token_id": 1,
"bot_token_id": 5,
"do_sample": true,
"eos_token_id": 2,
"max_new_tokens": 2048,
"pad_token_id": 3,
"repetition_penalty": 1.02,
"start_token_id": 1,
"temperature": 0.3,
"top_k": 5,
"top_p": 0.85,
"transformers_version": "4.47.0",
"user_token_id": 4
}

3
model-00001-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a684761935648221e1a5f08455bee3afa3f4ef9944636235b011ccbe2a19f021
size 4999924128

3
model-00002-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:516630ac142cfd2df041a10ab723341179534af5f4ae43ba4e2b55e6cfcc4fb2
size 4932891792

3
model-00003-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:974047f758d906834212959326b79a9c476e6f99c72e731c5408759615c0d4ba
size 4228180816

3
model-00004-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:08ee6c489c7721f86aafa914441b5901cb987fd4abf5ad0b74079b09eeebf4c5
size 1073741952

310
model.safetensors.index.json Normal file
View File

@@ -0,0 +1,310 @@
{
"metadata": {
"total_size": 15234703360
},
"weight_map": {
"lm_head.weight": "model-00004-of-00004.safetensors",
"transformer.h.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.0.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.0.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.0.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.0.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.1.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.1.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.10.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.10.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.10.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.10.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.10.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.11.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.11.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.12.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.12.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.13.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.13.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.14.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.14.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.15.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.15.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.16.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.16.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.17.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.17.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.18.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.18.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.19.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.19.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.2.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.2.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.2.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.2.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.20.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.20.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.20.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.20.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.20.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.h.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.21.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.21.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.21.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.21.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.21.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.22.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.22.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.23.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.23.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.24.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.24.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.25.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.25.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.26.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.26.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.27.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.27.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.28.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.28.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
"transformer.h.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.self_attention.dense.bias": "model-00003-of-00004.safetensors",
"transformer.h.29.self_attention.dense.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.self_attention.key_value.weight": "model-00003-of-00004.safetensors",
"transformer.h.29.self_attention.query.weight": "model-00003-of-00004.safetensors",
"transformer.h.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.3.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.3.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.3.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.3.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.4.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.4.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.5.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.5.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.6.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.6.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.7.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.7.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"transformer.h.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.self_attention.dense.bias": "model-00001-of-00004.safetensors",
"transformer.h.8.self_attention.dense.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.self_attention.key_value.weight": "model-00001-of-00004.safetensors",
"transformer.h.8.self_attention.query.weight": "model-00001-of-00004.safetensors",
"transformer.h.9.input_layernorm.weight": "model-00001-of-00004.safetensors",
"transformer.h.9.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
"transformer.h.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"transformer.h.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"transformer.h.9.self_attention.dense.bias": "model-00002-of-00004.safetensors",
"transformer.h.9.self_attention.dense.weight": "model-00002-of-00004.safetensors",
"transformer.h.9.self_attention.key_value.weight": "model-00002-of-00004.safetensors",
"transformer.h.9.self_attention.query.weight": "model-00002-of-00004.safetensors",
"transformer.ln_f.weight": "model-00003-of-00004.safetensors",
"transformer.word_embeddings.weight": "model-00001-of-00004.safetensors"
}
}

855
modeling_telechat2.py Normal file
View File

@@ -0,0 +1,855 @@
# coding=utf-8
# Copyright 2022 HuggingFace Inc. team and BigScience workshop.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2021 EleutherAI
# This file is based on code by the authors denoted below and has been modified from its original version.
#
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch TELECHAT model."""
import warnings
from typing import Optional, Tuple, Union, List, Dict
from threading import Thread
import torch
import math
import copy
from torch import nn
import torch.utils.checkpoint
from torch.nn import functional as F
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss
from transformers.modeling_outputs import (
BaseModelOutputWithPastAndCrossAttentions,
CausalLMOutputWithCrossAttentions
)
from transformers.modeling_utils import PreTrainedModel
from transformers.utils import logging
from transformers import GenerationConfig
from .configuration_telechat2 import Telechat2Config
logger = logging.get_logger(__name__)
_CHECKPOINT_FOR_DOC = "telechat"
_CONFIG_FOR_DOC = "Telechat2Config"
TELECHAT_PRETRAINED_MODEL_ARCHIVE_LIST = []
try:
from einops import rearrange
except ImportError:
rearrange = None
use_flash_attn = True
try:
from flash_attn.flash_attn_interface import flash_attn_unpadded_func
except ImportError:
try:
from flash_attn.flash_attn_interface import flash_attn_varlen_func as flash_attn_unpadded_func
except ImportError:
flash_attn_unpadded_func = None
class RotaryEmbedding(torch.nn.Module):
# Extracted from: https://github.com/EleutherAI/gpt-neox
def __init__(self, dim, config, base=10000, precision=torch.half):
super().__init__()
self.config = config
self.dim = dim
self.base = base
self.inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float().half() / dim)).cuda()
self.max_seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
self.precision = precision
def get_mscale(self, scale=1):
if scale <= 1:
return 1.0
return 0.1 * math.log(scale) + 1.0
def get_ntk_alpha(self, true_seq_len):
context_value = math.log(true_seq_len / self.config.base_seqlen, 2) + 1
# ntk_alpha = 2 ** context_value - 1
ntk_alpha = 2 ** math.ceil(context_value) - 1
ntk_alpha = max(ntk_alpha, 1)
return ntk_alpha
def forward(self, x, seq_dim=0, seq_len=None):
if seq_len is None:
seq_len = x.shape[seq_dim]
seq_len = max(seq_len, self.config.training_seqlen)
ntk_alpha = self.get_ntk_alpha(seq_len)
self.mscale = float(self.get_mscale(seq_len / self.config.training_seqlen))
if True:
base = self.base * ntk_alpha ** (self.dim / (self.dim - 2))
self.inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2, device=x.device).float() / self.dim))
self.max_seq_len_cached = seq_len
t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype)
freqs = torch.einsum('i,j->ij', t, self.inv_freq)
# Different from paper, but it uses a different permutation in order to obtain the same calculation
emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
if self.precision == torch.bfloat16:
emb = emb.float()
# [sx, 1 (b * np), hn]
self.cos_cached = self.mscale * emb.cos()[:, None, :].half()
self.sin_cached = self.mscale * emb.sin()[:, None, :].half()
if self.precision == torch.bfloat16:
self.cos_cached = self.cos_cached.bfloat16()
self.sin_cached = self.sin_cached.bfloat16()
return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
# rotary pos emb helpers:
def rotate_half(x):
x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
return torch.cat((-x2, x1), dim=x1.ndim - 1) # dim=-1 triggers a bug in earlier torch versions
def apply_rotary_pos_emb_torch(q, k, cos, sin, offset: int = 0): # jitting fails with bf16
cos, sin = cos[offset:q.shape[0] + offset, ...], sin[offset:q.shape[0] + offset, ...]
return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
class MixedFusedRMSNorm(nn.Module):
# Extracted from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py
def __init__(self, hidden_size, eps=1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states):
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
class FlashSelfAttention(torch.nn.Module):
# Extracted from https://github.com/microsoft/Megatron-DeepSpeed/blob/main/megatron/model/transformer.py
"""Implement the scaled dot product attention with softmax.
Arguments
---------
softmax_scale: The temperature to use for the softmax attention.
(default: 1/sqrt(d_keys) where d_keys is computed at
runtime)
attention_dropout: The dropout rate to apply to the attention
(default: 0.0)
"""
def __init__(self, causal=False, softmax_scale=None, attention_dropout=0.0,
device=None, dtype=None):
super().__init__()
assert flash_attn_unpadded_func is not None, ('Please install FlashAttention first, '
'e.g., with pip install flash-attn')
assert rearrange is not None, 'Please install einops first, e.g., with pip install einops'
self.causal = causal
self.softmax_scale = softmax_scale
self.dropout_p = attention_dropout
def forward(self, q, k, v):
"""Implements the multihead softmax attention.
Arguments
---------
q, k, v: The tensor containing the query, key, and value. (B, S, H, D)
"""
assert all((i.dtype in [torch.float16, torch.bfloat16] for i in (q, k, v)))
assert all((i.is_cuda for i in (q, k, v)))
batch_size, seqlen_q = q.shape[0], q.shape[1]
seqlen_k = k.shape[1]
q, k, v = [rearrange(x, 'b s ... -> (b s) ...') for x in [q, k, v]]
cu_seqlens_q = torch.arange(0, (batch_size + 1) * seqlen_q, step=seqlen_q, dtype=torch.int32,
device=q.device)
self.training = False
if self.training:
# during training q,k,v always have same seqlen
assert seqlen_k == seqlen_q
is_causal = self.causal
cu_seqlens_k = cu_seqlens_q
dropout_p = self.dropout_p
else:
# turn off FA causal mask after first inference autoregressive iteration
# only on first autoregressive step q,k,v have same seqlen
is_causal = seqlen_q == seqlen_k
cu_seqlens_k = torch.arange(0, (batch_size + 1) * seqlen_k, step=seqlen_k, dtype=torch.int32,
device=q.device)
dropout_p = 0
output = flash_attn_unpadded_func(
q, k, v, cu_seqlens_q, cu_seqlens_k, seqlen_q, seqlen_k,
dropout_p=dropout_p,
softmax_scale=self.softmax_scale, causal=is_causal
)
output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
return output
def _make_causal_mask(
input_ids_shape: torch.Size, device: torch.device, past_key_values_length: int
) -> torch.BoolTensor:
"""
Make causal mask used for self-attention.
"""
batch_size, target_length = input_ids_shape
mask = torch.empty((target_length, target_length + past_key_values_length), dtype=torch.bool, device=device)
# ONNX doesn't support `torch.Tensor.triu` properly, thus we use this workaround
seq_ids = torch.arange(target_length, device=device)
mask[:, past_key_values_length:] = seq_ids[:, None] < seq_ids[None, :]
if past_key_values_length > 0:
mask[:, :past_key_values_length] = False
expanded_mask = mask[None, None, :, :].expand(batch_size, 1, target_length, target_length + past_key_values_length)
return expanded_mask
def _expand_mask(mask: torch.Tensor, tgt_length: int) -> torch.BoolTensor:
"""
Expands attention_mask from `[batch_size, src_length]` to `[batch_size, 1, tgt_length, src_length]`.
"""
batch_size, src_length = mask.shape
tgt_length = tgt_length if tgt_length is not None else src_length
expanded_mask = ~(mask[:, None, None, :].to(torch.bool))
return expanded_mask.expand(batch_size, 1, tgt_length, src_length)
def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor:
"""
Dropout add function
Args:
x (`torch.tensor`, *required*):
input tensor
residual (`torch.tensor`, *required*):
residual tensor
prob (`float`, *required*):
dropout probability
training (`bool`, *required*):
training mode
"""
out = F.dropout(x, p=prob, training=training)
out = residual + out
return out
def telechat_gelu_forward(x: torch.Tensor) -> torch.Tensor:
"""
Custom bias GELU function. Adapted from Megatron-DeepSpeed code. Here we use a simple implementation (inference) to
make the model jitable.
Args:
x (`torch.tensor`, *required*):
input hidden states
"""
return x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)))
def telechat_gelu_back(g: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
"""
gradient of tanh approximation of gelu gradient of actual gelu is: 0.5 * (1. + torch.erf(x * 0.70710678)) +
0.3989423 * x * torch.exp(-0.5 * x * x)
Args:
g (`torch.tensor`, *required*):
gradient output tensor
x (`torch.tensor`, *required*):
input tensor
"""
x = x[0] # x is a tuple of 1 element, needs to unpack it first
tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))
# sqrt(2/pi) * 3 * 0.044715 -> 0.1070322243
ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (1 + tanh_out)
return ff * g
class GeLUFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, input: torch.Tensor) -> torch.Tensor:
ctx.save_for_backward(input)
return telechat_gelu_forward(input)
@staticmethod
def backward(ctx, grad_output: torch.Tensor) -> torch.Tensor:
input = ctx.saved_tensors
tmp = telechat_gelu_back(grad_output, input)
return tmp
class TelechatGelu(nn.Module):
"""
TelechatBiasGelu wrapper function that make use of the simple function on inference mode to make the model
torchscriptable and use the autograd function in training mode to get the accurate results of the gradients Partly
copied from Megatron-DeepSpeed code and adapted for our needs
See here why autograd functions are not torchscriptable: https://github.com/pytorch/pytorch/issues/22329
"""
def __init__(self):
super().__init__()
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.training:
return GeLUFunction.apply(x)
else:
return telechat_gelu_forward(x)
class TelechatAttention(nn.Module):
def __init__(self, config: Telechat2Config, layer_idx):
super().__init__()
self.kv_cache = None
self.layer_idx = layer_idx
self.hidden_size = config.hidden_size
self.num_heads = config.n_head
self.head_dim = self.hidden_size // self.num_heads
self.split_size = self.hidden_size
self.hidden_dropout = config.hidden_dropout
self.config = config
if self.head_dim * self.num_heads != self.hidden_size:
raise ValueError(
f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:"
f" {self.num_heads})."
)
# Layer-wise attention scaling
self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim)
self.beta = 1.0
self.num_key_value_heads = config.num_key_value_heads if config.num_key_value_heads else self.num_heads
self.kv_projection_size = self.head_dim * self.num_key_value_heads
self.num_key_value_groups = self.num_heads // self.num_key_value_heads
self.query = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
self.key_value = nn.Linear(self.hidden_size, self.kv_projection_size * 2, bias=False)
self.dense = nn.Linear(self.hidden_size, self.hidden_size)
self.attention_dropout = nn.Dropout(config.attention_dropout)
self.rotary_emb = RotaryEmbedding(self.head_dim, config=config)
if config.flash_attn:
self.core_attention_flash = FlashSelfAttention(
causal=True, attention_dropout=config.attention_dropout
)
self.last_key_layer = None
# logn_list = [math.log(i, 4096) if i > 4096 else 1 for i in range(1, 32768)]
# self.logn_tensor = torch.tensor(logn_list)[None, :, None, None].half().cuda()
def repeat_kv(self, hidden_states, n_rep):
slen, batch, num_key_value_heads_per_partition, head_dim = hidden_states.shape
if n_rep == 1:
return hidden_states
hidden_states = hidden_states[:, :, :, None, :].expand(slen, batch, num_key_value_heads_per_partition, n_rep,
head_dim)
return hidden_states.reshape(slen, batch, num_key_value_heads_per_partition * n_rep, head_dim)
def split_tensor_along_last_dim(self,
tensor: torch.Tensor,
num_partitions: int,
contiguous_split_chunks: bool = False,
):
# Get the size and dimension.
last_dim = tensor.dim() - 1
last_dim_size = tensor.size()[last_dim] // num_partitions
# Split.
tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
# Note: torch.split does not create contiguous tensors by default.
if contiguous_split_chunks:
return tuple(chunk.contiguous() for chunk in tensor_list)
return tensor_list
def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
batch_size_and_num_heads, seq_length, _ = x.shape
batch_size = batch_size_and_num_heads // self.num_heads
x = x.view(batch_size, self.num_heads, seq_length, self.head_dim)
x = x.permute(0, 2, 1, 3)
return x.reshape(batch_size, seq_length, self.num_heads * self.head_dim)
def forward(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
attention_mask: torch.Tensor,
layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
use_cache: bool = False,
output_attentions: bool = False,
):
hidden_states = hidden_states.transpose(1, 0)
query_layer = self.query(hidden_states)
new_tensor_shape = query_layer.size()[:-1] + \
(self.num_heads,
self.head_dim)
query_layer = query_layer.view(*new_tensor_shape)
mixed_kv_layer = self.key_value(hidden_states)
new_tensor_shape = mixed_kv_layer.size()[:-1] + \
(self.num_key_value_heads,
2 * self.head_dim)
mixed_kv_layer = mixed_kv_layer.view(*new_tensor_shape)
(key_layer, value_layer) = self.split_tensor_along_last_dim(mixed_kv_layer, 2)
output_size = (query_layer.size(1),
query_layer.size(2),
query_layer.size(0),
key_layer.size(0),
key_layer.size(2)
)
query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1)
key_layer = key_layer.view(output_size[3], output_size[0] * output_size[4], -1)
apply_rotary_fn = apply_rotary_pos_emb_torch
seq_len = key_layer.shape[0]
offset = 0
if use_cache and layer_past != None:
past_key, past_value = layer_past
offset = past_key.shape[0]
seq_len += offset
cos, sin = self.rotary_emb(value_layer, seq_len=seq_len)
query_layer, key_layer = apply_rotary_fn(query_layer, key_layer, cos, sin, offset=offset)
if use_cache:
if layer_past != None:
past_key, past_value = layer_past
key_layer = torch.cat((past_key, key_layer[-1, ...].unsqueeze(0)), dim=0)
value_layer = torch.cat((past_value, value_layer[-1, ...].unsqueeze(0)), dim=0)
layer_past = key_layer, value_layer
s_value, bz, kv_head, dim = value_layer.shape
s_key = key_layer.shape[0]
s_query = query_layer.shape[0]
q_head = output_size[1]
query_layer = query_layer.reshape((s_query, bz, q_head, dim))
key_layer = key_layer.reshape((s_key, bz, kv_head, dim))
key_layer = self.repeat_kv(key_layer, self.num_key_value_groups)
value_layer = self.repeat_kv(value_layer, self.num_key_value_groups)
if self.config.flash_attn:
q, k, v = [rearrange(x, 's b ... -> b s ...').contiguous() for x in
(query_layer, key_layer, value_layer)]
context_layer = self.core_attention_flash(q, k, v)
context_layer = rearrange(context_layer, 'b s h d -> b s (h d)').contiguous()
else:
##[sq, b, np, hn] -> [sq, b * np, hn]
query_layer = query_layer.reshape(s_query, bz * self.num_heads, dim)
# [sk, b, np, hn] -> [sk, b * np, hn]
key_layer = key_layer.reshape(s_key, bz * self.num_heads, dim)
matmul_result = self.inv_norm_factor * torch.einsum('bik,bkj->bij', query_layer.transpose(0, 1),
key_layer.transpose(0, 1).transpose(1, 2))
attention_scores = matmul_result.view(bz, self.num_heads, s_query, s_key)
input_dtype = attention_scores.dtype
if input_dtype == torch.float16:
attention_scores = attention_scores.to(torch.float)
attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min)
attention_probs = F.softmax(attn_weights, dim=-1).to(input_dtype) ##dtype = torch.float32
attention_probs = self.attention_dropout(attention_probs)
attention_probs_reshaped = attention_probs.view(bz * self.num_heads, s_query, s_key)
value_layer = value_layer.reshape(s_key, bz * self.num_heads, dim)
context_layer = torch.bmm(attention_probs_reshaped, value_layer.transpose(0, 1))
context_layer = self._merge_heads(context_layer)
output_tensor = self.dense(context_layer)
output_tensor = dropout_add(output_tensor, residual, self.hidden_dropout, self.training)
present = None
outputs = (output_tensor, present)
if output_attentions:
outputs += (attention_probs,)
return output_tensor, layer_past
class TelechatMLP(nn.Module):
def __init__(self, config: Telechat2Config):
super().__init__()
hidden_size = config.hidden_size
self.gate_proj = nn.Linear(hidden_size, config.ffn_hidden_size, bias=False)
self.up_proj = nn.Linear(hidden_size, config.ffn_hidden_size, bias=False)
self.down_proj = nn.Linear(config.ffn_hidden_size, hidden_size, bias=True)
self.hidden_dropout = config.hidden_dropout
def forward(self, hidden_states: torch.Tensor, residual: torch.Tensor) -> torch.Tensor:
intermediate_output = self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
output = dropout_add(intermediate_output, residual, self.hidden_dropout, self.training)
return output
class TelechatBlock(nn.Module):
def __init__(self, config: Telechat2Config, layer_idx):
super().__init__()
hidden_size = config.hidden_size
self.input_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon)
self.num_heads = config.n_head
self.layer_idx = layer_idx
self.self_attention = TelechatAttention(config, layer_idx)
self.post_attention_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon)
self.mlp = TelechatMLP(config)
self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm
self.hidden_dropout = config.hidden_dropout
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
use_cache: bool = False,
output_attentions: bool = False,
):
layernorm_output = self.input_layernorm(hidden_states)
if self.apply_residual_connection_post_layernorm:
residual = layernorm_output
else:
residual = hidden_states
attn_outputs = self.self_attention(
layernorm_output,
residual,
layer_past=layer_past,
attention_mask=attention_mask,
use_cache=use_cache,
output_attentions=output_attentions,
)
attention_output = attn_outputs[0]
outputs = attn_outputs[1:]
layernorm_output = self.post_attention_layernorm(attention_output)
if self.apply_residual_connection_post_layernorm:
residual = layernorm_output
else:
residual = attention_output
output = self.mlp(layernorm_output, residual)
if use_cache:
outputs = (output,) + outputs
else:
outputs = (output,) + outputs[1:]
return outputs
class TelechatPreTrainedModel(PreTrainedModel):
config_class = Telechat2Config
base_model_prefix = "transformer"
supports_gradient_checkpointing = True
_no_split_modules = ["TelechatBlock"]
_skip_keys_device_placement = "past_key_values"
def __init__(self, *inputs, **kwargs):
super().__init__(*inputs, **kwargs)
def _init_weights(self, module: nn.Module):
"""Initialize the weights."""
if isinstance(module, nn.Linear):
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
if module.bias is not None:
module.bias.data.zero_()
elif isinstance(module, nn.Embedding):
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
if module.padding_idx is not None:
module.weight.data[module.padding_idx].zero_()
elif isinstance(module, LayerNorm):
module.bias.data.zero_()
module.weight.data.fill_(1.0)
def _set_gradient_checkpointing(self, module: nn.Module, value: bool = False):
if isinstance(module, TelechatModel):
module.gradient_checkpointing = value
class TelechatModel(TelechatPreTrainedModel):
def __init__(self, config: Telechat2Config):
super().__init__(config)
self.embed_dim = config.hidden_size
self.num_heads = config.n_head
self.config = config
self.word_embeddings = nn.Embedding(config.vocab_size, self.embed_dim)
if self.config.embed_layernorm:
self.word_embeddings_layernorm = MixedFusedRMSNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.h = nn.ModuleList([TelechatBlock(config, _) for _ in range(config.num_hidden_layers)])
self.ln_f = MixedFusedRMSNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.gradient_checkpointing = False
self.post_init()
def get_input_embeddings(self):
return self.word_embeddings
def _prepare_attn_mask(
self, attention_mask: torch.Tensor, input_shape: Tuple[int, int], past_key_values_length: int
) -> torch.BoolTensor:
combined_attention_mask = None
device = attention_mask.device
_, src_length = input_shape
if src_length > 1:
combined_attention_mask = _make_causal_mask(
input_shape, device=device, past_key_values_length=past_key_values_length
)
expanded_attn_mask = _expand_mask(attention_mask, tgt_length=src_length)
combined_attention_mask = (
expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask | combined_attention_mask
)
return combined_attention_mask
def set_input_embeddings(self, new_embeddings: torch.Tensor):
self.word_embeddings = new_embeddings
def forward(
self,
input_ids: Optional[torch.LongTensor] = None,
past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
attention_mask: Optional[torch.Tensor] = None,
inputs_embeds: Optional[torch.LongTensor] = None,
use_cache: Optional[bool] = None,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
**deprecated_arguments,
) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]:
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
)
use_cache = use_cache if use_cache is not None else self.config.use_cache
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if input_ids is not None:
batch_size, seq_length = input_ids.shape
elif inputs_embeds is not None:
batch_size, seq_length, _ = inputs_embeds.shape
if past_key_values is None:
past_key_values = tuple([None] * len(self.h))
# input_ids = torch.load("Megatron-LM-0624-3B/tensors/input_ids.pt").to(input_ids.device)
if inputs_embeds is None:
inputs_embeds = self.word_embeddings(input_ids)
hidden_states = inputs_embeds
# print(f"[INFO_Telechat]: inputs_embeds={inputs_embeds}")
if self.config.embed_layernorm:
hidden_states = self.word_embeddings_layernorm(inputs_embeds)
presents = () if use_cache else None
all_self_attentions = () if output_attentions else None
all_hidden_states = () if output_hidden_states else None
if self.gradient_checkpointing and self.training:
if use_cache:
use_cache = False
seq_length_with_past = seq_length
past_key_values_length = 0
if past_key_values[0] is not None:
past_key_values_length = past_key_values[0][0].shape[2]
seq_length_with_past = seq_length_with_past + past_key_values_length
if attention_mask is None:
attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device)
else:
attention_mask = attention_mask.to(hidden_states.device)
causal_mask = self._prepare_attn_mask(
attention_mask,
input_shape=(batch_size, seq_length),
past_key_values_length=past_key_values_length,
)
# print(f"[INFO_Telechat]: word_embeddings_layernorm={hidden_states}")
for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
if output_hidden_states:
all_hidden_states = all_hidden_states + (hidden_states,)
if self.gradient_checkpointing and self.training:
def create_custom_forward(module):
def custom_forward(*inputs):
# None for past_key_value
return module(*inputs, use_cache=use_cache, output_attentions=output_attentions)
return custom_forward
outputs = torch.utils.checkpoint.checkpoint(
create_custom_forward(block),
hidden_states,
causal_mask,
layer_past,
)
else:
outputs = block(
hidden_states,
layer_past=layer_past,
attention_mask=causal_mask,
use_cache=use_cache,
output_attentions=output_attentions,
)
# print(f"[INFO_Telechat]: outputs{i}={outputs}")
hidden_states = outputs[0]
if use_cache is True:
presents = presents + (outputs[1],)
if output_attentions:
all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
hidden_states = self.ln_f(hidden_states)
# print(f"[INFO_Telechat]: hidden_states={hidden_states}")
# ref = torch.load("Megatron-LM-0624-3B/tensors/final_layernorm.pt")
# print(hidden_states.squeeze()[2048:])
# print(ref.squeeze())
# print(torch.max(hidden_states.squeeze()[2048:] - ref.squeeze().to(hidden_states.device)))
# exit()
# print(ref.shape,hidden_states.shape)
# print(hidden_states)
# exit()
if output_hidden_states:
all_hidden_states = all_hidden_states + (hidden_states,)
if not return_dict:
return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
return BaseModelOutputWithPastAndCrossAttentions(
last_hidden_state=hidden_states,
past_key_values=presents,
hidden_states=all_hidden_states,
attentions=all_self_attentions,
)
class Telechat2ForCausalLM(TelechatPreTrainedModel):
# _tied_weights_keys = ["lm_head.weight"]
_keys_to_ignore_on_load_missing = [r"lm_head.weight"]
def __init__(self, config: Telechat2Config):
super().__init__(config)
self.transformer = TelechatModel(config)
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
self.post_init()
def get_output_embeddings(self):
return self.lm_head
def set_output_embeddings(self, new_embeddings: torch.Tensor):
self.lm_head = new_embeddings
def prepare_inputs_for_generation(
self,
input_ids: torch.LongTensor,
past_key_values: Optional[torch.Tensor] = None,
attention_mask: Optional[torch.Tensor] = None,
inputs_embeds: Optional[torch.Tensor] = None,
**kwargs,
) -> dict:
if past_key_values:
input_ids = input_ids[:, -1].unsqueeze(-1)
if inputs_embeds is not None and past_key_values is None:
model_inputs = {"inputs_embeds": inputs_embeds}
else:
model_inputs = {"input_ids": input_ids}
model_inputs.update(
{
"past_key_values": past_key_values,
"use_cache": kwargs.get("use_cache"),
"attention_mask": attention_mask,
}
)
return model_inputs
def forward(
self,
input_ids: Optional[torch.LongTensor] = None,
past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
attention_mask: Optional[torch.Tensor] = None,
inputs_embeds: Optional[torch.Tensor] = None,
labels: Optional[torch.Tensor] = None,
use_cache: Optional[bool] = None,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
**deprecated_arguments,
) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]:
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
transformer_outputs = self.transformer(
input_ids,
past_key_values=past_key_values,
attention_mask=attention_mask,
inputs_embeds=inputs_embeds,
use_cache=use_cache,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
return_dict=return_dict,
)
hidden_states = transformer_outputs[0]
lm_logits = self.lm_head(hidden_states)
loss = None
if labels is not None:
labels = labels.to(lm_logits.device)
shift_logits = lm_logits[..., :-1, :].contiguous()
shift_labels = labels[..., 1:].contiguous()
batch_size, seq_length, vocab_size = shift_logits.shape
loss_fct = CrossEntropyLoss()
loss = loss_fct(
shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length)
)
if not return_dict:
output = (lm_logits,) + transformer_outputs[1:]
return ((loss,) + output) if loss is not None else output
return CausalLMOutputWithCrossAttentions(
loss=loss,
logits=lm_logits,
past_key_values=transformer_outputs.past_key_values,
hidden_states=transformer_outputs.hidden_states,
attentions=transformer_outputs.attentions,
)

261
sft_args.json Normal file
View File

@@ -0,0 +1,261 @@
{
"model_type": "telechat2-115b",
"model_id_or_path": "/data/Telechat/TeleChat2/TeleChat2-7B",
"model_revision": "master",
"full_determinism": false,
"sft_type": "full",
"freeze_parameters": [],
"freeze_vit": false,
"freeze_parameters_ratio": 0.0,
"additional_trainable_parameters": [],
"tuner_backend": "peft",
"template_type": "telechat2",
"output_dir": "/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240",
"add_output_dir_suffix": true,
"ddp_backend": "nccl",
"ddp_find_unused_parameters": null,
"ddp_broadcast_buffers": null,
"ddp_timeout": 1800,
"seed": 42,
"resume_from_checkpoint": null,
"resume_only_model": false,
"ignore_data_skip": false,
"dtype": "fp16",
"packing": false,
"train_backend": "transformers",
"tp": 1,
"pp": 1,
"min_lr": null,
"sequence_parallel": false,
"model_kwargs": {},
"loss_name": null,
"dataset": [
"/data/Telechat/shuyun_deepseek_Bespoke-Stratos-10k.jsonl#2000",
"/data/Telechat/Chinese-DeepSeek-R1-Distill-data-110k/distill_r1_10k_clean_no_system.jsonl#5500"
],
"val_dataset": [],
"dataset_seed": 42,
"dataset_test_ratio": 0.01,
"use_loss_scale": false,
"loss_scale_config_path": "/root/anaconda3/envs/telechat2/lib/python3.9/site-packages/swift/llm/agent/default_loss_scale_config.json",
"system": null,
"tools_prompt": "react_en",
"max_length": 5120,
"truncation_strategy": "delete",
"check_dataset_strategy": "none",
"streaming": false,
"streaming_val_size": 0,
"streaming_buffer_size": 16384,
"model_name": [
"小服",
"Xiao Fu"
],
"model_author": [
"客服中心",
"kefu_center"
],
"quant_method": null,
"quantization_bit": 0,
"hqq_axis": 0,
"hqq_dynamic_config_path": null,
"bnb_4bit_comp_dtype": "fp16",
"bnb_4bit_quant_type": "nf4",
"bnb_4bit_use_double_quant": true,
"bnb_4bit_quant_storage": null,
"rescale_image": -1,
"target_modules": [
"dense",
"key_value",
"up_proj",
"down_proj",
"query",
"gate_proj"
],
"target_regex": null,
"modules_to_save": [],
"lora_rank": 8,
"lora_alpha": 32,
"lora_dropout": 0.05,
"lora_bias_trainable": "none",
"lora_dtype": null,
"lora_lr_ratio": null,
"use_rslora": false,
"use_dora": false,
"init_lora_weights": true,
"fourier_n_frequency": 2000,
"fourier_scaling": 300.0,
"rope_scaling": null,
"boft_block_size": 4,
"boft_block_num": 0,
"boft_n_butterfly_factor": 1,
"boft_dropout": 0.0,
"vera_rank": 256,
"vera_projection_prng_key": 0,
"vera_dropout": 0.0,
"vera_d_initial": 0.1,
"adapter_act": "gelu",
"adapter_length": 128,
"use_galore": false,
"galore_target_modules": null,
"galore_rank": 128,
"galore_update_proj_gap": 50,
"galore_scale": 1.0,
"galore_proj_type": "std",
"galore_optim_per_parameter": false,
"galore_with_embedding": false,
"galore_quantization": false,
"galore_proj_quant": false,
"galore_proj_bits": 4,
"galore_proj_group_size": 256,
"galore_cos_threshold": 0.4,
"galore_gamma_proj": 2,
"galore_queue_size": 5,
"adalora_target_r": 8,
"adalora_init_r": 12,
"adalora_tinit": 0,
"adalora_tfinal": 0,
"adalora_deltaT": 1,
"adalora_beta1": 0.85,
"adalora_beta2": 0.85,
"adalora_orth_reg_weight": 0.5,
"ia3_feedforward_modules": [],
"llamapro_num_new_blocks": 4,
"llamapro_num_groups": null,
"neftune_noise_alpha": null,
"neftune_backend": "transformers",
"lisa_activated_layers": 0,
"lisa_step_interval": 20,
"reft_layer_key": null,
"reft_layers": null,
"reft_rank": 4,
"reft_intervention_type": "LoreftIntervention",
"reft_args": null,
"use_liger": false,
"gradient_checkpointing": true,
"vit_use_gc": true,
"deepspeed": null,
"batch_size": 1,
"eval_batch_size": 1,
"auto_find_batch_size": false,
"num_train_epochs": 7,
"max_steps": -1,
"optim": "adamw_torch",
"adam_beta1": 0.9,
"adam_beta2": 0.95,
"adam_epsilon": 1e-08,
"learning_rate": 0.0001,
"weight_decay": 0.1,
"gradient_accumulation_steps": 8,
"max_grad_norm": 1,
"predict_with_generate": false,
"lr_scheduler_type": "cosine",
"lr_scheduler_kwargs": {},
"warmup_ratio": 0.05,
"warmup_steps": 0,
"eval_steps": 50,
"save_steps": 20,
"save_only_model": false,
"save_total_limit": 15,
"logging_steps": 5,
"acc_steps": 1,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_drop_last": false,
"push_to_hub": false,
"hub_model_id": null,
"hub_token": null,
"hub_private_repo": false,
"hub_strategy": "every_save",
"test_oom_error": false,
"disable_tqdm": false,
"lazy_tokenize": false,
"preprocess_num_proc": 1,
"use_flash_attn": null,
"ignore_args_error": false,
"check_model_is_latest": true,
"logging_dir": "/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240/runs",
"report_to": [
"tensorboard"
],
"acc_strategy": "token",
"save_on_each_node": false,
"evaluation_strategy": "steps",
"save_strategy": "steps",
"save_safetensors": true,
"gpu_memory_fraction": null,
"include_num_input_tokens_seen": false,
"local_repo_path": null,
"custom_register_path": null,
"custom_dataset_info": null,
"device_map_config": null,
"device_max_memory": [],
"max_new_tokens": 2048,
"do_sample": null,
"temperature": null,
"top_k": null,
"top_p": null,
"repetition_penalty": null,
"num_beams": 1,
"fsdp": "",
"fsdp_config": null,
"sequence_parallel_size": 1,
"model_layer_cls_name": null,
"metric_warmup_step": 0,
"fsdp_num": 1,
"per_device_train_batch_size": null,
"per_device_eval_batch_size": null,
"eval_strategy": null,
"self_cognition_sample": 0,
"train_dataset_mix_ratio": 0.0,
"train_dataset_mix_ds": [
"ms-bench"
],
"train_dataset_sample": -1,
"val_dataset_sample": null,
"safe_serialization": null,
"only_save_model": null,
"neftune_alpha": null,
"deepspeed_config_path": null,
"model_cache_dir": null,
"lora_dropout_p": null,
"lora_target_modules": [
"dense",
"key_value",
"up_proj",
"down_proj",
"query",
"gate_proj"
],
"lora_target_regex": null,
"lora_modules_to_save": [],
"boft_target_modules": [],
"boft_modules_to_save": [],
"vera_target_modules": [],
"vera_modules_to_save": [],
"ia3_target_modules": [],
"ia3_modules_to_save": [],
"custom_train_dataset_path": [],
"custom_val_dataset_path": [],
"device_map_config_path": null,
"push_hub_strategy": null,
"use_self_cognition": false,
"is_multimodal": false,
"is_vision": false,
"lora_use_embedding": false,
"lora_use_all": true,
"lora_m2s_use_embedding": false,
"lora_m2s_use_ln": false,
"torch_dtype": "torch.float16",
"fp16": true,
"bf16": false,
"rank": 0,
"local_rank": 0,
"world_size": 2,
"local_world_size": 2,
"bnb_4bit_compute_dtype": "torch.float16",
"load_in_4bit": false,
"load_in_8bit": false,
"train_sampler_random": true,
"train_type": "sft",
"training_args": "Seq2SeqTrainingArguments(output_dir='/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=0.0001, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1, num_train_epochs=7, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs={}, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=20, save_total_limit=15, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=False, fp16=True, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend='nccl', tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=50, dataloader_num_workers=1, dataloader_prefetch_factor=None, past_index=-1, run_name='/data/Telechat/TeleChat2/TeleChat2-7B/output/telechat2-115b/v0-20250307-174240', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=False, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', evaluation_strategy=None, push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, split_batches=None, include_tokens_per_second=False, include_num_input_tokens_seen=False, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, eval_use_gather_object=False, average_tokens_across_devices=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=GenerationConfig {\n \"bos_token_id\": 1,\n \"bot_token_id\": 5,\n \"do_sample\": true,\n \"eos_token_id\": 2,\n \"max_new_tokens\": 2048,\n \"pad_token_id\": 3,\n \"repetition_penalty\": 1.02,\n \"start_token_id\": 1,\n \"temperature\": 0.3,\n \"top_k\": 5,\n \"top_p\": 0.85,\n \"user_token_id\": 4\n}\n, acc_strategy='token', loss_name=None, additional_saved_files=[], train_sampler_random=True, metric_warmup_step=0, train_dataset_sample=-1)"
}

42
special_tokens_map.json Normal file
View File

@@ -0,0 +1,42 @@
{
"additional_special_tokens": [
"<_start>",
"<_end>",
"<_pad>",
"<_user>",
"<_bot>",
"<_system>",
"<tool_call>",
"</tool_call>",
"<tool_response>",
"</tool_response>"
],
"bos_token": {
"content": "<_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<_end>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<_pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

223
tokenization_telechat2.py Normal file
View File

@@ -0,0 +1,223 @@
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple
import sentencepiece as spm
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
from transformers.utils import logging
logger = logging.get_logger(__name__)
VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
# TODO: when we get download url from huggingface, refresh the map
PRETRAINED_VOCAB_FILES_MAP = {
"vocab_file": {},
"tokenizer_file": {},
}
class Telechat2Tokenizer(PreTrainedTokenizer):
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
model_input_names = ["input_ids", "attention_mask"]
def __init__(
self,
vocab_file,
unk_token="<unk>",
bos_token="<_start>",
eos_token="<_end>",
pad_token="<_pad>",
sp_model_kwargs: Optional[Dict[str, Any]] = None,
add_bos_token=True,
add_eos_token=False,
clean_up_tokenization_spaces=False,
**kwargs,
):
self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
self.sp_model.Load(vocab_file)
super().__init__(
bos_token=bos_token,
eos_token=eos_token,
unk_token=unk_token,
pad_token=pad_token,
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
sp_model_kwargs=self.sp_model_kwargs,
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
**kwargs,
)
self.vocab_file = vocab_file
self.add_bos_token = add_bos_token
self.add_eos_token = add_eos_token
def __getstate__(self):
state = self.__dict__.copy()
state["sp_model"] = None
return state
def __setstate__(self, d):
self.__dict__ = d
self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
self.sp_model.Load(self.vocab_file)
@property
def vocab_size(self):
"""Returns vocab size"""
return self.sp_model.get_piece_size()
def get_vocab(self):
"""Returns vocab as a dict"""
vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
vocab.update(self.added_tokens_encoder)
return vocab
@property
def vocab(self):
return self.get_vocab()
def _tokenize(self, text):
"""Returns a tokenized string."""
return self.sp_model.encode(text, out_type=str)
def _convert_token_to_id(self, token):
"""Converts a token (str) in an id using the vocab."""
return self.sp_model.piece_to_id(token)
def _convert_id_to_token(self, index):
"""Converts an index (integer) in a token (str) using the vocab."""
token = self.sp_model.IdToPiece(index)
return token
def convert_tokens_to_string(self, tokens):
"""Converts a sequence of tokens (string) in a single string."""
current_sub_tokens = []
out_string = ""
# prev_is_special = False
for i, token in enumerate(tokens):
# make sure that special tokens are not decoded using sentencepiece model
if token in self.all_special_tokens:
# if not prev_is_special and i != 0:
# out_string += " "
out_string += self.sp_model.decode(current_sub_tokens) + token
# prev_is_special = True
current_sub_tokens = []
else:
current_sub_tokens.append(token)
# prev_is_special = False
out_string += self.sp_model.decode(current_sub_tokens)
return out_string
def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
"""
Save the vocabulary and special tokens file to a directory.
Args:
save_directory (`str`):
The directory in which to save the vocabulary.
Returns:
`Tuple(str)`: Paths to the files saved.
"""
if not os.path.isdir(save_directory):
logger.error(f"Vocabulary path ({save_directory}) should be a directory")
return
out_vocab_file = os.path.join(
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
)
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
copyfile(self.vocab_file, out_vocab_file)
elif not os.path.isfile(self.vocab_file):
with open(out_vocab_file, "wb") as fi:
content_spiece_model = self.sp_model.serialized_model_proto()
fi.write(content_spiece_model)
return (out_vocab_file,)
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
bos_token_id = [self.bos_token_id] if self.add_bos_token else []
eos_token_id = [self.eos_token_id] if self.add_eos_token else []
output = bos_token_id + token_ids_0 + eos_token_id
if token_ids_1 is not None:
output = output + bos_token_id + token_ids_1 + eos_token_id
return output
def get_special_tokens_mask(
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None,
already_has_special_tokens: bool = False
) -> List[int]:
"""
Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `prepare_for_model` method.
Args:
token_ids_0 (`List[int]`):
List of IDs.
token_ids_1 (`List[int]`, *optional*):
Optional second list of IDs for sequence pairs.
already_has_special_tokens (`bool`, *optional*, defaults to `False`):
Whether or not the token list is already formatted with special tokens for the model.
Returns:
`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
"""
if already_has_special_tokens:
return super().get_special_tokens_mask(
token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
)
bos_token_id = [1] if self.add_bos_token else []
eos_token_id = [1] if self.add_eos_token else []
if token_ids_1 is None:
return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
return (
bos_token_id
+ ([0] * len(token_ids_0))
+ eos_token_id
+ bos_token_id
+ ([0] * len(token_ids_1))
+ eos_token_id
)
def create_token_type_ids_from_sequences(
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
"""
Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
sequence pair mask has the following format:
```
0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
| first sequence | second sequence |
```
if token_ids_1 is None, only returns the first portion of the mask (0s).
Args:
token_ids_0 (`List[int]`):
List of ids.
token_ids_1 (`List[int]`, *optional*):
Optional second list of IDs for sequence pairs.
Returns:
`List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
bos_token_id = [self.bos_token_id] if self.add_bos_token else []
eos_token_id = [self.eos_token_id] if self.add_eos_token else []
output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
if token_ids_1 is not None:
output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
return output

3
tokenizer.model Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a7a5b465bbc9465b214e0962076c1170783a8ee88fb01454b0c33609bd3cf954
size 2197499

124
tokenizer_config.json Normal file
View File

@@ -0,0 +1,124 @@
{
"add_bos_token": false,
"add_eos_token": false,
"added_tokens_decoder": {
"1": {
"content": "<_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<_end>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "<_pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "<_user>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "<_bot>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "<_system>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"9": {
"content": "<tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"10": {
"content": "</tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"11": {
"content": "<tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"12": {
"content": "</tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"131072": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<_start>",
"<_end>",
"<_pad>",
"<_user>",
"<_bot>",
"<_system>",
"<tool_call>",
"</tool_call>",
"<tool_response>",
"</tool_response>"
],
"auto_map": {
"AutoTokenizer": [
"tokenization_telechat2.Telechat2Tokenizer",
null
]
},
"bos_token": "<_start>",
"chat_template": "{%- if tools %}\n {%- if messages[0]['role'] == 'system' %}\n {{-'<_system>'+messages[0]['content'] }}\n {%- else %}\n {{- '<_system>'+'你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。' }}\n {%- endif %}\n {{- '\\n\\n# 可用工具\\n你可以调用<tools></tools>标签中包含的一个或多个工具来辅助你回答问题,以下是可用工具详情：\\n<tools>\\n' }}\n {%- for tool in tools %}\n {{- tool | tojson }}\n {{-'\\n'}}\n {%- endfor %}\n {{- '</tools>\\n\\n# 调用方法\\n你需要遵循工具的要求，使用json格式返回工具名称及参数，并用<tool_call></tool_call>包含。下方是一个调用模板：\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call>\\n\\n' }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<_system>' + messages[0]['content'] + '\\n' }}\n {%- else %}\n {{- '<_system>'+'你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == 'user') %}\n {{- '<_user>' + message.content }}\n {%- elif message.role == 'bot' %}\n {{- '<_bot>' }}\n {%- if message.content %}\n {{- message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.index0 == 0 %}\n {{-'<tool_call>'}}\n {%- else %}\n {{-'\\n<tool_call>'}}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}{{ tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<_end>\\n' }}\n {%- elif message.role == 'tool' %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != 'tool') %}\n {{- '<_user>'+'<tool_response>\\n' }}\n {%- else %}\n {{- '\\n<tool_response>\\n' }}\n {%- endif %}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<_bot>' }}\n{%- endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<_end>",
"extra_special_tokens": {},
"model_max_length": 100000000,
"pad_token": "<_pad>",
"sp_model_kwargs": {},
"split_special_tokens": false,
"tokenizer_class": "Telechat2Tokenizer",
"unk_token": "<unk>",
"use_fast": false
}