commit 89c333fe7b2ec115c55a6a3929c9cbd922b6285f Author: ModelHub XC Date: Mon May 25 16:25:14 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Fengshenbang/Ziya-LLaMA-13B-v1.1 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..97c8e80 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,32 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..dfac4b5 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +--- +license: GPL-3.0 +tasks: +- text-generation +language: +- en +- zh + +studios: +- Fengshenbang/Ziya_LLaMA_13B_v1_online + +--- + + +# Ziya-LLaMA-13B-v1.1 + +- Main Page:[Fengshenbang](https://fengshenbang-lm.com/) +- Github: [Fengshenbang-LM](https://github.com/IDEA-CCNL/Fengshenbang-LM) + + +# 姜子牙系列模型 +- [Ziya-LLaMA-13B-v1.1](https://modelscope.cn/models/Fengshenbang/Ziya-LLaMA-13B-v1.1) +- [Ziya-LLaMA-13B-v1](https://modelscope.cn/models/Fengshenbang/Ziya-LLaMA-13B-v1/summary) +- [Ziya-LLaMA-7B-Reward](https://huggingface.co/IDEA-CCNL/Ziya-LLaMA-7B-Reward) +- [Ziya-LLaMA-13B-Pretrain-v1](https://huggingface.co/IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1) +- [Ziya-BLIP2-14B-Visual-v1](https://modelscope.cn/models/Fengshenbang/Ziya-BLIP2-14B-Visual-v1/summary) + +## 简介 Brief Introduction +我们对Ziya-LLaMA-13B-v1模型进行继续优化，推出开源版本Ziya-LLaMA-13B-v1.1。通过调整微调数据的比例和采用更优的强化学习策略，本版本在问答准确性、数学能力以及安全性等方面得到了提升，详细能力分析如下图所示。 + +We have further optimized the Ziya-LLaMA-13B-v1 model and released the open-source version Ziya-LLaMA-13B-v1.1. By adjusting the proportion of fine-tuning data and adopting a better reinforcement learning strategy, this version has achieved improvements in question-answering accuracy, mathematical ability, and safety, as shown in the following figure in detail. + + + +## 软件依赖 +``` +pip install torch==1.12.1 tokenizers==0.13.3 git+https://github.com/huggingface/transformers +``` +## 使用 Usage +``` +from modelscope.utils.constant import Tasks +from modelscope.pipelines import pipeline +pipe = pipeline(task=Tasks.text_generation, model='Fengshenbang/Ziya-LLaMA-13B-v1.1', model_revision='v1.0.2', device='cuda') +query="帮我写一份去西安的旅游计划" +inputs = ':' + query.strip() + '\n:' +result = pipe(inputs) +print(result) + +``` + + + +## 引用 Citation + +如果您在您的工作中使用了我们的模型，可以引用我们的[论文](https://arxiv.org/abs/2210.08590)： + +If you are using the resource for your work, please cite the our [paper](https://arxiv.org/abs/2210.08590): + +```text +@article{fengshenbang, + author = {Jiaxing Zhang and Ruyi Gan and Junjie Wang and Yuxiang Zhang and Lin Zhang and Ping Yang and Xinyu Gao and Ziwei Wu and Xiaoqun Dong and Junqing He and Jianheng Zhuo and Qi Yang and Yongfeng Huang and Xiayu Li and Yanghan Wu and Junyu Lu and Xinyu Zhu and Weifeng Chen and Ting Han and Kunhao Pan and Rui Wang and Hao Wang and Xiaojun Wu and Zhongshen Zeng and Chongpei Chen}, + title = {Fengshenbang 1.0: Being the Foundation of Chinese Cognitive Intelligence}, + journal = {CoRR}, + volume = {abs/2209.02970}, + year = {2022} +} +``` + +You can also cite our [website](https://github.com/IDEA-CCNL/Fengshenbang-LM/): + +欢迎引用我们的[网站](https://github.com/IDEA-CCNL/Fengshenbang-LM/): +```text +@misc{Fengshenbang-LM, + title={Fengshenbang-LM}, + author={IDEA-CCNL}, + year={2021}, + howpublished={\url{https://github.com/IDEA-CCNL/Fengshenbang-LM}}, +} +``` \ No newline at end of file diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..ac47931 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,4 @@ +{ + "": 39409, + "": 39408 +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..4ba69b8 --- /dev/null +++ b/config.json @@ -0,0 +1,23 @@ +{ + "_name_or_path": "/cognitive_comp/wanghao/models/llama_sft/llama_13B_v2_S2.2_S3_S3.1_rlhf_0601_RM13B0525_step44_hf/", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 13824, + "max_position_embeddings": 2048, + "model_type": "llama", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "pad_token_id": 0, + "rms_norm_eps": 1e-06, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.29.0.dev0", + "use_cache": true, + "vocab_size": 39424 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..c587c3c --- /dev/null +++ b/configuration.json @@ -0,0 +1,11 @@ +{ + "framework": "pytorch", + "task": "text-generation", + "model": { + "type": "Ziya-LLaMA-13B-v1" + }, + "pipeline": { + "type": "Ziya-LLaMA-13B-v1-text-generation-pipe" + }, + "allow_remote": true +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..2c05748 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.29.0.dev0" +} diff --git a/ms_wrapper.py b/ms_wrapper.py new file mode 100644 index 0000000..4eb2fa6 --- /dev/null +++ b/ms_wrapper.py @@ -0,0 +1,67 @@ +import os +from typing import Union, Dict, Any +from modelscope.pipelines.builder import PIPELINES +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks +from modelscope.pipelines.base import Pipeline +from modelscope.models.base import Model, TorchModel +from modelscope.utils.logger import get_logger +from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers import LlamaForCausalLM + +if 'CUDA_VISIBLE_DEVICES' not in os.environ: + os.environ['CUDA_VISIBLE_DEVICES'] = '0' + + + +@PIPELINES.register_module(Tasks.text_generation, module_name='Ziya-LLaMA-13B-v1-text-generation-pipe') +class ZiyaLLaMA13Bv1TextGenerationPipeline(Pipeline): + def __init__( + self, + model: Union[Model, str], + *args, + **kwargs): + model = ZiyaLLaMA13Bv1TextGeneration(model) if isinstance(model, str) else model + super().__init__(model=model, **kwargs) + + def preprocess(self, inputs, **preprocess_params) -> Dict[str, Any]: + return inputs + + # define the forward pass + def forward(self, inputs: Dict, **forward_params) -> Dict[str, Any]: + return self.model(inputs) + + # format the outputs from pipeline + def postprocess(self, input, **kwargs) -> Dict[str, Any]: + return input + + + +@MODELS.register_module(Tasks.text_generation, module_name='Ziya-LLaMA-13B-v1') +class ZiyaLLaMA13Bv1TextGeneration(TorchModel): + def __init__(self, model_dir=None, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.logger = get_logger() + # loading tokenizer + self.tokenizer = AutoTokenizer.from_pretrained(model_dir) + self.model = LlamaForCausalLM.from_pretrained(model_dir, device_map="auto") + self.model = self.model.eval() + + def forward(self,input: Dict) -> Dict[str, Any]: + output = {} + res = self.infer(input) + output['text'] = res + return output + + def quantize(self, bits: int): + self.model = self.model.quantize(bits) + return self + + def infer(self, input): + device = self.model.device + input_ids = self.tokenizer(input, return_tensors="pt").input_ids.to(device) + logits = self.model.generate(input_ids, max_new_tokens=1024, do_sample = True, + top_p = 0.85, temperature = 1.0, repetition_penalty=1., + eos_token_id=2,bos_token_id=1,pad_token_id=0) + out = self.tokenizer.batch_decode(logits)[0] + return out diff --git a/pk.png b/pk.png new file mode 100644 index 0000000..f45c754 Binary files /dev/null and b/pk.png differ diff --git a/pytorch_model-00001-of-00028.bin b/pytorch_model-00001-of-00028.bin new file mode 100644 index 0000000..ea042a8 --- /dev/null +++ b/pytorch_model-00001-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e38683c5ea691cb0ae75bd15c5f3f059b5af559084569f53b7650c334dbba5f2 +size 896534991 diff --git a/pytorch_model-00002-of-00028.bin b/pytorch_model-00002-of-00028.bin new file mode 100644 index 0000000..db428ac --- /dev/null +++ b/pytorch_model-00002-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b340ee8c36cf345ec05313eb538656945798c81aa7455fb543816d136c20753d +size 985707823 diff --git a/pytorch_model-00003-of-00028.bin b/pytorch_model-00003-of-00028.bin new file mode 100644 index 0000000..8892266 --- /dev/null +++ b/pytorch_model-00003-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:766dfaf59c5131b868148f24a4c809ed6e115950e54b99067cda83b4b6455ad1 +size 917528001 diff --git a/pytorch_model-00004-of-00028.bin b/pytorch_model-00004-of-00028.bin new file mode 100644 index 0000000..bcc2d60 --- /dev/null +++ b/pytorch_model-00004-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dae93f62b033700fee2dc87c49d2d7e62cc28dc037d73a369f88b9b65ecc6b +size 985707823 diff --git a/pytorch_model-00005-of-00028.bin b/pytorch_model-00005-of-00028.bin new file mode 100644 index 0000000..e72a97f --- /dev/null +++ b/pytorch_model-00005-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ba69524024bdff7da885fd7a03fac6f155393f49f158ffee2c1c107423eb71b +size 917528001 diff --git a/pytorch_model-00006-of-00028.bin b/pytorch_model-00006-of-00028.bin new file mode 100644 index 0000000..da0e2bb --- /dev/null +++ b/pytorch_model-00006-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04e189dfc68405bb9f3db09f3eab7625b5e910aff95d099e81d7945dce2ecded +size 985707823 diff --git a/pytorch_model-00007-of-00028.bin b/pytorch_model-00007-of-00028.bin new file mode 100644 index 0000000..95545f6 --- /dev/null +++ b/pytorch_model-00007-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:324e3511699012c539ef6147542e7dc02e6d0299bad720dae4b6d356aa4e3607 +size 917528001 diff --git a/pytorch_model-00008-of-00028.bin b/pytorch_model-00008-of-00028.bin new file mode 100644 index 0000000..5ba737c --- /dev/null +++ b/pytorch_model-00008-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bc63c615d088a118f94150b92cf0bd9c05ebce6db9b268703b9a0d883c1172c +size 985707823 diff --git a/pytorch_model-00009-of-00028.bin b/pytorch_model-00009-of-00028.bin new file mode 100644 index 0000000..05c194c --- /dev/null +++ b/pytorch_model-00009-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0321aa53ca25f671730d430e467a838d9e0bf7750d32e5b41e9c272ab7aeddaf +size 917528001 diff --git a/pytorch_model-00010-of-00028.bin b/pytorch_model-00010-of-00028.bin new file mode 100644 index 0000000..fd43a97 --- /dev/null +++ b/pytorch_model-00010-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af43d0f5fba3fb49587b139077d8de090dd45031cc82a4d8734d28f5d7e1cd39 +size 985707823 diff --git a/pytorch_model-00011-of-00028.bin b/pytorch_model-00011-of-00028.bin new file mode 100644 index 0000000..dbcbd91 --- /dev/null +++ b/pytorch_model-00011-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:876126243e1bbd4bcd70805374b3e312fe6e7a1663b67d85175ee510193e717b +size 917528001 diff --git a/pytorch_model-00012-of-00028.bin b/pytorch_model-00012-of-00028.bin new file mode 100644 index 0000000..1a7b802 --- /dev/null +++ b/pytorch_model-00012-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a61eff9376f5957c5fe42ea4a50884589f0c2b7571c184ebd862c964b65b370e +size 985707823 diff --git a/pytorch_model-00013-of-00028.bin b/pytorch_model-00013-of-00028.bin new file mode 100644 index 0000000..513353c --- /dev/null +++ b/pytorch_model-00013-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e791930766b21f12ae62c698d9cf4cc70e80d0cb75e0e43236b04970317909c4 +size 917528001 diff --git a/pytorch_model-00014-of-00028.bin b/pytorch_model-00014-of-00028.bin new file mode 100644 index 0000000..f2b8307 --- /dev/null +++ b/pytorch_model-00014-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef188d54af85e3a64583033c6e46d9d8da988da0ec1a439bb5e4e93f98fcbec5 +size 985707823 diff --git a/pytorch_model-00015-of-00028.bin b/pytorch_model-00015-of-00028.bin new file mode 100644 index 0000000..a9ae59d --- /dev/null +++ b/pytorch_model-00015-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5873cfb977b925c00cbd36f5ab23fc97c3c2001e3b3edd050550de7ea69b4d17 +size 917528001 diff --git a/pytorch_model-00016-of-00028.bin b/pytorch_model-00016-of-00028.bin new file mode 100644 index 0000000..74f59a0 --- /dev/null +++ b/pytorch_model-00016-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89221e84bf635fa868df498ddc2deabaee6f355bbe198cc56b7985a2c9dc8e65 +size 985707823 diff --git a/pytorch_model-00017-of-00028.bin b/pytorch_model-00017-of-00028.bin new file mode 100644 index 0000000..3046c05 --- /dev/null +++ b/pytorch_model-00017-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:007cb822dce729d63b378e55831bcd7d63c34cdfbc9ea73ec4162a89a77e50c6 +size 917528001 diff --git a/pytorch_model-00018-of-00028.bin b/pytorch_model-00018-of-00028.bin new file mode 100644 index 0000000..c95f7c7 --- /dev/null +++ b/pytorch_model-00018-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0deb620e5b18bf2a4ed73010877dc4b6400bbf4c214a8fbe0be5ea0b85067eb2 +size 985707823 diff --git a/pytorch_model-00019-of-00028.bin b/pytorch_model-00019-of-00028.bin new file mode 100644 index 0000000..6effde5 --- /dev/null +++ b/pytorch_model-00019-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be386fe1cb540d304a3c3c593fdd71952da98e6e539294f30acf7c02166287d8 +size 917528001 diff --git a/pytorch_model-00020-of-00028.bin b/pytorch_model-00020-of-00028.bin new file mode 100644 index 0000000..0483263 --- /dev/null +++ b/pytorch_model-00020-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7df2b424a382a241ebd68b901db3795ad869501fbf8a21855918455104abebc +size 985707823 diff --git a/pytorch_model-00021-of-00028.bin b/pytorch_model-00021-of-00028.bin new file mode 100644 index 0000000..ede42e6 --- /dev/null +++ b/pytorch_model-00021-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a929861d68ca3924a326e32f97f0b42cafd3b59ffddca4d8eb864da1f504205 +size 917528001 diff --git a/pytorch_model-00022-of-00028.bin b/pytorch_model-00022-of-00028.bin new file mode 100644 index 0000000..4254113 --- /dev/null +++ b/pytorch_model-00022-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43cc16d8db3d1f5ab9efb615e4afe7427d656948a3b04c59cce2527c0f3dd5a8 +size 985707823 diff --git a/pytorch_model-00023-of-00028.bin b/pytorch_model-00023-of-00028.bin new file mode 100644 index 0000000..d4f1b43 --- /dev/null +++ b/pytorch_model-00023-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73cf6e8aeee34899d2e962fc6433ba08174220303fd92cd51980f1affcfd8883 +size 917528001 diff --git a/pytorch_model-00024-of-00028.bin b/pytorch_model-00024-of-00028.bin new file mode 100644 index 0000000..0c2e183 --- /dev/null +++ b/pytorch_model-00024-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d5070b88d0c6b467f36e4060eaf8ccf36eee269d489813a7e3f3c4df3a2e556 +size 985707823 diff --git a/pytorch_model-00025-of-00028.bin b/pytorch_model-00025-of-00028.bin new file mode 100644 index 0000000..2c97c7c --- /dev/null +++ b/pytorch_model-00025-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa00813562f79aa9bb5ad0a0d04f2a8ad7409d298180568432ec4a4da2a7d768 +size 917528001 diff --git a/pytorch_model-00026-of-00028.bin b/pytorch_model-00026-of-00028.bin new file mode 100644 index 0000000..c6868d7 --- /dev/null +++ b/pytorch_model-00026-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:062d05043f5923592e56c642417702bcd99082960bba3b24dbc8b1d2c947b970 +size 985707823 diff --git a/pytorch_model-00027-of-00028.bin b/pytorch_model-00027-of-00028.bin new file mode 100644 index 0000000..40a33b5 --- /dev/null +++ b/pytorch_model-00027-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:434f427c34596366b531a4438e71123c401b18421d3a4d0444468484f4b5763e +size 917528001 diff --git a/pytorch_model-00028-of-00028.bin b/pytorch_model-00028-of-00028.bin new file mode 100644 index 0000000..7a819cc --- /dev/null +++ b/pytorch_model-00028-of-00028.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:569b67b362ddb717b858d336b592ecd6136228950a824050a2d61e17180829fd +size 545291867 diff --git a/pytorch_model.bin.index.json b/pytorch_model.bin.index.json new file mode 100644 index 0000000..6792b22 --- /dev/null +++ b/pytorch_model.bin.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd2aeaa1baf2bcc094dfe61143c5f49147d8ba1cba70d1182d9f4bd732caf86c +size 33444 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..c0fc1d5 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,27 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..4ed847f --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fd7e445833dd0889206aba242c2a51ecbae2437fd328d1759a35475fd8c0423 +size 588619 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..400e3de --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,33 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "clean_up_tokenization_spaces": false, + "eos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "sp_model_kwargs": {}, + "tokenizer_class": "LlamaTokenizer", + "unk_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +}