初始化项目，由ModelHub XC社区提供模型
Model: inclusionAI/Ring-mini-sparse-2.0-exp Source: Original Platform
This commit is contained in:
ModelHub XC
53
.gitattributes
vendored
Normal file
53
.gitattributes
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 inclusionAI
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
153
README.md
Normal file
153
README.md
Normal file
@@ -0,0 +1,153 @@
|
||||
---
|
||||
license: mit
|
||||
language:
|
||||
- en
|
||||
base_model:
|
||||
- inclusionAI/Ling-mini-base-2.0-20T
|
||||
pipeline_tag: text-generation
|
||||
library_name: transformers
|
||||
tags:
|
||||
- moe
|
||||
---
|
||||
# Ring-mini-sparse-2.0-exp
|
||||
|
||||
<p align="center">
|
||||
<img src="https://mdn.alipayobjects.com/huamei_qa8qxu/afts/img/A*4QxcQrBlTiAAAAAAQXAAAAgAemJ7AQ/original" width="100"/>
|
||||
<p>
|
||||
<p align="center">🤗 <a href="https://huggingface.co/inclusionAI/Ring-mini-sparse-2.0-exp">Hugging Face</a>   |   🤖 <a href="https://modelscope.cn/organization/inclusionAI/Ring-mini-sparse-2.0-exp">ModelScope</a></p>
|
||||
|
||||
## Introduction
|
||||
|
||||
We are excited to annouce the official release of Ring-mini-sparse-2.0-exp. This model employs a Mixture of Block Attention (MoBA) architecture, delivering highly efficient inference without compromising performance. This model inherts from [Ling-mini-base-2.0](https://huggingface.co/inclusionAI/Ling-mini-base-2.0-20T), continually trained on an additional 100B tokens. The performance of the MoBA-based model is on par with the standard attention models of the same size (e.g., Ring-mini-v2). Furthermore, by applying YaRN-based 4× window extrapolation, we extend the context length to 128K tokens, delivering superior inference speed on tasks that involve long inputs and outputs.
|
||||
|
||||
<div style="display: flex; justify-content: center;">
|
||||
<div style="text-align: center;">
|
||||
<img src="https://mdn.alipayobjects.com/huamei_9mcypc/afts/img/PIoSTKEzmsEAAAAAU5AAAAgADlCHAQFr/original" width="800">
|
||||
<p style="margin-top: 8px; font-size: 14px;"><strong>Figure 1:</strong> The Model Architecture of Ring-mini-sparse-2.0-exp</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
## Evaluation
|
||||
|
||||
To comprehensively assess the reasoning capability of our model, we conducted evaluations on five challenging benchmarks spanning mathematics, coding, and science, comparing it with Ring-mini-2.0, Qwen3-8B-Thinking, and GPT-OSS-20B-Medium. The MoBA architecture demonstrates comparable performance to full softmax attention models.
|
||||
|
||||
<div style="display: flex; justify-content: center;">
|
||||
<div style="text-align: center;">
|
||||
<img src="https://mdn.alipayobjects.com/huamei_9mcypc/afts/img/Yr7eRreHNNUAAAAAWfAAAAgADlCHAQFr/original" width="100%">
|
||||
<p style="margin-top: 8px; font-size: 14px;"><strong>Figure 2:</strong> Model Performance Comparison </p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
## Highly Sparse, High-Speed Generation
|
||||
|
||||
Ring-mini-sparse-2.0-exp achieves high inference efficiency through highly sparse attention and a Mixture-of-Experts (MoE) architecture. Unlike MoBA used in Kimi, our approach shares the same KV block selection across all heads within a GQA group, reducing the total number of KV tokens each query head retrieves from the KV cache during decoding. During 64K-context decoding, only 8,192 key-value (KV) tokens are activated per query—reducing KV cache retrieval overhead by 87.5% compared to full attention and delivering up to 3× inference speedup over Ring-mini-2.0. This design significantly lowers computational costs for high-concurrency scenarios involving reasoning-intensive models while maintaining competitive performance. Additionally, with YaRN extrapolation, the model extends context capacity to 128K tokens, achieving up to 2× relative speedup in long-input scenarios compared to Ring-mini-2.0 (full softmax attention).
|
||||
|
||||
<div style="text-align: center;">
|
||||
<p align="center">
|
||||
<img src="https://mdn.alipayobjects.com/huamei_9mcypc/afts/img/iL_eTZP-FVEAAAAATOAAAAgADlCHAQFr/original" width="500">
|
||||
</p>
|
||||
<p style="margin-top: 8px; font-size: 14px;"><strong>Figure 4:</strong> Inference speedup ratios of Ring-mini-sparse-2.0-exp compared to Ring-mini-2.0.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
## Quickstart
|
||||
|
||||
### 🤗 Hugging Face Transformers
|
||||
Installation requirements:
|
||||
|
||||
```shell
|
||||
pip install flash-attn==2.6.3
|
||||
pip install transformers==4.56.1
|
||||
```
|
||||
|
||||
Here is a code snippet to show you how to use the chat model with `transformers`:
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
model_name = "inclusionAI/Ring-mini-sparse-2.0-exp"
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name,
|
||||
dtype="auto",
|
||||
device_map="auto",
|
||||
trust_remote_code=True,
|
||||
attn_implementation="flash_attention_2",
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
|
||||
|
||||
prompts = [
|
||||
"Give me a short introduction to large language models."
|
||||
]
|
||||
input_texts = []
|
||||
for prompt in prompts:
|
||||
messages = [
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
input_texts.append(text)
|
||||
|
||||
print(input_texts)
|
||||
|
||||
model_inputs = tokenizer(input_texts, return_tensors="pt", return_token_type_ids=False, padding=True, padding_side='left').to(model.device)
|
||||
|
||||
generated_ids = model.generate(
|
||||
**model_inputs,
|
||||
max_new_tokens=8192,
|
||||
do_sample=False,
|
||||
)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
|
||||
responses = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
|
||||
|
||||
print("*" * 30)
|
||||
print(responses)
|
||||
print("*" * 30)
|
||||
```
|
||||
|
||||
### 🚀 SGLang
|
||||
|
||||
#### Environment Preparation
|
||||
|
||||
We have submitted our PR to SGLang official release and it will be merged later, for now we can prepare the environment following steps, firstly install the community version SGLang and required packages:
|
||||
```shell
|
||||
pip install sglang==0.5.3 sgl-kernel==0.3.15 torch==2.8.0 torchvision==0.23.0 torchao
|
||||
```
|
||||
|
||||
Then you should install our sglang wheel package:
|
||||
```shell
|
||||
git clone https://github.com/inclusionAI/Ring-V2.git
|
||||
pip install Ring-V2/moba/whls/sglang-0.5.3.post1-py3-none-any.whl --no-deps --force-reinstall
|
||||
```
|
||||
|
||||
#### Run Inference
|
||||
|
||||
Our model is supported by SGLang now. You can launch the sever with the command in the following:
|
||||
|
||||
- Start server:
|
||||
```shell
|
||||
python -m sglang.launch_server \
|
||||
--model-path <model_path> \
|
||||
--trust-remote-code \
|
||||
--tp-size 4 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size 0 \
|
||||
--attention-backend moba
|
||||
```
|
||||
|
||||
- Client:
|
||||
|
||||
```shell
|
||||
curl -s http://localhost:${PORT}/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "auto", "temperature": 0.6, "messages": [{"role": "user", "content": "Give me a short introduction to large language models."}]}'
|
||||
```
|
||||
|
||||
More usage can be found [here](https://docs.sglang.ai/basic_usage/send_request.html)
|
||||
54
config.json
Normal file
54
config.json
Normal file
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"architectures": [
|
||||
"BailingMoeV2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_bailing_moe_v2.BailingMoeV2Config",
|
||||
"AutoModel": "modeling_bailing_moe_v2.BailingMoeV2Model",
|
||||
"AutoModelForCausalLM": "modeling_bailing_moe_v2.BailingMoeV2ForCausalLM"
|
||||
},
|
||||
"num_hidden_layers": 20,
|
||||
"hidden_size": 2048,
|
||||
"intermediate_size": 5120,
|
||||
"eos_token_id": 156892,
|
||||
"pad_token_id": 156892,
|
||||
"first_k_dense_replace": 1,
|
||||
"hidden_act": "silu",
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "bailing_moe",
|
||||
"moe_intermediate_size": 512,
|
||||
"norm_topk_prob": true,
|
||||
"num_experts_per_tok": 8,
|
||||
"num_attention_heads": 16,
|
||||
"num_experts": 256,
|
||||
"num_key_value_heads": 4,
|
||||
"rope_theta": 600000,
|
||||
"rope_scaling": null,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.52.3",
|
||||
"use_bias": false,
|
||||
"use_rmsnorm": true,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"head_dim": 128,
|
||||
"num_shared_experts": 1,
|
||||
"use_cache": true,
|
||||
"use_qkv_bias": false,
|
||||
"embedding_dropout": 0.0,
|
||||
"output_dropout": 0.0,
|
||||
"vocab_size": 157184,
|
||||
"partial_rotary_factor": 0.5,
|
||||
"router_dtype": "fp32",
|
||||
"moe_router_enable_expert_bias": true,
|
||||
"routed_scaling_factor": 2.5,
|
||||
"n_group": 8,
|
||||
"topk_group": 4,
|
||||
"use_qk_norm": true,
|
||||
"score_function": "sigmoid",
|
||||
"moe_shared_expert_intermediate_size": 512,
|
||||
"moba_block_size": 1024,
|
||||
"moba_topk": 8,
|
||||
"use_moba_decode": true,
|
||||
"moba_layer_freq": [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework":"Pytorch","task":"question-answering"}
|
||||
84
configuration_bailing_moe_v2.py
Normal file
84
configuration_bailing_moe_v2.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""Bailing MoE V2 model configuration"""
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
|
||||
class BailingMoeV2Config(PretrainedConfig):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=157184,
|
||||
hidden_size=2048,
|
||||
intermediate_size=5120,
|
||||
num_hidden_layers=20,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=4,
|
||||
hidden_act="silu",
|
||||
use_qkv_bias=False, # bailing only
|
||||
use_bias=False, # bailing only
|
||||
rms_norm_eps=1e-06,
|
||||
tie_word_embeddings=False, # PretrainedConfig key, here change default value.
|
||||
embedding_dropout=0.0,
|
||||
attention_dropout=0.0,
|
||||
output_dropout=0.0,
|
||||
initializer_range=0.02,
|
||||
max_position_embeddings=32768,
|
||||
rope_theta=600000.0,
|
||||
use_cache=True,
|
||||
max_window_layers=20,
|
||||
rope_scaling=None,
|
||||
pad_token_id=156892,
|
||||
eos_token_id=156892,
|
||||
num_experts=256,
|
||||
num_shared_experts=1,
|
||||
num_experts_per_tok=8,
|
||||
n_group=8,
|
||||
topk_group=4,
|
||||
moe_intermediate_size=512,
|
||||
first_k_dense_replace=1,
|
||||
head_dim=128,
|
||||
output_router_logits=False,
|
||||
use_qk_norm=True,
|
||||
num_nextn_predict_layers=0,
|
||||
mtp_loss_scaling_factor=0,
|
||||
moe_router_enable_expert_bias=True,
|
||||
routed_scaling_factor=1.0,
|
||||
**kwargs,
|
||||
):
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.use_qkv_bias = use_qkv_bias
|
||||
self.use_bias = use_bias
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.embedding_dropout = embedding_dropout
|
||||
self.attention_dropout = attention_dropout
|
||||
self.output_dropout = output_dropout
|
||||
self.num_nextn_predict_layers = num_nextn_predict_layers
|
||||
self.mtp_loss_scaling_factor = mtp_loss_scaling_factor
|
||||
self.initializer_range = initializer_range
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rope_theta = rope_theta
|
||||
self.use_cache = use_cache
|
||||
self.max_window_layers = max_window_layers
|
||||
self.head_dim = head_dim or self.hidden_size // self.num_attention_heads
|
||||
self.rope_scaling = rope_scaling
|
||||
self.use_qk_norm = use_qk_norm
|
||||
self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
|
||||
# MoE configs
|
||||
self.num_experts = num_experts
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.n_group = n_group
|
||||
self.topk_group = topk_group
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.first_k_dense_replace = first_k_dense_replace
|
||||
self.output_router_logits = output_router_logits
|
||||
|
||||
super().__init__(pad_token_id=pad_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)
|
||||
9
generation_config.json
Normal file
9
generation_config.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"bos_token_id": 156891,
|
||||
"eos_token_id": [
|
||||
156892,
|
||||
156895
|
||||
],
|
||||
"pad_token_id": 156892,
|
||||
"transformers_version": "4.56.1"
|
||||
}
|
||||
3
model-00001-of-00004.safetensors
Normal file
3
model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:efab52319e26654aba6a683fe3c5f7526ac5405fa64f42b68eca7695b599984f
|
||||
size 8951195664
|
||||
3
model-00002-of-00004.safetensors
Normal file
3
model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:aef03dc0f0606de5a240c3e993234461774a18a272bf2a072c7929e6ba8643f8
|
||||
size 9834183392
|
||||
3
model-00003-of-00004.safetensors
Normal file
3
model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5cc079480e86885dacb049a7069151f69d28e23034860827adc02acd66be419a
|
||||
size 9834186472
|
||||
3
model-00004-of-00004.safetensors
Normal file
3
model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:46bb29007589c392f841974b70298b5f6e6ce787c1bde7b24e5cdf63620714b8
|
||||
size 3893569552
|
||||
14820
model.safetensors.index.json
Normal file
14820
model.safetensors.index.json
Normal file
File diff suppressed because it is too large
Load Diff
1904
modeling_bailing_moe_v2.py
Normal file
1904
modeling_bailing_moe_v2.py
Normal file
File diff suppressed because it is too large
Load Diff
7
special_tokens_map.json
Normal file
7
special_tokens_map.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"bos_token": "<|startoftext|>",
|
||||
"cls_token": "[CLS]",
|
||||
"eos_token": "<|endoftext|>",
|
||||
"gmask_token": "[gMASK]",
|
||||
"pad_token": "<|endoftext|>"
|
||||
}
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:23895938c755ebef359350a758831dc230a481428155d0f50a236d572e860b21
|
||||
size 7663404
|
||||
17
tokenizer_config.json
Normal file
17
tokenizer_config.json
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_eos_token": false,
|
||||
"bos_token": "<|startoftext|>",
|
||||
"chat_template": "{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role><think>\n' }}{% endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"cls_token": "[CLS]",
|
||||
"eos_token": "<|endoftext|>",
|
||||
"fast_tokenizer": true,
|
||||
"gmask_token": "[gMASK]",
|
||||
"merges_file": null,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"tokenizer_class": "PreTrainedTokenizerFast",
|
||||
"trust_remote_code": true,
|
||||
"vocab_file": null
|
||||
}
|
||||
Reference in New Issue
Block a user