mlx-community/helium-1-preview-2b
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: mlx-community/helium-1-preview-2b
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-20 16:43:02 +08:00
commit e7720367ee
9 changed files with 194287 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
model.safetensors filter=lfs diff=lfs merge=lfs -text

43
README.md Normal file
View File

@@ -0,0 +1,43 @@
---
library_name: transformers
license: cc-by-4.0
language:
- en
- fr
- de
- it
- pt
- es
pipeline_tag: text-generation
base_model: kyutai/helium-1-preview-2b
tags:
- mlx
---
# mlx-community/helium-1-preview-2b
The Model [mlx-community/helium-1-preview-2b](https://huggingface.co/mlx-community/helium-1-preview-2b) was
converted to MLX format from [kyutai/helium-1-preview-2b](https://huggingface.co/kyutai/helium-1-preview-2b)
using mlx-lm version **0.21.0**.
## Use with mlx
```bash
pip install mlx-lm
```
```python
from mlx_lm import load, generate
model, tokenizer = load("mlx-community/helium-1-preview-2b")
prompt = "hello"
if tokenizer.chat_template is not None:
messages = [{"role": "user", "content": prompt}]
prompt = tokenizer.apply_chat_template(
messages, add_generation_prompt=True
)
response = generate(model, tokenizer, prompt=prompt, verbose=True)
```

28
config.json Normal file
View File

@@ -0,0 +1,28 @@
{
"architectures": [
"HeliumForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 2560,
"initializer_range": 0.02,
"intermediate_size": 7040,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "helium",
"num_attention_heads": 20,
"num_hidden_layers": 24,
"num_key_value_heads": 20,
"pretraining_tp": 1,
"rms_norm_eps": 1e-08,
"rope_theta": 100000.0,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.45.0.dev0",
"use_cache": true,
"vocab_size": 48000
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:09fa3483e39803f7b8c2ca2d976b463414375babb736e04c981db7a92d64a567
size 4345312713

226
model.safetensors.index.json Normal file
View File

@@ -0,0 +1,226 @@
{
"metadata": {
"total_size": 4345287680
},
"weight_map": {
"lm_head.weight": "model.safetensors",
"model.embed_tokens.weight": "model.safetensors",
"model.layers.0.input_layernorm.weight": "model.safetensors",
"model.layers.0.mlp.down_proj.weight": "model.safetensors",
"model.layers.0.mlp.gate_proj.weight": "model.safetensors",
"model.layers.0.mlp.up_proj.weight": "model.safetensors",
"model.layers.0.post_attention_layernorm.weight": "model.safetensors",
"model.layers.0.self_attn.k_proj.weight": "model.safetensors",
"model.layers.0.self_attn.o_proj.weight": "model.safetensors",
"model.layers.0.self_attn.q_proj.weight": "model.safetensors",
"model.layers.0.self_attn.v_proj.weight": "model.safetensors",
"model.layers.1.input_layernorm.weight": "model.safetensors",
"model.layers.1.mlp.down_proj.weight": "model.safetensors",
"model.layers.1.mlp.gate_proj.weight": "model.safetensors",
"model.layers.1.mlp.up_proj.weight": "model.safetensors",
"model.layers.1.post_attention_layernorm.weight": "model.safetensors",
"model.layers.1.self_attn.k_proj.weight": "model.safetensors",
"model.layers.1.self_attn.o_proj.weight": "model.safetensors",
"model.layers.1.self_attn.q_proj.weight": "model.safetensors",
"model.layers.1.self_attn.v_proj.weight": "model.safetensors",
"model.layers.10.input_layernorm.weight": "model.safetensors",
"model.layers.10.mlp.down_proj.weight": "model.safetensors",
"model.layers.10.mlp.gate_proj.weight": "model.safetensors",
"model.layers.10.mlp.up_proj.weight": "model.safetensors",
"model.layers.10.post_attention_layernorm.weight": "model.safetensors",
"model.layers.10.self_attn.k_proj.weight": "model.safetensors",
"model.layers.10.self_attn.o_proj.weight": "model.safetensors",
"model.layers.10.self_attn.q_proj.weight": "model.safetensors",
"model.layers.10.self_attn.v_proj.weight": "model.safetensors",
"model.layers.11.input_layernorm.weight": "model.safetensors",
"model.layers.11.mlp.down_proj.weight": "model.safetensors",
"model.layers.11.mlp.gate_proj.weight": "model.safetensors",
"model.layers.11.mlp.up_proj.weight": "model.safetensors",
"model.layers.11.post_attention_layernorm.weight": "model.safetensors",
"model.layers.11.self_attn.k_proj.weight": "model.safetensors",
"model.layers.11.self_attn.o_proj.weight": "model.safetensors",
"model.layers.11.self_attn.q_proj.weight": "model.safetensors",
"model.layers.11.self_attn.v_proj.weight": "model.safetensors",
"model.layers.12.input_layernorm.weight": "model.safetensors",
"model.layers.12.mlp.down_proj.weight": "model.safetensors",
"model.layers.12.mlp.gate_proj.weight": "model.safetensors",
"model.layers.12.mlp.up_proj.weight": "model.safetensors",
"model.layers.12.post_attention_layernorm.weight": "model.safetensors",
"model.layers.12.self_attn.k_proj.weight": "model.safetensors",
"model.layers.12.self_attn.o_proj.weight": "model.safetensors",
"model.layers.12.self_attn.q_proj.weight": "model.safetensors",
"model.layers.12.self_attn.v_proj.weight": "model.safetensors",
"model.layers.13.input_layernorm.weight": "model.safetensors",
"model.layers.13.mlp.down_proj.weight": "model.safetensors",
"model.layers.13.mlp.gate_proj.weight": "model.safetensors",
"model.layers.13.mlp.up_proj.weight": "model.safetensors",
"model.layers.13.post_attention_layernorm.weight": "model.safetensors",
"model.layers.13.self_attn.k_proj.weight": "model.safetensors",
"model.layers.13.self_attn.o_proj.weight": "model.safetensors",
"model.layers.13.self_attn.q_proj.weight": "model.safetensors",
"model.layers.13.self_attn.v_proj.weight": "model.safetensors",
"model.layers.14.input_layernorm.weight": "model.safetensors",
"model.layers.14.mlp.down_proj.weight": "model.safetensors",
"model.layers.14.mlp.gate_proj.weight": "model.safetensors",
"model.layers.14.mlp.up_proj.weight": "model.safetensors",
"model.layers.14.post_attention_layernorm.weight": "model.safetensors",
"model.layers.14.self_attn.k_proj.weight": "model.safetensors",
"model.layers.14.self_attn.o_proj.weight": "model.safetensors",
"model.layers.14.self_attn.q_proj.weight": "model.safetensors",
"model.layers.14.self_attn.v_proj.weight": "model.safetensors",
"model.layers.15.input_layernorm.weight": "model.safetensors",
"model.layers.15.mlp.down_proj.weight": "model.safetensors",
"model.layers.15.mlp.gate_proj.weight": "model.safetensors",
"model.layers.15.mlp.up_proj.weight": "model.safetensors",
"model.layers.15.post_attention_layernorm.weight": "model.safetensors",
"model.layers.15.self_attn.k_proj.weight": "model.safetensors",
"model.layers.15.self_attn.o_proj.weight": "model.safetensors",
"model.layers.15.self_attn.q_proj.weight": "model.safetensors",
"model.layers.15.self_attn.v_proj.weight": "model.safetensors",
"model.layers.16.input_layernorm.weight": "model.safetensors",
"model.layers.16.mlp.down_proj.weight": "model.safetensors",
"model.layers.16.mlp.gate_proj.weight": "model.safetensors",
"model.layers.16.mlp.up_proj.weight": "model.safetensors",
"model.layers.16.post_attention_layernorm.weight": "model.safetensors",
"model.layers.16.self_attn.k_proj.weight": "model.safetensors",
"model.layers.16.self_attn.o_proj.weight": "model.safetensors",
"model.layers.16.self_attn.q_proj.weight": "model.safetensors",
"model.layers.16.self_attn.v_proj.weight": "model.safetensors",
"model.layers.17.input_layernorm.weight": "model.safetensors",
"model.layers.17.mlp.down_proj.weight": "model.safetensors",
"model.layers.17.mlp.gate_proj.weight": "model.safetensors",
"model.layers.17.mlp.up_proj.weight": "model.safetensors",
"model.layers.17.post_attention_layernorm.weight": "model.safetensors",
"model.layers.17.self_attn.k_proj.weight": "model.safetensors",
"model.layers.17.self_attn.o_proj.weight": "model.safetensors",
"model.layers.17.self_attn.q_proj.weight": "model.safetensors",
"model.layers.17.self_attn.v_proj.weight": "model.safetensors",
"model.layers.18.input_layernorm.weight": "model.safetensors",
"model.layers.18.mlp.down_proj.weight": "model.safetensors",
"model.layers.18.mlp.gate_proj.weight": "model.safetensors",
"model.layers.18.mlp.up_proj.weight": "model.safetensors",
"model.layers.18.post_attention_layernorm.weight": "model.safetensors",
"model.layers.18.self_attn.k_proj.weight": "model.safetensors",
"model.layers.18.self_attn.o_proj.weight": "model.safetensors",
"model.layers.18.self_attn.q_proj.weight": "model.safetensors",
"model.layers.18.self_attn.v_proj.weight": "model.safetensors",
"model.layers.19.input_layernorm.weight": "model.safetensors",
"model.layers.19.mlp.down_proj.weight": "model.safetensors",
"model.layers.19.mlp.gate_proj.weight": "model.safetensors",
"model.layers.19.mlp.up_proj.weight": "model.safetensors",
"model.layers.19.post_attention_layernorm.weight": "model.safetensors",
"model.layers.19.self_attn.k_proj.weight": "model.safetensors",
"model.layers.19.self_attn.o_proj.weight": "model.safetensors",
"model.layers.19.self_attn.q_proj.weight": "model.safetensors",
"model.layers.19.self_attn.v_proj.weight": "model.safetensors",
"model.layers.2.input_layernorm.weight": "model.safetensors",
"model.layers.2.mlp.down_proj.weight": "model.safetensors",
"model.layers.2.mlp.gate_proj.weight": "model.safetensors",
"model.layers.2.mlp.up_proj.weight": "model.safetensors",
"model.layers.2.post_attention_layernorm.weight": "model.safetensors",
"model.layers.2.self_attn.k_proj.weight": "model.safetensors",
"model.layers.2.self_attn.o_proj.weight": "model.safetensors",
"model.layers.2.self_attn.q_proj.weight": "model.safetensors",
"model.layers.2.self_attn.v_proj.weight": "model.safetensors",
"model.layers.20.input_layernorm.weight": "model.safetensors",
"model.layers.20.mlp.down_proj.weight": "model.safetensors",
"model.layers.20.mlp.gate_proj.weight": "model.safetensors",
"model.layers.20.mlp.up_proj.weight": "model.safetensors",
"model.layers.20.post_attention_layernorm.weight": "model.safetensors",
"model.layers.20.self_attn.k_proj.weight": "model.safetensors",
"model.layers.20.self_attn.o_proj.weight": "model.safetensors",
"model.layers.20.self_attn.q_proj.weight": "model.safetensors",
"model.layers.20.self_attn.v_proj.weight": "model.safetensors",
"model.layers.21.input_layernorm.weight": "model.safetensors",
"model.layers.21.mlp.down_proj.weight": "model.safetensors",
"model.layers.21.mlp.gate_proj.weight": "model.safetensors",
"model.layers.21.mlp.up_proj.weight": "model.safetensors",
"model.layers.21.post_attention_layernorm.weight": "model.safetensors",
"model.layers.21.self_attn.k_proj.weight": "model.safetensors",
"model.layers.21.self_attn.o_proj.weight": "model.safetensors",
"model.layers.21.self_attn.q_proj.weight": "model.safetensors",
"model.layers.21.self_attn.v_proj.weight": "model.safetensors",
"model.layers.22.input_layernorm.weight": "model.safetensors",
"model.layers.22.mlp.down_proj.weight": "model.safetensors",
"model.layers.22.mlp.gate_proj.weight": "model.safetensors",
"model.layers.22.mlp.up_proj.weight": "model.safetensors",
"model.layers.22.post_attention_layernorm.weight": "model.safetensors",
"model.layers.22.self_attn.k_proj.weight": "model.safetensors",
"model.layers.22.self_attn.o_proj.weight": "model.safetensors",
"model.layers.22.self_attn.q_proj.weight": "model.safetensors",
"model.layers.22.self_attn.v_proj.weight": "model.safetensors",
"model.layers.23.input_layernorm.weight": "model.safetensors",
"model.layers.23.mlp.down_proj.weight": "model.safetensors",
"model.layers.23.mlp.gate_proj.weight": "model.safetensors",
"model.layers.23.mlp.up_proj.weight": "model.safetensors",
"model.layers.23.post_attention_layernorm.weight": "model.safetensors",
"model.layers.23.self_attn.k_proj.weight": "model.safetensors",
"model.layers.23.self_attn.o_proj.weight": "model.safetensors",
"model.layers.23.self_attn.q_proj.weight": "model.safetensors",
"model.layers.23.self_attn.v_proj.weight": "model.safetensors",
"model.layers.3.input_layernorm.weight": "model.safetensors",
"model.layers.3.mlp.down_proj.weight": "model.safetensors",
"model.layers.3.mlp.gate_proj.weight": "model.safetensors",
"model.layers.3.mlp.up_proj.weight": "model.safetensors",
"model.layers.3.post_attention_layernorm.weight": "model.safetensors",
"model.layers.3.self_attn.k_proj.weight": "model.safetensors",
"model.layers.3.self_attn.o_proj.weight": "model.safetensors",
"model.layers.3.self_attn.q_proj.weight": "model.safetensors",
"model.layers.3.self_attn.v_proj.weight": "model.safetensors",
"model.layers.4.input_layernorm.weight": "model.safetensors",
"model.layers.4.mlp.down_proj.weight": "model.safetensors",
"model.layers.4.mlp.gate_proj.weight": "model.safetensors",
"model.layers.4.mlp.up_proj.weight": "model.safetensors",
"model.layers.4.post_attention_layernorm.weight": "model.safetensors",
"model.layers.4.self_attn.k_proj.weight": "model.safetensors",
"model.layers.4.self_attn.o_proj.weight": "model.safetensors",
"model.layers.4.self_attn.q_proj.weight": "model.safetensors",
"model.layers.4.self_attn.v_proj.weight": "model.safetensors",
"model.layers.5.input_layernorm.weight": "model.safetensors",
"model.layers.5.mlp.down_proj.weight": "model.safetensors",
"model.layers.5.mlp.gate_proj.weight": "model.safetensors",
"model.layers.5.mlp.up_proj.weight": "model.safetensors",
"model.layers.5.post_attention_layernorm.weight": "model.safetensors",
"model.layers.5.self_attn.k_proj.weight": "model.safetensors",
"model.layers.5.self_attn.o_proj.weight": "model.safetensors",
"model.layers.5.self_attn.q_proj.weight": "model.safetensors",
"model.layers.5.self_attn.v_proj.weight": "model.safetensors",
"model.layers.6.input_layernorm.weight": "model.safetensors",
"model.layers.6.mlp.down_proj.weight": "model.safetensors",
"model.layers.6.mlp.gate_proj.weight": "model.safetensors",
"model.layers.6.mlp.up_proj.weight": "model.safetensors",
"model.layers.6.post_attention_layernorm.weight": "model.safetensors",
"model.layers.6.self_attn.k_proj.weight": "model.safetensors",
"model.layers.6.self_attn.o_proj.weight": "model.safetensors",
"model.layers.6.self_attn.q_proj.weight": "model.safetensors",
"model.layers.6.self_attn.v_proj.weight": "model.safetensors",
"model.layers.7.input_layernorm.weight": "model.safetensors",
"model.layers.7.mlp.down_proj.weight": "model.safetensors",
"model.layers.7.mlp.gate_proj.weight": "model.safetensors",
"model.layers.7.mlp.up_proj.weight": "model.safetensors",
"model.layers.7.post_attention_layernorm.weight": "model.safetensors",
"model.layers.7.self_attn.k_proj.weight": "model.safetensors",
"model.layers.7.self_attn.o_proj.weight": "model.safetensors",
"model.layers.7.self_attn.q_proj.weight": "model.safetensors",
"model.layers.7.self_attn.v_proj.weight": "model.safetensors",
"model.layers.8.input_layernorm.weight": "model.safetensors",
"model.layers.8.mlp.down_proj.weight": "model.safetensors",
"model.layers.8.mlp.gate_proj.weight": "model.safetensors",
"model.layers.8.mlp.up_proj.weight": "model.safetensors",
"model.layers.8.post_attention_layernorm.weight": "model.safetensors",
"model.layers.8.self_attn.k_proj.weight": "model.safetensors",
"model.layers.8.self_attn.o_proj.weight": "model.safetensors",
"model.layers.8.self_attn.q_proj.weight": "model.safetensors",
"model.layers.8.self_attn.v_proj.weight": "model.safetensors",
"model.layers.9.input_layernorm.weight": "model.safetensors",
"model.layers.9.mlp.down_proj.weight": "model.safetensors",
"model.layers.9.mlp.gate_proj.weight": "model.safetensors",
"model.layers.9.mlp.up_proj.weight": "model.safetensors",
"model.layers.9.post_attention_layernorm.weight": "model.safetensors",
"model.layers.9.self_attn.k_proj.weight": "model.safetensors",
"model.layers.9.self_attn.o_proj.weight": "model.safetensors",
"model.layers.9.self_attn.q_proj.weight": "model.safetensors",
"model.layers.9.self_attn.v_proj.weight": "model.safetensors",
"model.norm.weight": "model.safetensors"
}
}

3
special_tokens_map.json Normal file
View File

@@ -0,0 +1,3 @@
{
"pad_token": "<pad>"
}

193082
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

865
tokenizer_config.json Normal file
View File

@@ -0,0 +1,865 @@
{
"added_tokens_decoder": {
"1": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": true
},
"3": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": true
},
"4": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"5": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"6": {
"content": "<|im_sp_00|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"7": {
"content": "<|im_sp_01|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"8": {
"content": "<|im_sp_02|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"9": {
"content": "<|im_sp_03|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"10": {
"content": "<|im_sp_04|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"11": {
"content": "<|im_sp_05|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"12": {
"content": "<|im_sp_06|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"13": {
"content": "<|im_sp_07|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"14": {
"content": "<|im_sp_08|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"15": {
"content": "<|im_sp_09|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"16": {
"content": "<|im_sp_10|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"17": {
"content": "<|im_sp_11|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"18": {
"content": "<|im_sp_12|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"19": {
"content": "<|im_sp_13|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"20": {
"content": "<|im_sp_14|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"21": {
"content": "<|im_sp_15|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"22": {
"content": "<|im_sp_16|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"23": {
"content": "<|im_sp_17|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"24": {
"content": "<|im_sp_18|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"25": {
"content": "<|im_sp_19|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"26": {
"content": "<|im_sp_20|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"27": {
"content": "<|im_sp_21|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"28": {
"content": "<|im_sp_22|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"29": {
"content": "<|im_sp_23|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"30": {
"content": "<|im_sp_24|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"31": {
"content": "<|im_sp_25|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"32": {
"content": "<|im_sp_26|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"33": {
"content": "<|im_sp_27|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"34": {
"content": "<|im_sp_28|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"35": {
"content": "<|im_sp_29|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"36": {
"content": "<|im_sp_30|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"37": {
"content": "<|im_sp_31|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"38": {
"content": "<|im_sp_32|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"39": {
"content": "<|im_sp_33|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"40": {
"content": "<|im_sp_34|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"41": {
"content": "<|im_sp_35|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"42": {
"content": "<|im_sp_36|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"43": {
"content": "<|im_sp_37|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"44": {
"content": "<|im_sp_38|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"45": {
"content": "<|im_sp_39|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"46": {
"content": "<|im_sp_40|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"47": {
"content": "<|im_sp_41|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"48": {
"content": "<|im_sp_42|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"49": {
"content": "<|im_sp_43|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"50": {
"content": "<|im_sp_44|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"51": {
"content": "<|im_sp_45|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"52": {
"content": "<|im_sp_46|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"53": {
"content": "<|im_sp_47|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"54": {
"content": "<|im_sp_48|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"55": {
"content": "<|im_sp_49|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"56": {
"content": "<|im_sp_50|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"57": {
"content": "<|im_sp_51|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"58": {
"content": "<|im_sp_52|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"59": {
"content": "<|im_sp_53|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"60": {
"content": "<|im_sp_54|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"61": {
"content": "<|im_sp_55|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"62": {
"content": "<|im_sp_56|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"63": {
"content": "<|im_sp_57|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"64": {
"content": "<|im_sp_58|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"65": {
"content": "<|im_sp_59|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"66": {
"content": "<|im_sp_60|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"67": {
"content": "<|im_sp_61|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"68": {
"content": "<|im_sp_62|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"69": {
"content": "<|im_sp_63|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"70": {
"content": "<|im_sp_64|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"71": {
"content": "<|im_sp_65|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"72": {
"content": "<|im_sp_66|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"73": {
"content": "<|im_sp_67|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"74": {
"content": "<|im_sp_68|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"75": {
"content": "<|im_sp_69|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"76": {
"content": "<|im_sp_70|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"77": {
"content": "<|im_sp_71|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"78": {
"content": "<|im_sp_72|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"79": {
"content": "<|im_sp_73|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"80": {
"content": "<|im_sp_74|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"81": {
"content": "<|im_sp_75|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"82": {
"content": "<|im_sp_76|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"83": {
"content": "<|im_sp_77|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"84": {
"content": "<|im_sp_78|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"85": {
"content": "<|im_sp_79|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"86": {
"content": "<|im_sp_80|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"87": {
"content": "<|im_sp_81|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"88": {
"content": "<|im_sp_82|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"89": {
"content": "<|im_sp_83|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"90": {
"content": "<|im_sp_84|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"91": {
"content": "<|im_sp_85|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"92": {
"content": "<|im_sp_86|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"93": {
"content": "<|im_sp_87|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"94": {
"content": "<|im_sp_88|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"95": {
"content": "<|im_sp_89|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"96": {
"content": "<|im_sp_90|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"97": {
"content": "<|im_sp_91|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"98": {
"content": "<|im_sp_92|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"99": {
"content": "<|im_sp_93|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"100": {
"content": "<|im_sp_94|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"101": {
"content": "<|im_sp_95|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"102": {
"content": "<|im_sp_96|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"103": {
"content": "<|im_sp_97|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"104": {
"content": "<|im_sp_98|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"105": {
"content": "<|im_sp_99|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": true,
"special": false
},
"116": {
"content": "\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
}
},
"clean_up_tokenization_spaces": false,
"extra_special_tokens": {},
"max_length": null,
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 1000000000000000019884624838656,
"pad_to_multiple_of": null,
"pad_token": "<pad>",
"pad_token_type_id": 0,
"padding_side": "right",
"tokenizer_class": "PreTrainedTokenizerFast"
}