初始化项目,由ModelHub XC社区提供模型
Model: pramodkoujalagi/SmolLM2-360M-Instruct-Text-2-JSON Source: Original Platform
This commit is contained in:
39
.gitattributes
vendored
Normal file
39
.gitattributes
vendored
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
models/SmolLM2-360M-Instruct-Text-2-JSON.F16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
|
models/SmolLM2-360M-Instruct-Text-2-JSON.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
|
SmolLM2-360M-Instruct-Text-2-JSON.F16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
|
SmolLM2-360M-Instruct-Text-2-JSON.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
167
README.md
Normal file
167
README.md
Normal file
@@ -0,0 +1,167 @@
|
|||||||
|
---
|
||||||
|
license: apache-2.0
|
||||||
|
language:
|
||||||
|
- en
|
||||||
|
metrics:
|
||||||
|
- accuracy
|
||||||
|
- perplexity
|
||||||
|
base_model:
|
||||||
|
- unsloth/SmolLM2-360M-Instruct-bnb-4bit
|
||||||
|
tags:
|
||||||
|
- transformers
|
||||||
|
- unsloth
|
||||||
|
- trl
|
||||||
|
- llama
|
||||||
|
- text-to-json
|
||||||
|
- Text-2-JSON
|
||||||
|
- Text-To-JSON
|
||||||
|
- calendar-parsing
|
||||||
|
- entity-extraction
|
||||||
|
- json-structured-output
|
||||||
|
- event-scheduling
|
||||||
|
- 4-bit
|
||||||
|
library_name: transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
Developed by: [Pramod Koujalagi](https://www.pramodkoujalagi.com)
|
||||||
|
|
||||||
|
# SmolLM2-360M-Instruct-Text-2-JSON
|
||||||
|
A fine-tuned version of SmolLM2-360M-Instruct-bnb-4bit specialized for parsing unstructured calendar event requests into structured JSON data.
|
||||||
|
|
||||||
|
## Model Description
|
||||||
|
This model is fine-tuned on SmolLM2-360M-Instruct-bnb-4bit using QLoRA to extract structured calendar event information from natural language text. It identifies and structures key scheduling entities like action, date, time, attendees, location, duration, recurrence, and notes.
|
||||||
|
|
||||||
|
|
||||||
|
### 📦 Example Usage
|
||||||
|
|
||||||
|
You can use the `SmolLM2-360M-Instruct-Text-2-JSON` model to parse natural language event descriptions into structured JSON format.
|
||||||
|
|
||||||
|
```python
|
||||||
|
import torch
|
||||||
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||||
|
import json
|
||||||
|
|
||||||
|
# Load model and tokenizer
|
||||||
|
model_name = "pramodkoujalagi/SmolLM2-360M-Instruct-Text-2-JSON"
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(model_name)
|
||||||
|
|
||||||
|
def parse_calendar_event(text):
|
||||||
|
# Format the prompt
|
||||||
|
formatted_prompt = f"""<|im_start|>user
|
||||||
|
Extract the relevant event information from this text and organize it into a JSON structure with fields for action, date, time, attendees, location, duration, recurrence, and notes. If a field is not present, return null for that field.
|
||||||
|
|
||||||
|
Text: {text}
|
||||||
|
<|im_end|>
|
||||||
|
<|im_start|>assistant
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Generate response
|
||||||
|
inputs = tokenizer(formatted_prompt, return_tensors="pt").to(model.device)
|
||||||
|
with torch.no_grad():
|
||||||
|
outputs = model.generate(
|
||||||
|
**inputs,
|
||||||
|
max_new_tokens=512,
|
||||||
|
do_sample=True,
|
||||||
|
temperature=0.1,
|
||||||
|
top_p=0.95,
|
||||||
|
pad_token_id=tokenizer.eos_token_id
|
||||||
|
)
|
||||||
|
|
||||||
|
# Process response
|
||||||
|
output_text = tokenizer.decode(outputs[0], skip_special_tokens=False)
|
||||||
|
response = output_text.split("<|im_start|>assistant\n")[1].split("<|im_end|>")[0].strip()
|
||||||
|
|
||||||
|
# Return formatted JSON
|
||||||
|
parsed_json = json.loads(response)
|
||||||
|
return json.dumps(parsed_json, indent=2)
|
||||||
|
|
||||||
|
# Example input
|
||||||
|
event_text = "Plan an exhibition walkthrough on 15th, April 2028 at 3 PM with Harper, Grace, and Alex in the art gallery for 1 hour, bring bag."
|
||||||
|
|
||||||
|
# Output
|
||||||
|
print("Prompt:")
|
||||||
|
print(event_text)
|
||||||
|
print("\nModel Output:")
|
||||||
|
print(parse_calendar_event(event_text))
|
||||||
|
```
|
||||||
|
Output
|
||||||
|
```
|
||||||
|
Prompt:
|
||||||
|
Plan an exhibition walkthrough on 15th, April 2028 at 3 PM with Harper, Grace, and Alex in the art gallery for 1 hour, bring bag.
|
||||||
|
|
||||||
|
Model Output:
|
||||||
|
{
|
||||||
|
"action": "Plan an exhibition walkthrough",
|
||||||
|
"date": "15/04/2028",
|
||||||
|
"time": "3:00 PM",
|
||||||
|
"attendees": [
|
||||||
|
"Harper",
|
||||||
|
"Grace",
|
||||||
|
"Alex"
|
||||||
|
],
|
||||||
|
"location": "art gallery",
|
||||||
|
"duration": "1 hour",
|
||||||
|
"recurrence": null,
|
||||||
|
"notes": "Bring bag"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
<!--
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```
|
||||||
|
Input: "Plan an exhibition walkthrough on 15th, April 2028 at 3 PM with Harper, Grace, and Alex in the art gallery for 1 hour, bring bag."
|
||||||
|
```
|
||||||
|
```
|
||||||
|
Output: {
|
||||||
|
"action": "Plan an exhibition walkthrough",
|
||||||
|
"date": "15/04/2028",
|
||||||
|
"time": "3:00 PM",
|
||||||
|
"attendees": [
|
||||||
|
"Harper",
|
||||||
|
"Grace",
|
||||||
|
"Alex"
|
||||||
|
],
|
||||||
|
"location": "art gallery",
|
||||||
|
"duration": "1 hour",
|
||||||
|
"recurrence": null,
|
||||||
|
"notes": Bring bag
|
||||||
|
}
|
||||||
|
``` -->
|
||||||
|
|
||||||
|
### Resources for more information:
|
||||||
|
- GitHub Repository: [SmolLM2-360M-Instruct-Text-2-JSON](https://github.com/pramodkoujalagi/SmolLM2-360M-Instruct-Text-2-JSON)
|
||||||
|
- Base Model: [SmolLM2-360M-Instruct-bnb-4bit](https://huggingface.co/unsloth/SmolLM2-360M-Instruct-bnb-4bit) (Derived from HuggingfaceTB/SmolLM2-360)
|
||||||
|
|
||||||
|
## Use Cases
|
||||||
|
- Calendar application integration
|
||||||
|
- Personal assistant scheduling systems
|
||||||
|
- Meeting summarization tools
|
||||||
|
- Email processing for event extraction
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
The model was trained on a custom dataset consisting of 1,149 examples (1,034 training, 115 validation) of natural language event descriptions paired with structured JSON outputs. The dataset includes a wide variety of event types, date/time formats, and varying combinations of fields.
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
- **Fine-tuning method**: QLoRA (Quantized Low-Rank Adaptation)
|
||||||
|
- **LoRA configuration**:
|
||||||
|
- Rank: 64
|
||||||
|
- Alpha: 32
|
||||||
|
- Target modules: All key model components
|
||||||
|
- Rank-stabilized LoRA: Enabled
|
||||||
|
- **Training hyperparameters**:
|
||||||
|
- Batch size: 8 (2 per device × 4 gradient accumulation steps)
|
||||||
|
- Learning rate: 2e-4 with cosine scheduler
|
||||||
|
- Epochs: 3
|
||||||
|
- Weight decay: 0.01
|
||||||
|
- Optimizer: AdamW (8-bit)
|
||||||
|
- Gradient checkpointing: Enabled
|
||||||
|
- **Training time**: ~15 minutes
|
||||||
|
- **Hardware used**: [T4 GPU](https://www.nvidia.com/en-in/data-center/tesla-t4/)
|
||||||
|
|
||||||
|
### Training Metrics
|
||||||
|
- Final training loss:
|
||||||
|
- Final validation loss:
|
||||||
|
- Validation perplexity: 1.2091
|
||||||
3
SmolLM2-360M-Instruct-Text-2-JSON.F16.gguf
Normal file
3
SmolLM2-360M-Instruct-Text-2-JSON.F16.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e9652a4342910d5aca4706dbb07446e61847d9dc96eba2dd316d5fd1901cedbb
|
||||||
|
size 819929312
|
||||||
3
SmolLM2-360M-Instruct-Text-2-JSON.Q4_K_M.gguf
Normal file
3
SmolLM2-360M-Instruct-Text-2-JSON.Q4_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a845642046e9a48472bef3f868735e4ae5daf11bb9949696fd60e8e969597a74
|
||||||
|
size 303032448
|
||||||
3
adapter/adapter_model.safetensors
Normal file
3
adapter/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f0b30cd76ebffc41c47ea6bd35b0b1d611e4a46fafed7cd8217b4d507c4b2ac1
|
||||||
|
size 258276120
|
||||||
40
adapter_config.json
Normal file
40
adapter_config.json
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
{
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "unsloth/SmolLM2-360M-Instruct-bnb-4bit",
|
||||||
|
"bias": "none",
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 32,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": [
|
||||||
|
"embed_tokens",
|
||||||
|
"lm_head"
|
||||||
|
],
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"v_proj",
|
||||||
|
"k_proj",
|
||||||
|
"up_proj",
|
||||||
|
"gate_proj",
|
||||||
|
"down_proj"
|
||||||
|
],
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"use_dora": false,
|
||||||
|
"use_rslora": true
|
||||||
|
}
|
||||||
3
adapter_model.safetensors
Normal file
3
adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f0b30cd76ebffc41c47ea6bd35b0b1d611e4a46fafed7cd8217b4d507c4b2ac1
|
||||||
|
size 258276120
|
||||||
3
added_tokens.json
Normal file
3
added_tokens.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"<|PAD_TOKEN|>": 49152
|
||||||
|
}
|
||||||
39
config.json
Normal file
39
config.json
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 1,
|
||||||
|
"eos_token_id": 2,
|
||||||
|
"head_dim": 64,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 960,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 2560,
|
||||||
|
"is_llama_config": true,
|
||||||
|
"max_position_embeddings": 8192,
|
||||||
|
"mlp_bias": false,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 15,
|
||||||
|
"num_hidden_layers": 32,
|
||||||
|
"num_key_value_heads": 5,
|
||||||
|
"pad_token_id": 49152,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-05,
|
||||||
|
"rope_interleaved": false,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 100000,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"torch_dtype": "float16",
|
||||||
|
"transformers.js_config": {
|
||||||
|
"kv_cache_dtype": {
|
||||||
|
"fp16": "float16",
|
||||||
|
"q4f16": "float16"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"transformers_version": "4.51.1",
|
||||||
|
"unsloth_version": "2025.3.19",
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 49153
|
||||||
|
}
|
||||||
8
generation_config.json
Normal file
8
generation_config.json
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 1,
|
||||||
|
"eos_token_id": 2,
|
||||||
|
"max_length": 8192,
|
||||||
|
"pad_token_id": 49152,
|
||||||
|
"transformers_version": "4.51.1"
|
||||||
|
}
|
||||||
48901
merges.txt
Normal file
48901
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
models/SmolLM2-360M-Instruct-Text-2-JSON.F16.gguf
Normal file
3
models/SmolLM2-360M-Instruct-Text-2-JSON.F16.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e9652a4342910d5aca4706dbb07446e61847d9dc96eba2dd316d5fd1901cedbb
|
||||||
|
size 819929312
|
||||||
3
models/SmolLM2-360M-Instruct-Text-2-JSON.Q4_K_M.gguf
Normal file
3
models/SmolLM2-360M-Instruct-Text-2-JSON.Q4_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a845642046e9a48472bef3f868735e4ae5daf11bb9949696fd60e8e969597a74
|
||||||
|
size 303032448
|
||||||
3
models/pytorch_model.bin
Normal file
3
models/pytorch_model.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:3b181cd4a2ea77fe2876230383d780b758d6d6d31a6bce72b2f72010e7089d71
|
||||||
|
size 818116310
|
||||||
3
pytorch_model.bin
Normal file
3
pytorch_model.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:3b181cd4a2ea77fe2876230383d780b758d6d6d31a6bce72b2f72010e7089d71
|
||||||
|
size 818116310
|
||||||
28
special_tokens_map.json
Normal file
28
special_tokens_map.json
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>"
|
||||||
|
],
|
||||||
|
"bos_token": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "<|PAD_TOKEN|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"unk_token": "�"
|
||||||
|
}
|
||||||
244967
tokenizer.json
Normal file
244967
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
172
tokenizer_config.json
Normal file
172
tokenizer_config.json
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"0": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"3": {
|
||||||
|
"content": "<repo_name>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"4": {
|
||||||
|
"content": "<reponame>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"5": {
|
||||||
|
"content": "<file_sep>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"6": {
|
||||||
|
"content": "<filename>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"7": {
|
||||||
|
"content": "<gh_stars>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"8": {
|
||||||
|
"content": "<issue_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"9": {
|
||||||
|
"content": "<issue_comment>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"10": {
|
||||||
|
"content": "<issue_closed>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"11": {
|
||||||
|
"content": "<jupyter_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"12": {
|
||||||
|
"content": "<jupyter_text>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"13": {
|
||||||
|
"content": "<jupyter_code>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"14": {
|
||||||
|
"content": "<jupyter_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"15": {
|
||||||
|
"content": "<jupyter_script>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"16": {
|
||||||
|
"content": "<empty_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"24211": {
|
||||||
|
"content": "�",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"49152": {
|
||||||
|
"content": "<|PAD_TOKEN|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>"
|
||||||
|
],
|
||||||
|
"bos_token": "<|im_start|>",
|
||||||
|
"chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|im_end|>",
|
||||||
|
"extra_special_tokens": {},
|
||||||
|
"model_max_length": 8192,
|
||||||
|
"pad_token": "<|PAD_TOKEN|>",
|
||||||
|
"padding_side": "left",
|
||||||
|
"tokenizer_class": "GPT2Tokenizer",
|
||||||
|
"unk_token": "�",
|
||||||
|
"vocab_size": 49152
|
||||||
|
}
|
||||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user