neyssemhajji16/smollm2-360m-text2sql-sft
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: neyssemhajji16/smollm2-360m-text2sql-sft
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-13 09:53:12 +08:00
commit b1a04626fa
20 changed files with 490750 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.gitattributes vendored Normal file
View File

@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

58
README.md Normal file
View File

@@ -0,0 +1,58 @@
---
base_model: HuggingFaceTB/SmolLM2-360M-Instruct
library_name: transformers
model_name: smollm2-360m-text2sql-sft
tags:
- generated_from_trainer
- trl
- sft
licence: license
---
# Model Card for smollm2-360m-text2sql-sft
This model is a fine-tuned version of [HuggingFaceTB/SmolLM2-360M-Instruct](https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct).
It has been trained using [TRL](https://github.com/huggingface/trl).
## Quick start
```python
from transformers import pipeline
question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
generator = pipeline("text-generation", model="neyssemhajji16/smollm2-360m-text2sql-sft", device="cuda")
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
print(output["generated_text"])
```
## Training procedure
This model was trained with SFT.
### Framework versions
- TRL: 1.6.0
- Transformers: 5.0.0
- Pytorch: 2.10.0+cu128
- Datasets: 4.8.5
- Tokenizers: 0.22.2
## Citations
Cite TRL as:
```bibtex
@software{vonwerra2020trl,
title = {{TRL: Transformers Reinforcement Learning}},
author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
license = {Apache-2.0},
url = {https://github.com/huggingface/trl},
year = {2020}
}
```

6
chat_template.jinja Normal file
View File

@@ -0,0 +1,6 @@
{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
' }}{% endif %}{{'<|im_start|>' + message['role'] + '
' + message['content'] + '<|im_end|>' + '
'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
' }}{% endif %}

40
config.json Normal file
View File

@@ -0,0 +1,40 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"dtype": "float32",
"eos_token_id": 2,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 960,
"initializer_range": 0.02,
"intermediate_size": 2560,
"is_llama_config": true,
"max_position_embeddings": 8192,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 15,
"num_hidden_layers": 32,
"num_key_value_heads": 5,
"pad_token_id": 2,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_interleaved": false,
"rope_parameters": {
"rope_theta": 100000,
"rope_type": "default"
},
"tie_word_embeddings": true,
"transformers.js_config": {
"kv_cache_dtype": {
"fp16": "float16",
"q4f16": "float16"
}
},
"transformers_version": "5.0.0",
"use_cache": false,
"vocab_size": 49152
}

9
generation_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": [
2
],
"pad_token_id": 2,
"transformers_version": "5.0.0"
}

6
last-checkpoint/chat_template.jinja Normal file
View File

@@ -0,0 +1,6 @@
{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
' }}{% endif %}{{'<|im_start|>' + message['role'] + '
' + message['content'] + '<|im_end|>' + '
'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
' }}{% endif %}

40
last-checkpoint/config.json Normal file
View File

@@ -0,0 +1,40 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"dtype": "float32",
"eos_token_id": 2,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 960,
"initializer_range": 0.02,
"intermediate_size": 2560,
"is_llama_config": true,
"max_position_embeddings": 8192,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 15,
"num_hidden_layers": 32,
"num_key_value_heads": 5,
"pad_token_id": 2,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_interleaved": false,
"rope_parameters": {
"rope_theta": 100000,
"rope_type": "default"
},
"tie_word_embeddings": true,
"transformers.js_config": {
"kv_cache_dtype": {
"fp16": "float16",
"q4f16": "float16"
}
},
"transformers_version": "5.0.0",
"use_cache": false,
"vocab_size": 49152
}

9
last-checkpoint/generation_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": [
2
],
"pad_token_id": 2,
"transformers_version": "5.0.0"
}

3
last-checkpoint/model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4bc0f32c9d7a676d7eda9de1c7f20412b6e996fd169f0d29fb7a7a9093644635
size 1447317080

3
last-checkpoint/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3ae97483b28c34b269b788a6c029d1ce0fd24ea9e34fd4fe583a6e182b7c6a01
size 2894817739

3
last-checkpoint/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:236304ae89e49aae8260113165ee63419b9b745f79120014328a7fa31ed79b42
size 14645

3
last-checkpoint/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f758098500b9981733dc8b471e799f37872a9730b7abbb2135fe9f63ceeb15a8
size 1465

244974
last-checkpoint/tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

17
last-checkpoint/tokenizer_config.json Normal file
View File

@@ -0,0 +1,17 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": "<|im_start|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>"
],
"is_local": false,
"model_max_length": 8192,
"pad_token": "<|im_end|>",
"tokenizer_class": "TokenizersBackend",
"unk_token": "<|endoftext|>",
"vocab_size": 49152
}

544
last-checkpoint/trainer_state.json Normal file
View File

@@ -0,0 +1,544 @@
{
"best_global_step": 1200,
"best_metric": 0.10889480262994766,
"best_model_checkpoint": "./sft_output/checkpoint-1200",
"epoch": 2.895489947473284,
"eval_steps": 200,
"global_step": 2000,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"entropy": 0.1292651420040056,
"epoch": 0.07245064299945662,
"grad_norm": 0.5799852013587952,
"learning_rate": 0.00019527255185721178,
"loss": 0.1073553466796875,
"mean_token_accuracy": 0.9656643654406071,
"num_tokens": 115293.0,
"step": 50
},
{
"entropy": 0.14149306831415742,
"epoch": 0.14490128599891325,
"grad_norm": 0.4752981960773468,
"learning_rate": 0.00019044862518089725,
"loss": 0.12214879989624024,
"mean_token_accuracy": 0.9628654922544956,
"num_tokens": 228079.0,
"step": 100
},
{
"entropy": 0.1316419944167137,
"epoch": 0.21735192899836986,
"grad_norm": 0.3439595699310303,
"learning_rate": 0.00018562469850458274,
"loss": 0.11439435958862304,
"mean_token_accuracy": 0.9642146781086922,
"num_tokens": 346451.0,
"step": 150
},
{
"entropy": 0.12753444796428084,
"epoch": 0.2898025719978265,
"grad_norm": 0.3129945695400238,
"learning_rate": 0.0001808007718282682,
"loss": 0.1101201343536377,
"mean_token_accuracy": 0.9657260800898075,
"num_tokens": 463838.0,
"step": 200
},
{
"epoch": 0.2898025719978265,
"eval_entropy": 0.11204168354642802,
"eval_loss": 0.1371564269065857,
"eval_mean_token_accuracy": 0.9661400092059168,
"eval_num_tokens": 463838.0,
"eval_runtime": 135.07,
"eval_samples_per_second": 5.108,
"eval_steps_per_second": 0.644,
"step": 200
},
{
"entropy": 0.12514342403504997,
"epoch": 0.3622532149972831,
"grad_norm": 0.3354629874229431,
"learning_rate": 0.0001759768451519537,
"loss": 0.1064763069152832,
"mean_token_accuracy": 0.9663380682468414,
"num_tokens": 580179.0,
"step": 250
},
{
"entropy": 0.11715295719448476,
"epoch": 0.4347038579967397,
"grad_norm": 0.27767807245254517,
"learning_rate": 0.00017115291847563918,
"loss": 0.10067870140075684,
"mean_token_accuracy": 0.9680409373342991,
"num_tokens": 695580.0,
"step": 300
},
{
"entropy": 0.1172275464516133,
"epoch": 0.5071545009961963,
"grad_norm": 0.22333095967769623,
"learning_rate": 0.00016632899179932467,
"loss": 0.09892839431762696,
"mean_token_accuracy": 0.968158980756998,
"num_tokens": 813169.0,
"step": 350
},
{
"entropy": 0.13554750085109846,
"epoch": 0.579605143995653,
"grad_norm": 0.2716864049434662,
"learning_rate": 0.00016150506512301014,
"loss": 0.12139375686645508,
"mean_token_accuracy": 0.9637367476522922,
"num_tokens": 933136.0,
"step": 400
},
{
"epoch": 0.579605143995653,
"eval_entropy": 0.12780291261686677,
"eval_loss": 0.12689483165740967,
"eval_mean_token_accuracy": 0.9676476744399674,
"eval_num_tokens": 933136.0,
"eval_runtime": 135.0083,
"eval_samples_per_second": 5.111,
"eval_steps_per_second": 0.644,
"step": 400
},
{
"entropy": 0.14845358284655957,
"epoch": 0.6520557869951096,
"grad_norm": 0.2887682616710663,
"learning_rate": 0.0001566811384466956,
"loss": 0.12956516265869142,
"mean_token_accuracy": 0.961767661422491,
"num_tokens": 1046558.0,
"step": 450
},
{
"entropy": 0.1415072442078963,
"epoch": 0.7245064299945662,
"grad_norm": 0.26643216609954834,
"learning_rate": 0.0001518572117703811,
"loss": 0.12267910957336425,
"mean_token_accuracy": 0.9633507201075554,
"num_tokens": 1163902.0,
"step": 500
},
{
"entropy": 0.1404953494342044,
"epoch": 0.7969570729940229,
"grad_norm": 0.2829984128475189,
"learning_rate": 0.00014703328509406657,
"loss": 0.12197186470031739,
"mean_token_accuracy": 0.9628035886585713,
"num_tokens": 1276260.0,
"step": 550
},
{
"entropy": 0.13520956406835466,
"epoch": 0.8694077159934794,
"grad_norm": 0.2343612164258957,
"learning_rate": 0.00014220935841775204,
"loss": 0.11497893333435058,
"mean_token_accuracy": 0.964199658036232,
"num_tokens": 1388505.0,
"step": 600
},
{
"epoch": 0.8694077159934794,
"eval_entropy": 0.11246106024274881,
"eval_loss": 0.11639242619276047,
"eval_mean_token_accuracy": 0.9700561001383025,
"eval_num_tokens": 1388505.0,
"eval_runtime": 134.8927,
"eval_samples_per_second": 5.115,
"eval_steps_per_second": 0.645,
"step": 600
},
{
"entropy": 0.1313706226553768,
"epoch": 0.941858358992936,
"grad_norm": 0.2563211917877197,
"learning_rate": 0.00013738543174143754,
"loss": 0.1108535385131836,
"mean_token_accuracy": 0.965393750667572,
"num_tokens": 1506542.0,
"step": 650
},
{
"entropy": 0.12733429687870215,
"epoch": 1.0130411157399022,
"grad_norm": 0.20515012741088867,
"learning_rate": 0.000132561505065123,
"loss": 0.10263432502746582,
"mean_token_accuracy": 0.9680144694318602,
"num_tokens": 1625554.0,
"step": 700
},
{
"entropy": 0.09378954898566008,
"epoch": 1.0854917587393589,
"grad_norm": 0.1810425966978073,
"learning_rate": 0.0001277375783888085,
"loss": 0.07341335773468018,
"mean_token_accuracy": 0.9737974387407303,
"num_tokens": 1742352.0,
"step": 750
},
{
"entropy": 0.08947869850322604,
"epoch": 1.1579424017388154,
"grad_norm": 0.2107703685760498,
"learning_rate": 0.00012291365171249397,
"loss": 0.07443580627441407,
"mean_token_accuracy": 0.974107117652893,
"num_tokens": 1859066.0,
"step": 800
},
{
"epoch": 1.1579424017388154,
"eval_entropy": 0.08398804492477713,
"eval_loss": 0.11319845169782639,
"eval_mean_token_accuracy": 0.9715756554713194,
"eval_num_tokens": 1859066.0,
"eval_runtime": 134.9594,
"eval_samples_per_second": 5.113,
"eval_steps_per_second": 0.645,
"step": 800
},
{
"entropy": 0.09298701203661039,
"epoch": 1.230393044738272,
"grad_norm": 0.2118096500635147,
"learning_rate": 0.00011808972503617945,
"loss": 0.07880696296691894,
"mean_token_accuracy": 0.9729460710287094,
"num_tokens": 1974287.0,
"step": 850
},
{
"entropy": 0.09084601077716797,
"epoch": 1.3028436877377287,
"grad_norm": 0.14472977817058563,
"learning_rate": 0.00011326579835986493,
"loss": 0.07501977443695068,
"mean_token_accuracy": 0.973245297074318,
"num_tokens": 2089089.0,
"step": 900
},
{
"entropy": 0.09346557535696774,
"epoch": 1.3752943307371852,
"grad_norm": 0.5793337225914001,
"learning_rate": 0.00010844187168355043,
"loss": 0.07840880870819092,
"mean_token_accuracy": 0.9730692146718503,
"num_tokens": 2203786.0,
"step": 950
},
{
"entropy": 0.09212197210872546,
"epoch": 1.447744973736642,
"grad_norm": 0.22646810114383698,
"learning_rate": 0.0001036179450072359,
"loss": 0.07762556552886962,
"mean_token_accuracy": 0.9733995048701763,
"num_tokens": 2318554.0,
"step": 1000
},
{
"epoch": 1.447744973736642,
"eval_entropy": 0.08479796379975889,
"eval_loss": 0.11282521486282349,
"eval_mean_token_accuracy": 0.9713243239227383,
"eval_num_tokens": 2318554.0,
"eval_runtime": 135.0253,
"eval_samples_per_second": 5.11,
"eval_steps_per_second": 0.644,
"step": 1000
},
{
"entropy": 0.09136153725441545,
"epoch": 1.5201956167360984,
"grad_norm": 0.24745002388954163,
"learning_rate": 9.879401833092138e-05,
"loss": 0.0756511640548706,
"mean_token_accuracy": 0.9727346667647362,
"num_tokens": 2434178.0,
"step": 1050
},
{
"entropy": 0.08874542655656115,
"epoch": 1.592646259735555,
"grad_norm": 0.16458512842655182,
"learning_rate": 9.397009165460685e-05,
"loss": 0.0733512830734253,
"mean_token_accuracy": 0.9746969762444496,
"num_tokens": 2551600.0,
"step": 1100
},
{
"entropy": 0.08679104059003294,
"epoch": 1.6650969027350118,
"grad_norm": 0.2363063395023346,
"learning_rate": 8.914616497829233e-05,
"loss": 0.07282801628112794,
"mean_token_accuracy": 0.9748657821118831,
"num_tokens": 2670568.0,
"step": 1150
},
{
"entropy": 0.08530640854965896,
"epoch": 1.7375475457344685,
"grad_norm": 0.3205758035182953,
"learning_rate": 8.432223830197782e-05,
"loss": 0.07208527088165283,
"mean_token_accuracy": 0.9753711603581905,
"num_tokens": 2790395.0,
"step": 1200
},
{
"epoch": 1.7375475457344685,
"eval_entropy": 0.08271760874609838,
"eval_loss": 0.10889480262994766,
"eval_mean_token_accuracy": 0.9722844334854477,
"eval_num_tokens": 2790395.0,
"eval_runtime": 135.0173,
"eval_samples_per_second": 5.11,
"eval_steps_per_second": 0.644,
"step": 1200
},
{
"entropy": 0.08853163382969796,
"epoch": 1.8099981887339252,
"grad_norm": 0.2860631048679352,
"learning_rate": 7.949831162566329e-05,
"loss": 0.07203876972198486,
"mean_token_accuracy": 0.9745123912394047,
"num_tokens": 2906250.0,
"step": 1250
},
{
"entropy": 0.09159044577972963,
"epoch": 1.8824488317333816,
"grad_norm": 0.22217783331871033,
"learning_rate": 7.467438494934877e-05,
"loss": 0.0765086030960083,
"mean_token_accuracy": 0.9737336048483849,
"num_tokens": 3020888.0,
"step": 1300
},
{
"entropy": 0.08859600902302191,
"epoch": 1.954899474732838,
"grad_norm": 0.35960590839385986,
"learning_rate": 6.985045827303426e-05,
"loss": 0.07379885196685791,
"mean_token_accuracy": 0.9746835605800152,
"num_tokens": 3134684.0,
"step": 1350
},
{
"entropy": 0.08423688769122496,
"epoch": 2.0260822314798044,
"grad_norm": 0.1249847561120987,
"learning_rate": 6.502653159671973e-05,
"loss": 0.06818146228790284,
"mean_token_accuracy": 0.9766376161089987,
"num_tokens": 3245618.0,
"step": 1400
},
{
"epoch": 2.0260822314798044,
"eval_entropy": 0.07047315447152346,
"eval_loss": 0.11015637964010239,
"eval_mean_token_accuracy": 0.9731149563844177,
"eval_num_tokens": 3245618.0,
"eval_runtime": 134.9477,
"eval_samples_per_second": 5.113,
"eval_steps_per_second": 0.645,
"step": 1400
},
{
"entropy": 0.06371946840547026,
"epoch": 2.098532874479261,
"grad_norm": 0.16305267810821533,
"learning_rate": 6.0202604920405214e-05,
"loss": 0.05102997303009033,
"mean_token_accuracy": 0.9815227979421616,
"num_tokens": 3362965.0,
"step": 1450
},
{
"entropy": 0.0659942712332122,
"epoch": 2.1709835174787178,
"grad_norm": 0.1745850145816803,
"learning_rate": 5.537867824409069e-05,
"loss": 0.05241162776947021,
"mean_token_accuracy": 0.9807419489324093,
"num_tokens": 3475907.0,
"step": 1500
},
{
"entropy": 0.06338218597928062,
"epoch": 2.2434341604781745,
"grad_norm": 0.19584746658802032,
"learning_rate": 5.055475156777617e-05,
"loss": 0.051362857818603516,
"mean_token_accuracy": 0.9814340886473656,
"num_tokens": 3593538.0,
"step": 1550
},
{
"entropy": 0.06354268383933231,
"epoch": 2.3158848034776307,
"grad_norm": 0.19582241773605347,
"learning_rate": 4.573082489146165e-05,
"loss": 0.052291646003723144,
"mean_token_accuracy": 0.981376932412386,
"num_tokens": 3707445.0,
"step": 1600
},
{
"epoch": 2.3158848034776307,
"eval_entropy": 0.06323016272194085,
"eval_loss": 0.11573585867881775,
"eval_mean_token_accuracy": 0.973355067872453,
"eval_num_tokens": 3707445.0,
"eval_runtime": 135.1558,
"eval_samples_per_second": 5.105,
"eval_steps_per_second": 0.644,
"step": 1600
},
{
"entropy": 0.06490354960318655,
"epoch": 2.3883354464770874,
"grad_norm": 0.17022456228733063,
"learning_rate": 4.0906898215147135e-05,
"loss": 0.054670772552490234,
"mean_token_accuracy": 0.980761057138443,
"num_tokens": 3820475.0,
"step": 1650
},
{
"entropy": 0.06598329161293805,
"epoch": 2.460786089476544,
"grad_norm": 0.20498181879520416,
"learning_rate": 3.608297153883261e-05,
"loss": 0.05304833889007568,
"mean_token_accuracy": 0.9809189030528068,
"num_tokens": 3933729.0,
"step": 1700
},
{
"entropy": 0.06734917678404599,
"epoch": 2.533236732476001,
"grad_norm": 0.1891096830368042,
"learning_rate": 3.1259044862518086e-05,
"loss": 0.0533873462677002,
"mean_token_accuracy": 0.980241654664278,
"num_tokens": 4047398.0,
"step": 1750
},
{
"entropy": 0.05935755535727367,
"epoch": 2.6056873754754575,
"grad_norm": 0.17223551869392395,
"learning_rate": 2.643511818620357e-05,
"loss": 0.04701524257659912,
"mean_token_accuracy": 0.9830030533671379,
"num_tokens": 4168983.0,
"step": 1800
},
{
"epoch": 2.6056873754754575,
"eval_entropy": 0.06277676872994709,
"eval_loss": 0.11459118127822876,
"eval_mean_token_accuracy": 0.9735316360133818,
"eval_num_tokens": 4168983.0,
"eval_runtime": 135.0545,
"eval_samples_per_second": 5.109,
"eval_steps_per_second": 0.644,
"step": 1800
},
{
"entropy": 0.06294791254680604,
"epoch": 2.6781380184749137,
"grad_norm": 0.21268633008003235,
"learning_rate": 2.161119150988905e-05,
"loss": 0.05056027889251709,
"mean_token_accuracy": 0.9813353289663792,
"num_tokens": 4286132.0,
"step": 1850
},
{
"entropy": 0.06273156431736425,
"epoch": 2.7505886614743704,
"grad_norm": 0.166688933968544,
"learning_rate": 1.678726483357453e-05,
"loss": 0.05039352893829346,
"mean_token_accuracy": 0.9812411935627461,
"num_tokens": 4403199.0,
"step": 1900
},
{
"entropy": 0.06210695546586067,
"epoch": 2.823039304473827,
"grad_norm": 0.20107661187648773,
"learning_rate": 1.1963338157260012e-05,
"loss": 0.050140600204467776,
"mean_token_accuracy": 0.9815662078559398,
"num_tokens": 4519243.0,
"step": 1950
},
{
"entropy": 0.06227384155383334,
"epoch": 2.895489947473284,
"grad_norm": 0.1902814358472824,
"learning_rate": 7.13941148094549e-06,
"loss": 0.048079490661621094,
"mean_token_accuracy": 0.9821359276771545,
"num_tokens": 4636178.0,
"step": 2000
},
{
"epoch": 2.895489947473284,
"eval_entropy": 0.06232287272297103,
"eval_loss": 0.11387787759304047,
"eval_mean_token_accuracy": 0.9737936319976017,
"eval_num_tokens": 4636178.0,
"eval_runtime": 135.2272,
"eval_samples_per_second": 5.103,
"eval_steps_per_second": 0.643,
"step": 2000
}
],
"logging_steps": 50,
"max_steps": 2073,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 8752228753593600.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}

3
last-checkpoint/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b7e6e32b7b4c39db2ce22a63a6c9e57c801af7a944a43cef486fcdd7f354c897
size 5649

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3f2eac6f883ad04b7c5d8b5b2a77ca3f7d0abee21fed0da24f7d5a0c03582d81
size 1447317080

244974
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

17
tokenizer_config.json Normal file
View File

@@ -0,0 +1,17 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": "<|im_start|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>"
],
"is_local": false,
"model_max_length": 8192,
"pad_token": "<|im_end|>",
"tokenizer_class": "TokenizersBackend",
"unk_token": "<|endoftext|>",
"vocab_size": 49152
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b7e6e32b7b4c39db2ce22a63a6c9e57c801af7a944a43cef486fcdd7f354c897
size 5649