alexredna/TinyLlama-1.1B-Chat-v1.0-reasoning-v2
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: alexredna/TinyLlama-1.1B-Chat-v1.0-reasoning-v2
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-14 02:06:09 +08:00
commit ae5ef8740f
21 changed files with 754 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.gitattributes vendored Normal file
View File

@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

67
README.md Normal file
View File

@@ -0,0 +1,67 @@
---
license: apache-2.0
base_model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
tags:
- trl
- sft
- generated_from_trainer
datasets:
- generator
model-index:
- name: TinyLlama-1.1B-Chat-v1.0-reasoning-v2
results: []
---
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->
# TinyLlama-1.1B-Chat-v1.0-reasoning-v2
This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) on the generator dataset.
It achieves the following results on the evaluation set:
- Loss: 1.0186
## Model description
More information needed
## Intended uses & limitations
More information needed
## Training and evaluation data
More information needed
## Training procedure
### Training hyperparameters
The following hyperparameters were used during training:
- learning_rate: 4e-05
- train_batch_size: 10
- eval_batch_size: 4
- seed: 42
- distributed_type: multi-GPU
- gradient_accumulation_steps: 10
- total_train_batch_size: 100
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: cosine
- num_epochs: 4
### Training results
| Training Loss | Epoch | Step | Validation Loss |
|:-------------:|:-----:|:----:|:---------------:|
| 1.0456 | 1.0 | 347 | 1.0360 |
| 0.9714 | 2.0 | 695 | 1.0180 |
| 0.9335 | 3.0 | 1042 | 1.0176 |
| 0.9348 | 3.99 | 1388 | 1.0186 |
### Framework versions
- Transformers 4.36.2
- Pytorch 2.1.0+cu118
- Datasets 2.14.6
- Tokenizers 0.15.0

13
all_results.json Normal file
View File

@@ -0,0 +1,13 @@
{
"epoch": 3.99,
"eval_loss": 1.0186352729797363,
"eval_runtime": 19.987,
"eval_samples": 1771,
"eval_samples_per_second": 13.959,
"eval_steps_per_second": 3.502,
"train_loss": 0.9866050820185747,
"train_runtime": 37340.4665,
"train_samples": 195052,
"train_samples_per_second": 3.722,
"train_steps_per_second": 0.037
}

28
config.json Normal file
View File

@@ -0,0 +1,28 @@
{
"_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 5632,
"max_position_embeddings": 2048,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 22,
"num_key_value_heads": 4,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.36.2",
"use_cache": false,
"vocab_size": 32000
}

8
eval_results.json Normal file
View File

@@ -0,0 +1,8 @@
{
"epoch": 3.99,
"eval_loss": 1.0186352729797363,
"eval_runtime": 19.987,
"eval_samples": 1771,
"eval_samples_per_second": 13.959,
"eval_steps_per_second": 3.502
}

7
generation_config.json Normal file
View File

@@ -0,0 +1,7 @@
{
"bos_token_id": 1,
"eos_token_id": 2,
"max_length": 2048,
"pad_token_id": 0,
"transformers_version": "4.36.2"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:952296589bea8d6e696a52fccedaab119430294947b362b5b2fb9a4fc2842f08
size 2200119864

3
runs/Jan05_00-15-37_ml-xtx-machine/events.out.tfevents.1704410561.ml-xtx-machine.23653.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:16ac64c56655a64247630d949b05e442bb20c1a72807094775e7e49585a9677c
size 4880

3
runs/Jan05_13-41-47_ml-xtx-machine/events.out.tfevents.1704459785.ml-xtx-machine.10909.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:89adeae48e15b73128c20ea892493dc6ffb41d8fa317bfd81cf0257020cef718
size 5050

3
runs/Jan05_19-28-09_baef1a11cdb4/events.out.tfevents.1704483814.baef1a11cdb4.2661.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:efa5d6da1d5a8cb369581f24d5b404f69da3db3581e98c8572a84e6f483aff17
size 4754

3
runs/Jan05_19-51-11_baef1a11cdb4/events.out.tfevents.1704485197.baef1a11cdb4.3085.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:84a870c24f7e63ea3d19a0204968d5443b2b62a5d83013fa75e977c5d219f0ec
size 4600

3
runs/Jan05_20-15-12_baef1a11cdb4/events.out.tfevents.1704486576.baef1a11cdb4.3546.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b167c8ec4f54ca99c4730377fbf031b6cc4766f3dd2c3d4639c8b8cc03c2bbbf
size 5992

3
runs/Jan05_20-18-13_ml-xtx-machine/events.out.tfevents.1704483528.ml-xtx-machine.7823.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d446f7e0947d56abbe80f4989d91058e34c900b731aeee2867402d8b0f06bb6f
size 4896

3
runs/Jan05_21-55-57_12c0322bb846/events.out.tfevents.1704492732.12c0322bb846.2137.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5de6f9f6ecc4a43d0504931b3847421007f877901312069b22ac511fd9e8581c
size 16988

3
runs/Jan05_21-55-57_12c0322bb846/events.out.tfevents.1704530092.12c0322bb846.2137.1 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:342e167a1a34ad3f8f6bf2009bb137b686c9cec915b35284b5fa25946ed64100
size 359

30
special_tokens_map.json Normal file
View File

@@ -0,0 +1,30 @@
{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}

BIN
tokenizer.model (Stored with Git LFS) Normal file
View File

Binary file not shown.

43
tokenizer_config.json Normal file
View File

@@ -0,0 +1,43 @@
{
"add_bos_token": true,
"add_eos_token": false,
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<s>",
"chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
"clean_up_tokenization_spaces": false,
"eos_token": "</s>",
"legacy": false,
"model_max_length": 2048,
"pad_token": "</s>",
"padding_side": "right",
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "LlamaTokenizer",
"unk_token": "<unk>",
"use_default_system_prompt": false
}

8
train_results.json Normal file
View File

@@ -0,0 +1,8 @@
{
"epoch": 3.99,
"train_loss": 0.9866050820185747,
"train_runtime": 37340.4665,
"train_samples": 195052,
"train_samples_per_second": 3.722,
"train_steps_per_second": 0.037
}

482
trainer_state.json Normal file
View File

@@ -0,0 +1,482 @@
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 3.9942446043165467,
"eval_steps": 500,
"global_step": 1388,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 3.999994877043978e-05,
"loss": 1.5749,
"step": 1
},
{
"epoch": 0.06,
"learning_rate": 3.997951166621575e-05,
"loss": 1.2447,
"step": 20
},
{
"epoch": 0.12,
"learning_rate": 3.9918088642045126e-05,
"loss": 1.1636,
"step": 40
},
{
"epoch": 0.17,
"learning_rate": 3.981585677303025e-05,
"loss": 1.1379,
"step": 60
},
{
"epoch": 0.23,
"learning_rate": 3.967302551523671e-05,
"loss": 1.1114,
"step": 80
},
{
"epoch": 0.29,
"learning_rate": 3.948988750611294e-05,
"loss": 1.1086,
"step": 100
},
{
"epoch": 0.35,
"learning_rate": 3.9266817964924905e-05,
"loss": 1.0915,
"step": 120
},
{
"epoch": 0.4,
"learning_rate": 3.900427392399429e-05,
"loss": 1.075,
"step": 140
},
{
"epoch": 0.46,
"learning_rate": 3.870279329231546e-05,
"loss": 1.0875,
"step": 160
},
{
"epoch": 0.52,
"learning_rate": 3.836299375346956e-05,
"loss": 1.0696,
"step": 180
},
{
"epoch": 0.58,
"learning_rate": 3.798557150009373e-05,
"loss": 1.0614,
"step": 200
},
{
"epoch": 0.63,
"learning_rate": 3.757129980749847e-05,
"loss": 1.0638,
"step": 220
},
{
"epoch": 0.69,
"learning_rate": 3.712102744935529e-05,
"loss": 1.0545,
"step": 240
},
{
"epoch": 0.75,
"learning_rate": 3.6635676958700946e-05,
"loss": 1.0508,
"step": 260
},
{
"epoch": 0.81,
"learning_rate": 3.611624273782092e-05,
"loss": 1.0566,
"step": 280
},
{
"epoch": 0.86,
"learning_rate": 3.556378902088484e-05,
"loss": 1.0577,
"step": 300
},
{
"epoch": 0.92,
"learning_rate": 3.4979447693508e-05,
"loss": 1.0428,
"step": 320
},
{
"epoch": 0.98,
"learning_rate": 3.436441597370635e-05,
"loss": 1.0456,
"step": 340
},
{
"epoch": 1.0,
"eval_loss": 1.035969614982605,
"eval_runtime": 20.059,
"eval_samples_per_second": 13.909,
"eval_steps_per_second": 3.49,
"step": 347
},
{
"epoch": 1.04,
"learning_rate": 3.371995395899618e-05,
"loss": 1.0082,
"step": 360
},
{
"epoch": 1.09,
"learning_rate": 3.304738204466437e-05,
"loss": 0.9889,
"step": 380
},
{
"epoch": 1.15,
"learning_rate": 3.234807821849838e-05,
"loss": 0.9786,
"step": 400
},
{
"epoch": 1.21,
"learning_rate": 3.162347523751894e-05,
"loss": 0.9881,
"step": 420
},
{
"epoch": 1.27,
"learning_rate": 3.0875057692499566e-05,
"loss": 0.9747,
"step": 440
},
{
"epoch": 1.32,
"learning_rate": 3.0104358966287503e-05,
"loss": 0.9842,
"step": 460
},
{
"epoch": 1.38,
"learning_rate": 2.9312958092157724e-05,
"loss": 0.9846,
"step": 480
},
{
"epoch": 1.44,
"learning_rate": 2.850247651863686e-05,
"loss": 0.9801,
"step": 500
},
{
"epoch": 1.5,
"learning_rate": 2.767457478742533e-05,
"loss": 0.9834,
"step": 520
},
{
"epoch": 1.55,
"learning_rate": 2.6830949131224118e-05,
"loss": 0.9831,
"step": 540
},
{
"epoch": 1.61,
"learning_rate": 2.5973327998436527e-05,
"loss": 0.9787,
"step": 560
},
{
"epoch": 1.67,
"learning_rate": 2.5103468511865456e-05,
"loss": 0.981,
"step": 580
},
{
"epoch": 1.73,
"learning_rate": 2.4223152868661535e-05,
"loss": 0.9845,
"step": 600
},
{
"epoch": 1.78,
"learning_rate": 2.3334184688898107e-05,
"loss": 0.9754,
"step": 620
},
{
"epoch": 1.84,
"learning_rate": 2.2438385320254234e-05,
"loss": 0.9779,
"step": 640
},
{
"epoch": 1.9,
"learning_rate": 2.1537590106376758e-05,
"loss": 0.9737,
"step": 660
},
{
"epoch": 1.96,
"learning_rate": 2.0633644626567007e-05,
"loss": 0.9714,
"step": 680
},
{
"epoch": 2.0,
"eval_loss": 1.0180176496505737,
"eval_runtime": 20.0699,
"eval_samples_per_second": 13.901,
"eval_steps_per_second": 3.488,
"step": 695
},
{
"epoch": 2.01,
"learning_rate": 1.9728400914496288e-05,
"loss": 0.9669,
"step": 700
},
{
"epoch": 2.07,
"learning_rate": 1.882371366369749e-05,
"loss": 0.9478,
"step": 720
},
{
"epoch": 2.13,
"learning_rate": 1.79214364276071e-05,
"loss": 0.9458,
"step": 740
},
{
"epoch": 2.19,
"learning_rate": 1.702341782194301e-05,
"loss": 0.9307,
"step": 760
},
{
"epoch": 2.24,
"learning_rate": 1.6131497737198942e-05,
"loss": 0.9435,
"step": 780
},
{
"epoch": 2.3,
"learning_rate": 1.5247503569015413e-05,
"loss": 0.945,
"step": 800
},
{
"epoch": 2.36,
"learning_rate": 1.437324647415053e-05,
"loss": 0.9416,
"step": 820
},
{
"epoch": 2.42,
"learning_rate": 1.3510517659721583e-05,
"loss": 0.9476,
"step": 840
},
{
"epoch": 2.47,
"learning_rate": 1.2661084713320093e-05,
"loss": 0.946,
"step": 860
},
{
"epoch": 2.53,
"learning_rate": 1.182668798151939e-05,
"loss": 0.9414,
"step": 880
},
{
"epoch": 2.59,
"learning_rate": 1.1009037004194424e-05,
"loss": 0.9439,
"step": 900
},
{
"epoch": 2.65,
"learning_rate": 1.020980701195946e-05,
"loss": 0.9486,
"step": 920
},
{
"epoch": 2.71,
"learning_rate": 9.430635493899609e-06,
"loss": 0.949,
"step": 940
},
{
"epoch": 2.76,
"learning_rate": 8.673118842628595e-06,
"loss": 0.9376,
"step": 960
},
{
"epoch": 2.82,
"learning_rate": 7.938809083546264e-06,
"loss": 0.9432,
"step": 980
},
{
"epoch": 2.88,
"learning_rate": 7.229210694997113e-06,
"loss": 0.9457,
"step": 1000
},
{
"epoch": 2.94,
"learning_rate": 6.545777525844883e-06,
"loss": 0.9357,
"step": 1020
},
{
"epoch": 2.99,
"learning_rate": 5.889909816778458e-06,
"loss": 0.9335,
"step": 1040
},
{
"epoch": 3.0,
"eval_loss": 1.0176299810409546,
"eval_runtime": 20.0069,
"eval_samples_per_second": 13.945,
"eval_steps_per_second": 3.499,
"step": 1042
},
{
"epoch": 3.05,
"learning_rate": 5.262951331452011e-06,
"loss": 0.937,
"step": 1060
},
{
"epoch": 3.11,
"learning_rate": 4.6661866033371506e-06,
"loss": 0.9351,
"step": 1080
},
{
"epoch": 3.17,
"learning_rate": 4.100838303927914e-06,
"loss": 0.9415,
"step": 1100
},
{
"epoch": 3.22,
"learning_rate": 3.5680647376905666e-06,
"loss": 0.9293,
"step": 1120
},
{
"epoch": 3.28,
"learning_rate": 3.0689574688907607e-06,
"loss": 0.9304,
"step": 1140
},
{
"epoch": 3.34,
"learning_rate": 2.604539085160218e-06,
"loss": 0.9254,
"step": 1160
},
{
"epoch": 3.4,
"learning_rate": 2.1757611023850876e-06,
"loss": 0.9293,
"step": 1180
},
{
"epoch": 3.45,
"learning_rate": 1.7835020152084116e-06,
"loss": 0.9391,
"step": 1200
},
{
"epoch": 3.51,
"learning_rate": 1.4285654971409902e-06,
"loss": 0.9363,
"step": 1220
},
{
"epoch": 3.57,
"learning_rate": 1.1116787539682571e-06,
"loss": 0.9506,
"step": 1240
},
{
"epoch": 3.63,
"learning_rate": 8.334910338268054e-07,
"loss": 0.9226,
"step": 1260
},
{
"epoch": 3.68,
"learning_rate": 5.945722970031332e-07,
"loss": 0.9305,
"step": 1280
},
{
"epoch": 3.74,
"learning_rate": 3.9541204817997283e-07,
"loss": 0.9306,
"step": 1300
},
{
"epoch": 3.8,
"learning_rate": 2.3641833352276768e-07,
"loss": 0.9344,
"step": 1320
},
{
"epoch": 3.86,
"learning_rate": 1.1791690466107286e-07,
"loss": 0.93,
"step": 1340
},
{
"epoch": 3.91,
"learning_rate": 4.0150551277724494e-08,
"loss": 0.9344,
"step": 1360
},
{
"epoch": 3.97,
"learning_rate": 3.2786036732557203e-09,
"loss": 0.9348,
"step": 1380
},
{
"epoch": 3.99,
"eval_loss": 1.0186352729797363,
"eval_runtime": 19.987,
"eval_samples_per_second": 13.959,
"eval_steps_per_second": 3.502,
"step": 1388
},
{
"epoch": 3.99,
"step": 1388,
"total_flos": 1.7642090681398723e+18,
"train_loss": 0.9866050820185747,
"train_runtime": 37340.4665,
"train_samples_per_second": 3.722,
"train_steps_per_second": 0.037
}
],
"logging_steps": 20,
"max_steps": 1388,
"num_input_tokens_seen": 0,
"num_train_epochs": 4,
"save_steps": 80,
"total_flos": 1.7642090681398723e+18,
"train_batch_size": 10,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4e17cbf8f3c0a176c6460d28bc958b598d7474046216c4028081f81e032aebe6
size 4856