303 lines
9.1 KiB
Markdown
303 lines
9.1 KiB
Markdown
|
---
|
||
|
|
library_name: transformers
|
||
|
|
base_model: minpeter/tiny-ko-187m-base-250725
|
||
|
|
tags:
|
||
|
|
- axolotl
|
||
|
|
- generated_from_trainer
|
||
|
|
datasets:
|
||
|
|
- HuggingFaceTB/smol-smoltalk
|
||
|
|
- trillionlabs/multisystem-curated
|
||
|
|
- allenai/tulu-3-sft-personas-instruction-following
|
||
|
|
- lemon-mint/smol-koreantalk
|
||
|
|
- lemon-mint/Korean-FineTome-100k
|
||
|
|
- heegyu/open-korean-instructions-v20231020
|
||
|
|
- coastral/korean-writing-style-instruct
|
||
|
|
- devngho/korean-instruction-mix
|
||
|
|
model-index:
|
||
|
|
- name: tiny-ko-187m-sft-250725
|
||
|
|
results: []
|
||
|
|
---
|
||
|
|
|
||
|
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||
|
|
should probably proofread and complete it, then remove this comment. -->
|
||
|
|
|
||
|
|
[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
|
||
|
|
<details><summary>See axolotl config</summary>
|
||
|
|
|
||
|
|
axolotl version: `0.12.0.dev0`
|
||
|
|
```yaml
|
||
|
|
base_model: minpeter/tiny-ko-187m-base-250725
|
||
|
|
|
||
|
|
hub_model_id: minpeter/tiny-ko-187m-sft-250725
|
||
|
|
output_dir: ./outputs/tiny-ko-187m-sft-250725
|
||
|
|
wandb_project: "axolotl"
|
||
|
|
wandb_entity: "kasfiekfs-e"
|
||
|
|
|
||
|
|
model_type: LlamaForCausalLM
|
||
|
|
tokenizer_type: AutoTokenizer
|
||
|
|
|
||
|
|
strict: false
|
||
|
|
|
||
|
|
chat_template: chatml
|
||
|
|
datasets:
|
||
|
|
- path: HuggingFaceTB/smol-smoltalk
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: messages
|
||
|
|
message_property_mappings:
|
||
|
|
role: role
|
||
|
|
content: content
|
||
|
|
|
||
|
|
- path: trillionlabs/multisystem-curated
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: messages
|
||
|
|
message_property_mappings:
|
||
|
|
role: role
|
||
|
|
content: content
|
||
|
|
|
||
|
|
- path: allenai/tulu-3-sft-personas-instruction-following
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: messages
|
||
|
|
message_property_mappings:
|
||
|
|
role: role
|
||
|
|
content: content
|
||
|
|
|
||
|
|
- path: lemon-mint/smol-koreantalk
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: messages
|
||
|
|
message_property_mappings:
|
||
|
|
role: role
|
||
|
|
content: content
|
||
|
|
|
||
|
|
- path: lemon-mint/Korean-FineTome-100k
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: messages
|
||
|
|
message_property_mappings:
|
||
|
|
role: role
|
||
|
|
content: content
|
||
|
|
|
||
|
|
- path: heegyu/open-korean-instructions-v20231020
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: conversations
|
||
|
|
message_property_mappings:
|
||
|
|
role: from
|
||
|
|
content: value
|
||
|
|
roles:
|
||
|
|
user: ["human", "user"]
|
||
|
|
assistant: ["gpt", "assistant", "bot"]
|
||
|
|
system: ["system", "input"]
|
||
|
|
|
||
|
|
- path: coastral/korean-writing-style-instruct
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: conversations
|
||
|
|
message_property_mappings:
|
||
|
|
role: from
|
||
|
|
content: value
|
||
|
|
|
||
|
|
- path: devngho/korean-instruction-mix
|
||
|
|
type: chat_template
|
||
|
|
split: train
|
||
|
|
field_messages: messages
|
||
|
|
message_property_mappings:
|
||
|
|
role: from
|
||
|
|
content: value
|
||
|
|
|
||
|
|
dataset_prepared_path: last_run_prepared
|
||
|
|
val_set_size: 0.001
|
||
|
|
save_safetensors: true
|
||
|
|
sequence_len: 8192
|
||
|
|
sample_packing: false
|
||
|
|
pad_to_sequence_len: false
|
||
|
|
use_pose: true
|
||
|
|
pose_max_context_len: 65536
|
||
|
|
|
||
|
|
overrides_of_model_config:
|
||
|
|
rope_theta: 1000000.0
|
||
|
|
max_position_embeddings: 65536
|
||
|
|
|
||
|
|
gradient_accumulation_steps: 8
|
||
|
|
micro_batch_size: 16
|
||
|
|
num_epochs: 1
|
||
|
|
optimizer: muon
|
||
|
|
lr_scheduler: cosine
|
||
|
|
learning_rate: 3e-4
|
||
|
|
|
||
|
|
train_on_inputs: false
|
||
|
|
group_by_length: false
|
||
|
|
bf16: true
|
||
|
|
fp16:
|
||
|
|
tf32: true
|
||
|
|
|
||
|
|
gradient_checkpointing: false
|
||
|
|
gradient_checkpointing_kwargs:
|
||
|
|
use_reentrant: true
|
||
|
|
early_stopping_patience:
|
||
|
|
resume_from_checkpoint:
|
||
|
|
local_rank:
|
||
|
|
logging_steps: 1
|
||
|
|
xformers_attention:
|
||
|
|
flash_attention: true
|
||
|
|
sdp_attention:
|
||
|
|
s2_attention:
|
||
|
|
|
||
|
|
save_steps: 200
|
||
|
|
warmup_steps: 20
|
||
|
|
eval_steps: 200
|
||
|
|
debug:
|
||
|
|
deepspeed:
|
||
|
|
weight_decay: 0.0
|
||
|
|
fsdp:
|
||
|
|
fsdp_config:
|
||
|
|
|
||
|
|
special_tokens:
|
||
|
|
eos_token: '<|im_end|>'
|
||
|
|
|
||
|
|
plugins:
|
||
|
|
- axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
|
||
|
|
- axolotl.integrations.liger.LigerPlugin
|
||
|
|
# - axolotl.integrations.lm_eval.LMEvalPlugin
|
||
|
|
|
||
|
|
# lm_eval_tasks:
|
||
|
|
# - gsm8k
|
||
|
|
# - hellaswag
|
||
|
|
# - arc_easy
|
||
|
|
# - arc_challenge
|
||
|
|
# - piqa
|
||
|
|
# - winogrande
|
||
|
|
# - openbookqa
|
||
|
|
# - wsc
|
||
|
|
# - boolq
|
||
|
|
|
||
|
|
liger_rope: true
|
||
|
|
liger_rms_norm: true
|
||
|
|
liger_glu_activation: true
|
||
|
|
liger_layer_norm: true
|
||
|
|
liger_fused_linear_cross_entropy: true
|
||
|
|
|
||
|
|
```
|
||
|
|
|
||
|
|
</details><br>
|
||
|
|
|
||
|
|
# tiny-ko-187m-sft-250725
|
||
|
|
|
||
|
|
This model is a fine-tuned version of [minpeter/tiny-ko-187m-base-250725](https://huggingface.co/minpeter/tiny-ko-187m-base-250725) on the HuggingFaceTB/smol-smoltalk, the trillionlabs/multisystem-curated, the allenai/tulu-3-sft-personas-instruction-following, the lemon-mint/smol-koreantalk, the lemon-mint/Korean-FineTome-100k, the heegyu/open-korean-instructions-v20231020, the coastral/korean-writing-style-instruct and the devngho/korean-instruction-mix datasets.
|
||
|
|
It achieves the following results on the evaluation set:
|
||
|
|
- Loss: 1.6810
|
||
|
|
|
||
|
|
## Model description
|
||
|
|
|
||
|
|
More information needed
|
||
|
|
|
||
|
|
## Intended uses & limitations
|
||
|
|
|
||
|
|
More information needed
|
||
|
|
|
||
|
|
## Training and evaluation data
|
||
|
|
|
||
|
|
More information needed
|
||
|
|
|
||
|
|
## Training procedure
|
||
|
|
|
||
|
|
### Training hyperparameters
|
||
|
|
|
||
|
|
The following hyperparameters were used during training:
|
||
|
|
- learning_rate: 0.0003
|
||
|
|
- train_batch_size: 16
|
||
|
|
- eval_batch_size: 16
|
||
|
|
- seed: 42
|
||
|
|
- gradient_accumulation_steps: 8
|
||
|
|
- total_train_batch_size: 128
|
||
|
|
- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
||
|
|
- lr_scheduler_type: cosine
|
||
|
|
- lr_scheduler_warmup_steps: 20
|
||
|
|
- training_steps: 13880
|
||
|
|
|
||
|
|
### Training results
|
||
|
|
|
||
|
|
| Training Loss | Epoch | Step | Validation Loss |
|
||
|
|
|:-------------:|:------:|:-----:|:---------------:|
|
||
|
|
| No log | 0 | 0 | 2.1583 |
|
||
|
|
| 2.0492 | 0.0144 | 200 | 1.9095 |
|
||
|
|
| 1.9084 | 0.0288 | 400 | 1.8642 |
|
||
|
|
| 1.8437 | 0.0432 | 600 | 1.8363 |
|
||
|
|
| 1.8601 | 0.0576 | 800 | 1.8160 |
|
||
|
|
| 1.8143 | 0.0720 | 1000 | 1.8001 |
|
||
|
|
| 1.7259 | 0.0865 | 1200 | 1.7871 |
|
||
|
|
| 1.8008 | 0.1009 | 1400 | 1.7764 |
|
||
|
|
| 1.7179 | 0.1153 | 1600 | 1.7668 |
|
||
|
|
| 1.8134 | 0.1297 | 1800 | 1.7587 |
|
||
|
|
| 1.7683 | 0.1441 | 2000 | 1.7518 |
|
||
|
|
| 1.8099 | 0.1585 | 2200 | 1.7458 |
|
||
|
|
| 1.7953 | 0.1729 | 2400 | 1.7397 |
|
||
|
|
| 1.7673 | 0.1873 | 2600 | 1.7339 |
|
||
|
|
| 1.721 | 0.2017 | 2800 | 1.7297 |
|
||
|
|
| 1.8727 | 0.2161 | 3000 | 1.7252 |
|
||
|
|
| 1.7334 | 0.2306 | 3200 | 1.7209 |
|
||
|
|
| 1.7078 | 0.2450 | 3400 | 1.7172 |
|
||
|
|
| 1.7751 | 0.2594 | 3600 | 1.7139 |
|
||
|
|
| 1.8221 | 0.2738 | 3800 | 1.7106 |
|
||
|
|
| 1.7385 | 0.2882 | 4000 | 1.7077 |
|
||
|
|
| 1.8214 | 0.3026 | 4200 | 1.7049 |
|
||
|
|
| 1.7042 | 0.3170 | 4400 | 1.7026 |
|
||
|
|
| 1.735 | 0.3314 | 4600 | 1.7005 |
|
||
|
|
| 1.647 | 0.3458 | 4800 | 1.6985 |
|
||
|
|
| 1.7222 | 0.3602 | 5000 | 1.6972 |
|
||
|
|
| 1.6963 | 0.3746 | 5200 | 1.6955 |
|
||
|
|
| 1.8047 | 0.3891 | 5400 | 1.6937 |
|
||
|
|
| 1.6151 | 0.4035 | 5600 | 1.6923 |
|
||
|
|
| 1.8008 | 0.4179 | 5800 | 1.6918 |
|
||
|
|
| 1.7152 | 0.4323 | 6000 | 1.6904 |
|
||
|
|
| 1.7522 | 0.4467 | 6200 | 1.6894 |
|
||
|
|
| 1.7645 | 0.4611 | 6400 | 1.6887 |
|
||
|
|
| 1.6721 | 0.4755 | 6600 | 1.6876 |
|
||
|
|
| 1.7343 | 0.4899 | 6800 | 1.6867 |
|
||
|
|
| 1.6748 | 0.5043 | 7000 | 1.6861 |
|
||
|
|
| 1.7417 | 0.5187 | 7200 | 1.6853 |
|
||
|
|
| 1.6245 | 0.5332 | 7400 | 1.6849 |
|
||
|
|
| 1.6081 | 0.5476 | 7600 | 1.6844 |
|
||
|
|
| 1.6696 | 0.5620 | 7800 | 1.6840 |
|
||
|
|
| 1.714 | 0.5764 | 8000 | 1.6836 |
|
||
|
|
| 1.8028 | 0.5908 | 8200 | 1.6832 |
|
||
|
|
| 1.6714 | 0.6052 | 8400 | 1.6828 |
|
||
|
|
| 1.7276 | 0.6196 | 8600 | 1.6827 |
|
||
|
|
| 1.6248 | 0.6340 | 8800 | 1.6823 |
|
||
|
|
| 1.7279 | 0.6484 | 9000 | 1.6823 |
|
||
|
|
| 1.7176 | 0.6628 | 9200 | 1.6820 |
|
||
|
|
| 1.8175 | 0.6773 | 9400 | 1.6819 |
|
||
|
|
| 1.7082 | 0.6917 | 9600 | 1.6817 |
|
||
|
|
| 1.7886 | 0.7061 | 9800 | 1.6815 |
|
||
|
|
| 1.7777 | 0.7205 | 10000 | 1.6814 |
|
||
|
|
| 1.7943 | 0.7349 | 10200 | 1.6814 |
|
||
|
|
| 1.79 | 0.7493 | 10400 | 1.6813 |
|
||
|
|
| 1.6757 | 0.7637 | 10600 | 1.6813 |
|
||
|
|
| 1.6708 | 0.7781 | 10800 | 1.6813 |
|
||
|
|
| 1.7519 | 0.7925 | 11000 | 1.6811 |
|
||
|
|
| 1.7789 | 0.8069 | 11200 | 1.6812 |
|
||
|
|
| 1.7562 | 0.8213 | 11400 | 1.6811 |
|
||
|
|
| 1.7137 | 0.8358 | 11600 | 1.6811 |
|
||
|
|
| 1.7407 | 0.8502 | 11800 | 1.6812 |
|
||
|
|
| 1.6515 | 0.8646 | 12000 | 1.6811 |
|
||
|
|
| 1.6929 | 0.8790 | 12200 | 1.6812 |
|
||
|
|
| 1.7125 | 0.8934 | 12400 | 1.6812 |
|
||
|
|
| 1.6112 | 0.9078 | 12600 | 1.6812 |
|
||
|
|
| 1.7437 | 0.9222 | 12800 | 1.6811 |
|
||
|
|
| 1.7824 | 0.9366 | 13000 | 1.6811 |
|
||
|
|
| 1.6166 | 0.9510 | 13200 | 1.6811 |
|
||
|
|
| 1.743 | 0.9654 | 13400 | 1.6812 |
|
||
|
|
| 1.6377 | 0.9799 | 13600 | 1.6812 |
|
||
|
|
| 1.7345 | 0.9943 | 13800 | 1.6810 |
|
||
|
|
|
||
|
|
|
||
|
|
### Framework versions
|
||
|
|
|
||
|
|
- Transformers 4.53.2
|
||
|
|
- Pytorch 2.7.1+cu126
|
||
|
|
- Datasets 4.0.0
|
||
|
|
- Tokenizers 0.21.2
|