brandonbaek/Bori-1-0.6B-Base
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: brandonbaek/Bori-1-0.6B-Base
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-17 04:24:09 +08:00
commit 42f87f7601
15 changed files with 915 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

37
README.md Normal file
View File

@@ -0,0 +1,37 @@
---
language:
- ko
- en
license: apache-2.0
tags:
- slm
- korean
- experimental
- qwen2
- text-generation
- continuous-pre-training
base_model: Qwen/Qwen2-0.6B
pipeline_tag: text-generation
library_name: transformers
new_version: brandonbaek/Bori-2-135M-Base
---
# 🌾 Bori-1 0.6B Base (Checkpoint 1000)
> **🚀 Newer Version Available:** The Bori project has evolved! Please see [Bori-2 135M Base](https://huggingface.co/brandonbaek/Bori-2-135M-Base) for the completed Phase 2 pre-training pipeline, or check out the Bori GitHub repository for the latest **Bori-3** developments.
**Bori-1** is the very first experimental proof-of-concept for the Bori project, aimed at exploring the feasibility of training bilingual (Korean-English) Small Language Models (SLMs) under extreme compute constraints.
⚠️ **Status**: This was a preliminary exploratory run paused early at **Checkpoint 1000**. It is published solely for historical tracking and to serve as a baseline for the architectural shifts made in Bori-2 and Bori-3.
## 🤖 Model Details
- **Base Architecture**: Qwen2
- **Parameter Count**: ~600M
- **Languages**: Korean, English
## 💻 Hardware & Compute
- **Hardware**: Trained on Kaggle Notebooks using **2x NVIDIA T4 GPUs** (16GB VRAM each).
- **Constraints**: Navigating the memory constraints of a 600M parameter model on 16GB GPUs without advanced quantization required aggressive gradient accumulation and small batch sizes, leading to the decision to pivot to the highly efficient ~135M architecture for Bori-2 to allow for more robust pre-training experimentation.
## ⚠️ Limitations & Intended Use
This model was paused very early in its training lifecycle. It is significantly undertrained and exhibits poor coherence. It should not be used for text generation, fine-tuning, or deployment. Its primary value is as a historical artifact demonstrating the early stages of the Bori project's development.

6
chat_template.jinja Normal file
View File

@@ -0,0 +1,6 @@
{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
You are a helpful assistant<|im_end|>
' }}{% endif %}{{'<|im_start|>' + message['role'] + '
' + message['content'] + '<|im_end|>' + '
'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
' }}{% endif %}

51
config.json Normal file
View File

@@ -0,0 +1,51 @@
{
"architectures": [
"Qwen2ForCausalLM"
],
"attention_dropout": 0.0,
"bos_token_id": 151643,
"dtype": "float32",
"eos_token_id": 151645,
"hidden_act": "silu",
"hidden_size": 1536,
"initializer_range": 0.02,
"intermediate_size": 4096,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 4096,
"max_window_layers": 18,
"model_type": "qwen2",
"num_attention_heads": 12,
"num_hidden_layers": 18,
"num_key_value_heads": 2,
"pad_token_id": null,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 1000000.0,
"rope_type": "default"
},
"sliding_window": null,
"tie_word_embeddings": true,
"transformers_version": "5.0.0",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 151646
}

9
generation_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"_from_model_config": true,
"bos_token_id": 151643,
"eos_token_id": 151645,
"output_attentions": false,
"output_hidden_states": false,
"transformers_version": "5.0.0",
"use_cache": false
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5c73584e92210c62ab2ba24f7131fb5cd7323e6c53f82f5b1f3eb3e2c1b796b6
size 2687428704

3
optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:81451746a37f9204ae62ce11fb59fcffbf27578183cf6e0c72c11ce008418972
size 3064587

3
rng_state_0.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4dd7671ce88d469c49c0530724ac76b2306574002d1ecd1ca9294e41621fd96a
size 14917

3
rng_state_1.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3246ef1170ccca541a03b89ad6f20e01c51eb6834a2c2211c78c71c70f896879
size 14917

3
scaler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:14ae2a2128444abab378aa06c09a61a84665f758fcc19fc46f5789b0bc1b5665
size 1383

3
scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d3b02050e99cc0fa62bf28bb11c5a90caba14124a2d1593d4856c7efc26f47ef
size 1465

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
size 11418266

18
tokenizer_config.json Normal file
View File

@@ -0,0 +1,18 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"errors": "replace",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>"
],
"is_local": false,
"model_max_length": 32768,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "TokenizersBackend",
"unk_token": null
}

734
trainer_state.json Normal file
View File

@@ -0,0 +1,734 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.2,
"eval_steps": 500,
"global_step": 1000,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.002,
"grad_norm": 2.2951126098632812,
"learning_rate": 5.399999999999999e-06,
"loss": 12.198476409912109,
"step": 10
},
{
"epoch": 0.004,
"grad_norm": 2.6434836387634277,
"learning_rate": 1.14e-05,
"loss": 11.658338928222657,
"step": 20
},
{
"epoch": 0.006,
"grad_norm": 2.0430407524108887,
"learning_rate": 1.74e-05,
"loss": 10.702792358398437,
"step": 30
},
{
"epoch": 0.008,
"grad_norm": 1.938822627067566,
"learning_rate": 2.34e-05,
"loss": 10.229283142089844,
"step": 40
},
{
"epoch": 0.01,
"grad_norm": 1.3679683208465576,
"learning_rate": 2.94e-05,
"loss": 9.728135681152343,
"step": 50
},
{
"epoch": 0.012,
"grad_norm": 2.546030044555664,
"learning_rate": 3.539999999999999e-05,
"loss": 9.183514404296876,
"step": 60
},
{
"epoch": 0.014,
"grad_norm": 1.7231528759002686,
"learning_rate": 4.14e-05,
"loss": 8.578328704833984,
"step": 70
},
{
"epoch": 0.016,
"grad_norm": 1.4366660118103027,
"learning_rate": 4.7399999999999993e-05,
"loss": 8.042523956298828,
"step": 80
},
{
"epoch": 0.018,
"grad_norm": 1.4448844194412231,
"learning_rate": 5.339999999999999e-05,
"loss": 7.64561767578125,
"step": 90
},
{
"epoch": 0.02,
"grad_norm": 1.3400251865386963,
"learning_rate": 5.94e-05,
"loss": 7.421743774414063,
"step": 100
},
{
"epoch": 0.022,
"grad_norm": 0.8998181223869324,
"learning_rate": 6.539999999999999e-05,
"loss": 7.200779724121094,
"step": 110
},
{
"epoch": 0.024,
"grad_norm": 0.8080795407295227,
"learning_rate": 7.139999999999999e-05,
"loss": 7.073184204101563,
"step": 120
},
{
"epoch": 0.026,
"grad_norm": 0.970500648021698,
"learning_rate": 7.74e-05,
"loss": 6.967037963867187,
"step": 130
},
{
"epoch": 0.028,
"grad_norm": 1.2163574695587158,
"learning_rate": 8.34e-05,
"loss": 6.91470947265625,
"step": 140
},
{
"epoch": 0.03,
"grad_norm": 1.1663590669631958,
"learning_rate": 8.939999999999999e-05,
"loss": 6.799385070800781,
"step": 150
},
{
"epoch": 0.032,
"grad_norm": 1.1311771869659424,
"learning_rate": 9.539999999999999e-05,
"loss": 6.620470428466797,
"step": 160
},
{
"epoch": 0.034,
"grad_norm": 1.1079429388046265,
"learning_rate": 0.0001014,
"loss": 6.601376342773437,
"step": 170
},
{
"epoch": 0.036,
"grad_norm": 1.3198282718658447,
"learning_rate": 0.00010739999999999998,
"loss": 6.559695434570313,
"step": 180
},
{
"epoch": 0.038,
"grad_norm": 1.0375511646270752,
"learning_rate": 0.00011339999999999999,
"loss": 6.444459533691406,
"step": 190
},
{
"epoch": 0.04,
"grad_norm": 1.1254252195358276,
"learning_rate": 0.0001194,
"loss": 6.329716491699219,
"step": 200
},
{
"epoch": 0.042,
"grad_norm": 1.0709344148635864,
"learning_rate": 0.00012539999999999999,
"loss": 6.242716979980469,
"step": 210
},
{
"epoch": 0.044,
"grad_norm": 1.0490570068359375,
"learning_rate": 0.0001314,
"loss": 6.146401214599609,
"step": 220
},
{
"epoch": 0.046,
"grad_norm": 0.9572665095329285,
"learning_rate": 0.0001374,
"loss": 6.086616134643554,
"step": 230
},
{
"epoch": 0.048,
"grad_norm": 1.1944881677627563,
"learning_rate": 0.0001434,
"loss": 6.035008239746094,
"step": 240
},
{
"epoch": 0.05,
"grad_norm": 1.139950156211853,
"learning_rate": 0.0001494,
"loss": 5.863430786132812,
"step": 250
},
{
"epoch": 0.052,
"grad_norm": 1.0705268383026123,
"learning_rate": 0.00015539999999999998,
"loss": 5.8445487976074215,
"step": 260
},
{
"epoch": 0.054,
"grad_norm": 1.228391170501709,
"learning_rate": 0.0001614,
"loss": 5.702362442016602,
"step": 270
},
{
"epoch": 0.056,
"grad_norm": 1.2548208236694336,
"learning_rate": 0.0001674,
"loss": 5.753414154052734,
"step": 280
},
{
"epoch": 0.058,
"grad_norm": 1.0533807277679443,
"learning_rate": 0.00017339999999999996,
"loss": 5.655702209472656,
"step": 290
},
{
"epoch": 0.06,
"grad_norm": 0.9166179895401001,
"learning_rate": 0.00017939999999999997,
"loss": 5.616793060302735,
"step": 300
},
{
"epoch": 0.062,
"grad_norm": 0.8547247648239136,
"learning_rate": 0.00018539999999999998,
"loss": 5.597354888916016,
"step": 310
},
{
"epoch": 0.064,
"grad_norm": 0.9937305450439453,
"learning_rate": 0.0001914,
"loss": 5.572035598754883,
"step": 320
},
{
"epoch": 0.066,
"grad_norm": 0.9027020335197449,
"learning_rate": 0.0001974,
"loss": 5.49364013671875,
"step": 330
},
{
"epoch": 0.068,
"grad_norm": 0.976992666721344,
"learning_rate": 0.00020339999999999998,
"loss": 5.4948478698730465,
"step": 340
},
{
"epoch": 0.07,
"grad_norm": 0.8595808148384094,
"learning_rate": 0.00020939999999999997,
"loss": 5.504860687255859,
"step": 350
},
{
"epoch": 0.072,
"grad_norm": 0.8206803798675537,
"learning_rate": 0.00021539999999999998,
"loss": 5.442459487915039,
"step": 360
},
{
"epoch": 0.074,
"grad_norm": 0.8065920472145081,
"learning_rate": 0.0002214,
"loss": 5.3627159118652346,
"step": 370
},
{
"epoch": 0.076,
"grad_norm": 1.3093833923339844,
"learning_rate": 0.00022739999999999997,
"loss": 5.357783508300781,
"step": 380
},
{
"epoch": 0.078,
"grad_norm": 1.0391067266464233,
"learning_rate": 0.00023339999999999998,
"loss": 5.305191421508789,
"step": 390
},
{
"epoch": 0.08,
"grad_norm": 1.1108404397964478,
"learning_rate": 0.0002394,
"loss": 5.3095954895019535,
"step": 400
},
{
"epoch": 0.082,
"grad_norm": 0.7273285388946533,
"learning_rate": 0.00024539999999999995,
"loss": 5.245541381835937,
"step": 410
},
{
"epoch": 0.084,
"grad_norm": 0.8222587704658508,
"learning_rate": 0.0002514,
"loss": 5.180028915405273,
"step": 420
},
{
"epoch": 0.086,
"grad_norm": 0.9594866633415222,
"learning_rate": 0.00025739999999999997,
"loss": 5.281720733642578,
"step": 430
},
{
"epoch": 0.088,
"grad_norm": 0.6788018345832825,
"learning_rate": 0.00026339999999999995,
"loss": 5.117179870605469,
"step": 440
},
{
"epoch": 0.09,
"grad_norm": 0.9051905870437622,
"learning_rate": 0.0002694,
"loss": 5.1706890106201175,
"step": 450
},
{
"epoch": 0.092,
"grad_norm": 0.7013774514198303,
"learning_rate": 0.00027539999999999997,
"loss": 5.212803649902344,
"step": 460
},
{
"epoch": 0.094,
"grad_norm": 0.8330399990081787,
"learning_rate": 0.00028139999999999996,
"loss": 5.102211761474609,
"step": 470
},
{
"epoch": 0.096,
"grad_norm": 0.7064741253852844,
"learning_rate": 0.00028739999999999994,
"loss": 5.0642646789550785,
"step": 480
},
{
"epoch": 0.098,
"grad_norm": 0.7228646278381348,
"learning_rate": 0.0002934,
"loss": 5.086193084716797,
"step": 490
},
{
"epoch": 0.1,
"grad_norm": 0.6500748991966248,
"learning_rate": 0.00029939999999999996,
"loss": 4.952143859863281,
"step": 500
},
{
"epoch": 0.102,
"grad_norm": 0.6131930947303772,
"learning_rate": 0.00029999703912842054,
"loss": 5.013114547729492,
"step": 510
},
{
"epoch": 0.104,
"grad_norm": 0.6693644523620605,
"learning_rate": 0.0002999868041668571,
"loss": 5.064251327514649,
"step": 520
},
{
"epoch": 0.106,
"grad_norm": 0.8424168825149536,
"learning_rate": 0.0002999692590600585,
"loss": 4.953443145751953,
"step": 530
},
{
"epoch": 0.108,
"grad_norm": 0.8079104423522949,
"learning_rate": 0.00029994440466314827,
"loss": 5.028744888305664,
"step": 540
},
{
"epoch": 0.11,
"grad_norm": 0.6316964030265808,
"learning_rate": 0.00029991224218749483,
"loss": 4.919719696044922,
"step": 550
},
{
"epoch": 0.112,
"grad_norm": 0.5195901393890381,
"learning_rate": 0.0002998727732006518,
"loss": 4.8272350311279295,
"step": 560
},
{
"epoch": 0.114,
"grad_norm": 0.8154641389846802,
"learning_rate": 0.00029982599962628185,
"loss": 4.901306533813477,
"step": 570
},
{
"epoch": 0.116,
"grad_norm": 0.6902443170547485,
"learning_rate": 0.0002997719237440631,
"loss": 4.899118041992187,
"step": 580
},
{
"epoch": 0.118,
"grad_norm": 0.7940948009490967,
"learning_rate": 0.0002997105481895778,
"loss": 4.76171989440918,
"step": 590
},
{
"epoch": 0.12,
"grad_norm": 0.6447590589523315,
"learning_rate": 0.00029964187595418377,
"loss": 4.846940231323242,
"step": 600
},
{
"epoch": 0.122,
"grad_norm": 0.5544273853302002,
"learning_rate": 0.00029956591038486895,
"loss": 4.747989654541016,
"step": 610
},
{
"epoch": 0.124,
"grad_norm": 0.5011790990829468,
"learning_rate": 0.000299482655184088,
"loss": 4.800208663940429,
"step": 620
},
{
"epoch": 0.126,
"grad_norm": 0.5372884273529053,
"learning_rate": 0.0002993921144095819,
"loss": 4.803700256347656,
"step": 630
},
{
"epoch": 0.128,
"grad_norm": 0.44695383310317993,
"learning_rate": 0.00029929429247418037,
"loss": 4.715145111083984,
"step": 640
},
{
"epoch": 0.13,
"grad_norm": 0.5330514907836914,
"learning_rate": 0.0002991891941455865,
"loss": 4.6798866271972654,
"step": 650
},
{
"epoch": 0.132,
"grad_norm": 0.7654022574424744,
"learning_rate": 0.0002990768245461445,
"loss": 4.705394744873047,
"step": 660
},
{
"epoch": 0.134,
"grad_norm": 0.6759101748466492,
"learning_rate": 0.0002989571891525901,
"loss": 4.711079406738281,
"step": 670
},
{
"epoch": 0.136,
"grad_norm": 0.5494880080223083,
"learning_rate": 0.00029883029379578375,
"loss": 4.6135414123535154,
"step": 680
},
{
"epoch": 0.138,
"grad_norm": 0.7019897103309631,
"learning_rate": 0.000298696144660426,
"loss": 4.6716865539550785,
"step": 690
},
{
"epoch": 0.14,
"grad_norm": 0.6551356911659241,
"learning_rate": 0.0002985547482847564,
"loss": 4.780976486206055,
"step": 700
},
{
"epoch": 0.142,
"grad_norm": 0.505607545375824,
"learning_rate": 0.000298406111560235,
"loss": 4.607578277587891,
"step": 710
},
{
"epoch": 0.144,
"grad_norm": 0.43605464696884155,
"learning_rate": 0.00029825024173120587,
"loss": 4.561976623535156,
"step": 720
},
{
"epoch": 0.146,
"grad_norm": 0.48921728134155273,
"learning_rate": 0.0002980871463945447,
"loss": 4.422502899169922,
"step": 730
},
{
"epoch": 0.148,
"grad_norm": 0.7877730131149292,
"learning_rate": 0.00029791683349928805,
"loss": 4.556784439086914,
"step": 740
},
{
"epoch": 0.15,
"grad_norm": 0.5935314893722534,
"learning_rate": 0.00029773931134624614,
"loss": 4.396469497680664,
"step": 750
},
{
"epoch": 0.152,
"grad_norm": 0.6277227401733398,
"learning_rate": 0.0002975545885875983,
"loss": 4.522803115844726,
"step": 760
},
{
"epoch": 0.154,
"grad_norm": 0.6810743808746338,
"learning_rate": 0.0002973626742264713,
"loss": 4.3536125183105465,
"step": 770
},
{
"epoch": 0.156,
"grad_norm": 0.4122457504272461,
"learning_rate": 0.00029716357761650007,
"loss": 4.4033355712890625,
"step": 780
},
{
"epoch": 0.158,
"grad_norm": 0.4085232615470886,
"learning_rate": 0.00029695730846137275,
"loss": 4.320112609863282,
"step": 790
},
{
"epoch": 0.16,
"grad_norm": 0.6114133596420288,
"learning_rate": 0.0002967438768143568,
"loss": 4.373843383789063,
"step": 800
},
{
"epoch": 0.162,
"grad_norm": 0.5036770105361938,
"learning_rate": 0.0002965232930778097,
"loss": 4.2889457702636715,
"step": 810
},
{
"epoch": 0.164,
"grad_norm": 0.4276726245880127,
"learning_rate": 0.0002962955680026715,
"loss": 4.212138366699219,
"step": 820
},
{
"epoch": 0.166,
"grad_norm": 1.472080111503601,
"learning_rate": 0.0002960607126879411,
"loss": 4.297634124755859,
"step": 830
},
{
"epoch": 0.168,
"grad_norm": 0.5528949499130249,
"learning_rate": 0.00029581873858013525,
"loss": 4.357148742675781,
"step": 840
},
{
"epoch": 0.17,
"grad_norm": 0.5142295956611633,
"learning_rate": 0.0002955696574727306,
"loss": 4.17579231262207,
"step": 850
},
{
"epoch": 0.172,
"grad_norm": 0.45028647780418396,
"learning_rate": 0.00029531348150558883,
"loss": 4.272770690917969,
"step": 860
},
{
"epoch": 0.174,
"grad_norm": 0.38256195187568665,
"learning_rate": 0.00029505022316436523,
"loss": 4.236406707763672,
"step": 870
},
{
"epoch": 0.176,
"grad_norm": 0.7847526669502258,
"learning_rate": 0.00029477989527989997,
"loss": 4.145956420898438,
"step": 880
},
{
"epoch": 0.178,
"grad_norm": 1.6398742198944092,
"learning_rate": 0.0002945025110275927,
"loss": 4.173336029052734,
"step": 890
},
{
"epoch": 0.18,
"grad_norm": 0.5771706700325012,
"learning_rate": 0.0002942180839267606,
"loss": 4.133999633789062,
"step": 900
},
{
"epoch": 0.182,
"grad_norm": 0.4774192273616791,
"learning_rate": 0.0002939266278399793,
"loss": 4.1810859680175785,
"step": 910
},
{
"epoch": 0.184,
"grad_norm": 0.4462798833847046,
"learning_rate": 0.00029362815697240723,
"loss": 4.148078155517578,
"step": 920
},
{
"epoch": 0.186,
"grad_norm": 0.5258499383926392,
"learning_rate": 0.0002933226858710934,
"loss": 4.151172637939453,
"step": 930
},
{
"epoch": 0.188,
"grad_norm": 0.4309222996234894,
"learning_rate": 0.00029301022942426844,
"loss": 4.16070442199707,
"step": 940
},
{
"epoch": 0.19,
"grad_norm": 0.40204179286956787,
"learning_rate": 0.00029269080286061866,
"loss": 4.0530345916748045,
"step": 950
},
{
"epoch": 0.192,
"grad_norm": 0.3796781301498413,
"learning_rate": 0.00029236442174854416,
"loss": 3.982533264160156,
"step": 960
},
{
"epoch": 0.194,
"grad_norm": 0.3593500256538391,
"learning_rate": 0.00029203110199539986,
"loss": 4.06689338684082,
"step": 970
},
{
"epoch": 0.196,
"grad_norm": 0.3557942807674408,
"learning_rate": 0.0002916908598467203,
"loss": 4.02539176940918,
"step": 980
},
{
"epoch": 0.198,
"grad_norm": 0.3429299294948578,
"learning_rate": 0.00029134371188542784,
"loss": 4.072704315185547,
"step": 990
},
{
"epoch": 0.2,
"grad_norm": 0.42700159549713135,
"learning_rate": 0.0002909896750310243,
"loss": 4.072047424316406,
"step": 1000
}
],
"logging_steps": 10,
"max_steps": 5000,
"num_input_tokens_seen": 0,
"num_train_epochs": 9223372036854775807,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 3.45182897700864e+17,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:56e6318c7c35297d402d9e3f5b9c8b652c3600a088f15e66198a1b18c89cde8e
size 5201