anwgpt/anwgpt4.1-base
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: anwgpt/anwgpt4.1-base
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-21 06:53:17 +08:00
commit 7c13817010
35 changed files with 302245 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.gitattributes vendored Normal file
View File

@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

18
README.md Normal file
View File

@@ -0,0 +1,18 @@
---
license: mit
language:
- en
library_name: transformers
---
# ANWGPT-4.1
## How do use the model
```py
# Use a pipeline as a high-level helper
from transformers import pipeline
pipe = pipeline("text-generation", model="anwgpt/anwgpt4.1-base")
# Example of text generation (you can modify the input text)
print(pipe("The sky is blue because"))
```

31
checkpoint-7000/config.json Normal file
View File

@@ -0,0 +1,31 @@
{
"activation_function": "gelu_new",
"architectures": [
"GPT2LMHeadModel"
],
"attn_pdrop": 0.1,
"bos_token_id": 50256,
"dtype": "float32",
"embd_pdrop": 0.1,
"eos_token_id": 50256,
"initializer_range": 0.02,
"layer_norm_epsilon": 1e-05,
"model_type": "gpt2",
"n_embd": 384,
"n_head": 6,
"n_inner": null,
"n_layer": 6,
"n_positions": 2048,
"reorder_and_upcast_attn": false,
"resid_pdrop": 0.1,
"scale_attn_by_inverse_layer_idx": false,
"scale_attn_weights": true,
"summary_activation": null,
"summary_first_dropout": 0.1,
"summary_proj_to_labels": true,
"summary_type": "cls_index",
"summary_use_proj": true,
"transformers_version": "4.57.1",
"use_cache": false,
"vocab_size": 50257
}

7
checkpoint-7000/generation_config.json Normal file
View File

@@ -0,0 +1,7 @@
{
"_from_model_config": true,
"bos_token_id": 50256,
"eos_token_id": 50256,
"transformers_version": "4.57.1",
"use_cache": false
}

50001
checkpoint-7000/merges.txt Normal file
View File

File diff suppressed because it is too large Load Diff

3
checkpoint-7000/model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0e4e32e5ee99a731eb706db30057700b885f4dce982ed75c0d20e2389430249f
size 122938232

3
checkpoint-7000/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:537045eb297bca7731d09d7dba6cb39c566e08d36c7cd887a506b00f76363fbf
size 245925707

3
checkpoint-7000/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4067afacbdcda557d9d81f9061ae9d16f2c99e1068b4a6eddbaaad8e5d0a2580
size 14645

3
checkpoint-7000/scaler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:be40a03f70bb2a35bb567114fb5ceb05ce1222c8df24b1e1100746c617c5be10
size 1383

3
checkpoint-7000/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a0cd06cf0d538153d64d60f62ca54ddfb9b03bf00288a8cd70a3031d73dc37e4
size 1465

24
checkpoint-7000/special_tokens_map.json Normal file
View File

@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<|endoftext|>",
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

23
checkpoint-7000/tokenizer_config.json Normal file
View File

@@ -0,0 +1,23 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"50256": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|endoftext|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"errors": "replace",
"extra_special_tokens": {},
"model_max_length": 1024,
"pad_token": "<|endoftext|>",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<|endoftext|>"
}

524
checkpoint-7000/trainer_state.json Normal file
View File

@@ -0,0 +1,524 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 11.2,
"eval_steps": 500,
"global_step": 7000,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.16,
"grad_norm": 1.6197667121887207,
"learning_rate": 9.900000000000001e-05,
"loss": 9.494,
"step": 100
},
{
"epoch": 0.32,
"grad_norm": 0.8194359540939331,
"learning_rate": 0.000199,
"loss": 7.1961,
"step": 200
},
{
"epoch": 0.48,
"grad_norm": 0.8688249588012695,
"learning_rate": 0.000299,
"loss": 6.6152,
"step": 300
},
{
"epoch": 0.64,
"grad_norm": 0.8056670427322388,
"learning_rate": 0.00039900000000000005,
"loss": 6.3023,
"step": 400
},
{
"epoch": 0.8,
"grad_norm": 0.7805109620094299,
"learning_rate": 0.000499,
"loss": 6.0878,
"step": 500
},
{
"epoch": 0.96,
"grad_norm": 0.8030532598495483,
"learning_rate": 0.0004934000000000001,
"loss": 5.9005,
"step": 600
},
{
"epoch": 1.12,
"grad_norm": 0.7863243222236633,
"learning_rate": 0.00048673333333333336,
"loss": 5.6836,
"step": 700
},
{
"epoch": 1.28,
"grad_norm": 0.7835010886192322,
"learning_rate": 0.00048006666666666666,
"loss": 5.578,
"step": 800
},
{
"epoch": 1.44,
"grad_norm": 0.9478884935379028,
"learning_rate": 0.0004734,
"loss": 5.458,
"step": 900
},
{
"epoch": 1.6,
"grad_norm": 0.8334749341011047,
"learning_rate": 0.00046673333333333337,
"loss": 5.395,
"step": 1000
},
{
"epoch": 1.76,
"grad_norm": 0.8591320514678955,
"learning_rate": 0.00046006666666666666,
"loss": 5.2882,
"step": 1100
},
{
"epoch": 1.92,
"grad_norm": 0.7951042652130127,
"learning_rate": 0.0004534,
"loss": 5.2288,
"step": 1200
},
{
"epoch": 2.08,
"grad_norm": 0.8483077883720398,
"learning_rate": 0.0004467333333333333,
"loss": 5.0694,
"step": 1300
},
{
"epoch": 2.24,
"grad_norm": 0.956531822681427,
"learning_rate": 0.00044006666666666667,
"loss": 4.9456,
"step": 1400
},
{
"epoch": 2.4,
"grad_norm": 0.8443823456764221,
"learning_rate": 0.0004334,
"loss": 4.8984,
"step": 1500
},
{
"epoch": 2.56,
"grad_norm": 0.856013834476471,
"learning_rate": 0.00042673333333333337,
"loss": 4.847,
"step": 1600
},
{
"epoch": 2.7199999999999998,
"grad_norm": 0.8549163341522217,
"learning_rate": 0.00042006666666666667,
"loss": 4.8367,
"step": 1700
},
{
"epoch": 2.88,
"grad_norm": 0.878078818321228,
"learning_rate": 0.0004134,
"loss": 4.781,
"step": 1800
},
{
"epoch": 3.04,
"grad_norm": 0.9541525840759277,
"learning_rate": 0.0004067333333333333,
"loss": 4.6976,
"step": 1900
},
{
"epoch": 3.2,
"grad_norm": 0.9714905619621277,
"learning_rate": 0.00040006666666666667,
"loss": 4.5125,
"step": 2000
},
{
"epoch": 3.36,
"grad_norm": 0.9237401485443115,
"learning_rate": 0.0003934,
"loss": 4.4952,
"step": 2100
},
{
"epoch": 3.52,
"grad_norm": 0.9763756394386292,
"learning_rate": 0.0003867333333333333,
"loss": 4.5136,
"step": 2200
},
{
"epoch": 3.68,
"grad_norm": 0.9771287441253662,
"learning_rate": 0.00038006666666666667,
"loss": 4.4895,
"step": 2300
},
{
"epoch": 3.84,
"grad_norm": 0.9347721338272095,
"learning_rate": 0.0003734,
"loss": 4.4725,
"step": 2400
},
{
"epoch": 4.0,
"grad_norm": 0.9864392876625061,
"learning_rate": 0.0003667333333333334,
"loss": 4.4327,
"step": 2500
},
{
"epoch": 4.16,
"grad_norm": 1.008345127105713,
"learning_rate": 0.00036006666666666667,
"loss": 4.1863,
"step": 2600
},
{
"epoch": 4.32,
"grad_norm": 1.0036407709121704,
"learning_rate": 0.0003534,
"loss": 4.1979,
"step": 2700
},
{
"epoch": 4.48,
"grad_norm": 1.0768797397613525,
"learning_rate": 0.0003467333333333333,
"loss": 4.2055,
"step": 2800
},
{
"epoch": 4.64,
"grad_norm": 1.0015532970428467,
"learning_rate": 0.0003400666666666667,
"loss": 4.2196,
"step": 2900
},
{
"epoch": 4.8,
"grad_norm": 0.9601279497146606,
"learning_rate": 0.00033339999999999997,
"loss": 4.2178,
"step": 3000
},
{
"epoch": 4.96,
"grad_norm": 0.9527409672737122,
"learning_rate": 0.0003267333333333333,
"loss": 4.2151,
"step": 3100
},
{
"epoch": 5.12,
"grad_norm": 0.9995954632759094,
"learning_rate": 0.0003200666666666667,
"loss": 4.0079,
"step": 3200
},
{
"epoch": 5.28,
"grad_norm": 1.1739501953125,
"learning_rate": 0.00031340000000000003,
"loss": 3.9542,
"step": 3300
},
{
"epoch": 5.44,
"grad_norm": 0.9967206120491028,
"learning_rate": 0.0003067333333333334,
"loss": 3.9829,
"step": 3400
},
{
"epoch": 5.6,
"grad_norm": 0.9680889844894409,
"learning_rate": 0.0003000666666666667,
"loss": 3.9858,
"step": 3500
},
{
"epoch": 5.76,
"grad_norm": 1.0430840253829956,
"learning_rate": 0.0002934,
"loss": 3.9846,
"step": 3600
},
{
"epoch": 5.92,
"grad_norm": 1.1499149799346924,
"learning_rate": 0.00028673333333333333,
"loss": 3.9907,
"step": 3700
},
{
"epoch": 6.08,
"grad_norm": 1.0168757438659668,
"learning_rate": 0.0002800666666666667,
"loss": 3.8551,
"step": 3800
},
{
"epoch": 6.24,
"grad_norm": 0.9971639513969421,
"learning_rate": 0.0002734,
"loss": 3.7603,
"step": 3900
},
{
"epoch": 6.4,
"grad_norm": 1.039982795715332,
"learning_rate": 0.00026673333333333333,
"loss": 3.7583,
"step": 4000
},
{
"epoch": 6.5600000000000005,
"grad_norm": 1.0709551572799683,
"learning_rate": 0.0002600666666666667,
"loss": 3.7758,
"step": 4100
},
{
"epoch": 6.72,
"grad_norm": 1.1284174919128418,
"learning_rate": 0.00025340000000000003,
"loss": 3.8042,
"step": 4200
},
{
"epoch": 6.88,
"grad_norm": 1.1228001117706299,
"learning_rate": 0.00024673333333333333,
"loss": 3.8001,
"step": 4300
},
{
"epoch": 7.04,
"grad_norm": 1.1634491682052612,
"learning_rate": 0.00024006666666666668,
"loss": 3.7497,
"step": 4400
},
{
"epoch": 7.2,
"grad_norm": 1.0996577739715576,
"learning_rate": 0.0002334,
"loss": 3.5699,
"step": 4500
},
{
"epoch": 7.36,
"grad_norm": 1.104560136795044,
"learning_rate": 0.00022673333333333336,
"loss": 3.5893,
"step": 4600
},
{
"epoch": 7.52,
"grad_norm": 1.1617519855499268,
"learning_rate": 0.00022006666666666666,
"loss": 3.6109,
"step": 4700
},
{
"epoch": 7.68,
"grad_norm": 1.185733675956726,
"learning_rate": 0.0002134,
"loss": 3.6237,
"step": 4800
},
{
"epoch": 7.84,
"grad_norm": 1.1075682640075684,
"learning_rate": 0.00020673333333333333,
"loss": 3.6226,
"step": 4900
},
{
"epoch": 8.0,
"grad_norm": 1.1072136163711548,
"learning_rate": 0.0002000666666666667,
"loss": 3.6264,
"step": 5000
},
{
"epoch": 8.16,
"grad_norm": 1.1280772686004639,
"learning_rate": 0.00019339999999999998,
"loss": 3.4084,
"step": 5100
},
{
"epoch": 8.32,
"grad_norm": 1.1734412908554077,
"learning_rate": 0.00018673333333333334,
"loss": 3.4317,
"step": 5200
},
{
"epoch": 8.48,
"grad_norm": 1.1547329425811768,
"learning_rate": 0.0001800666666666667,
"loss": 3.4479,
"step": 5300
},
{
"epoch": 8.64,
"grad_norm": 1.096431016921997,
"learning_rate": 0.0001734,
"loss": 3.4681,
"step": 5400
},
{
"epoch": 8.8,
"grad_norm": 1.1974066495895386,
"learning_rate": 0.00016673333333333334,
"loss": 3.4715,
"step": 5500
},
{
"epoch": 8.96,
"grad_norm": 1.143074631690979,
"learning_rate": 0.00016006666666666666,
"loss": 3.4956,
"step": 5600
},
{
"epoch": 9.12,
"grad_norm": 1.1572178602218628,
"learning_rate": 0.00015340000000000002,
"loss": 3.3488,
"step": 5700
},
{
"epoch": 9.28,
"grad_norm": 1.242713451385498,
"learning_rate": 0.00014673333333333334,
"loss": 3.3075,
"step": 5800
},
{
"epoch": 9.44,
"grad_norm": 1.339081883430481,
"learning_rate": 0.00014006666666666667,
"loss": 3.3196,
"step": 5900
},
{
"epoch": 9.6,
"grad_norm": 1.204089879989624,
"learning_rate": 0.0001334,
"loss": 3.3323,
"step": 6000
},
{
"epoch": 9.76,
"grad_norm": 1.2573519945144653,
"learning_rate": 0.00012673333333333334,
"loss": 3.3428,
"step": 6100
},
{
"epoch": 9.92,
"grad_norm": 1.2417329549789429,
"learning_rate": 0.00012006666666666667,
"loss": 3.3543,
"step": 6200
},
{
"epoch": 10.08,
"grad_norm": 1.2444055080413818,
"learning_rate": 0.0001134,
"loss": 3.2588,
"step": 6300
},
{
"epoch": 10.24,
"grad_norm": 1.3446458578109741,
"learning_rate": 0.00010673333333333333,
"loss": 3.1825,
"step": 6400
},
{
"epoch": 10.4,
"grad_norm": 1.2423597574234009,
"learning_rate": 0.00010006666666666667,
"loss": 3.2007,
"step": 6500
},
{
"epoch": 10.56,
"grad_norm": 1.2347921133041382,
"learning_rate": 9.34e-05,
"loss": 3.225,
"step": 6600
},
{
"epoch": 10.72,
"grad_norm": 1.2625757455825806,
"learning_rate": 8.673333333333333e-05,
"loss": 3.2249,
"step": 6700
},
{
"epoch": 10.88,
"grad_norm": 1.29766845703125,
"learning_rate": 8.006666666666666e-05,
"loss": 3.234,
"step": 6800
},
{
"epoch": 11.04,
"grad_norm": 1.3144736289978027,
"learning_rate": 7.340000000000001e-05,
"loss": 3.2187,
"step": 6900
},
{
"epoch": 11.2,
"grad_norm": 1.2403204441070557,
"learning_rate": 6.673333333333333e-05,
"loss": 3.109,
"step": 7000
}
],
"logging_steps": 100,
"max_steps": 8000,
"num_input_tokens_seen": 0,
"num_train_epochs": 13,
"save_steps": 1000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 7326878662656000.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}

3
checkpoint-7000/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4c689aafec40f451565006e9e33219d9db6178a1f17b046866be648459b84f7b
size 5777

50259
checkpoint-7000/vocab.json Normal file
View File

File diff suppressed because it is too large Load Diff

31
checkpoint-8000/config.json Normal file
View File

@@ -0,0 +1,31 @@
{
"activation_function": "gelu_new",
"architectures": [
"GPT2LMHeadModel"
],
"attn_pdrop": 0.1,
"bos_token_id": 50256,
"dtype": "float32",
"embd_pdrop": 0.1,
"eos_token_id": 50256,
"initializer_range": 0.02,
"layer_norm_epsilon": 1e-05,
"model_type": "gpt2",
"n_embd": 384,
"n_head": 6,
"n_inner": null,
"n_layer": 6,
"n_positions": 2048,
"reorder_and_upcast_attn": false,
"resid_pdrop": 0.1,
"scale_attn_by_inverse_layer_idx": false,
"scale_attn_weights": true,
"summary_activation": null,
"summary_first_dropout": 0.1,
"summary_proj_to_labels": true,
"summary_type": "cls_index",
"summary_use_proj": true,
"transformers_version": "4.57.1",
"use_cache": false,
"vocab_size": 50257
}

7
checkpoint-8000/generation_config.json Normal file
View File

@@ -0,0 +1,7 @@
{
"_from_model_config": true,
"bos_token_id": 50256,
"eos_token_id": 50256,
"transformers_version": "4.57.1",
"use_cache": false
}

50001
checkpoint-8000/merges.txt Normal file
View File

File diff suppressed because it is too large Load Diff

3
checkpoint-8000/model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:15e89bcb98594cce8ca16f6798eb3a902dfdc384105333a2e422d278e9f94452
size 122938232

3
checkpoint-8000/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7588a68abd8a4357a36d3058167bebd9583730ab2df39a673e31a8cc6007564a
size 245925707

3
checkpoint-8000/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:557fd5b8bec7868d945d4da2365dde97d248bc32a4f1761df030485a6d5663e4
size 14645

3
checkpoint-8000/scaler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:022915fa8ac65fae0f13853519b8aa7ee79efd632a399c1f64c6b87bab0b057a
size 1383

3
checkpoint-8000/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ea49485dc4f593ec4aa8e9118bbedfae7fc2d017cd1a1c5e4682c9a55d2c9e21
size 1465

24
checkpoint-8000/special_tokens_map.json Normal file
View File

@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<|endoftext|>",
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

23
checkpoint-8000/tokenizer_config.json Normal file
View File

@@ -0,0 +1,23 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"50256": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|endoftext|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"errors": "replace",
"extra_special_tokens": {},
"model_max_length": 1024,
"pad_token": "<|endoftext|>",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<|endoftext|>"
}

594
checkpoint-8000/trainer_state.json Normal file
View File

@@ -0,0 +1,594 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 12.8,
"eval_steps": 500,
"global_step": 8000,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.16,
"grad_norm": 1.6197667121887207,
"learning_rate": 9.900000000000001e-05,
"loss": 9.494,
"step": 100
},
{
"epoch": 0.32,
"grad_norm": 0.8194359540939331,
"learning_rate": 0.000199,
"loss": 7.1961,
"step": 200
},
{
"epoch": 0.48,
"grad_norm": 0.8688249588012695,
"learning_rate": 0.000299,
"loss": 6.6152,
"step": 300
},
{
"epoch": 0.64,
"grad_norm": 0.8056670427322388,
"learning_rate": 0.00039900000000000005,
"loss": 6.3023,
"step": 400
},
{
"epoch": 0.8,
"grad_norm": 0.7805109620094299,
"learning_rate": 0.000499,
"loss": 6.0878,
"step": 500
},
{
"epoch": 0.96,
"grad_norm": 0.8030532598495483,
"learning_rate": 0.0004934000000000001,
"loss": 5.9005,
"step": 600
},
{
"epoch": 1.12,
"grad_norm": 0.7863243222236633,
"learning_rate": 0.00048673333333333336,
"loss": 5.6836,
"step": 700
},
{
"epoch": 1.28,
"grad_norm": 0.7835010886192322,
"learning_rate": 0.00048006666666666666,
"loss": 5.578,
"step": 800
},
{
"epoch": 1.44,
"grad_norm": 0.9478884935379028,
"learning_rate": 0.0004734,
"loss": 5.458,
"step": 900
},
{
"epoch": 1.6,
"grad_norm": 0.8334749341011047,
"learning_rate": 0.00046673333333333337,
"loss": 5.395,
"step": 1000
},
{
"epoch": 1.76,
"grad_norm": 0.8591320514678955,
"learning_rate": 0.00046006666666666666,
"loss": 5.2882,
"step": 1100
},
{
"epoch": 1.92,
"grad_norm": 0.7951042652130127,
"learning_rate": 0.0004534,
"loss": 5.2288,
"step": 1200
},
{
"epoch": 2.08,
"grad_norm": 0.8483077883720398,
"learning_rate": 0.0004467333333333333,
"loss": 5.0694,
"step": 1300
},
{
"epoch": 2.24,
"grad_norm": 0.956531822681427,
"learning_rate": 0.00044006666666666667,
"loss": 4.9456,
"step": 1400
},
{
"epoch": 2.4,
"grad_norm": 0.8443823456764221,
"learning_rate": 0.0004334,
"loss": 4.8984,
"step": 1500
},
{
"epoch": 2.56,
"grad_norm": 0.856013834476471,
"learning_rate": 0.00042673333333333337,
"loss": 4.847,
"step": 1600
},
{
"epoch": 2.7199999999999998,
"grad_norm": 0.8549163341522217,
"learning_rate": 0.00042006666666666667,
"loss": 4.8367,
"step": 1700
},
{
"epoch": 2.88,
"grad_norm": 0.878078818321228,
"learning_rate": 0.0004134,
"loss": 4.781,
"step": 1800
},
{
"epoch": 3.04,
"grad_norm": 0.9541525840759277,
"learning_rate": 0.0004067333333333333,
"loss": 4.6976,
"step": 1900
},
{
"epoch": 3.2,
"grad_norm": 0.9714905619621277,
"learning_rate": 0.00040006666666666667,
"loss": 4.5125,
"step": 2000
},
{
"epoch": 3.36,
"grad_norm": 0.9237401485443115,
"learning_rate": 0.0003934,
"loss": 4.4952,
"step": 2100
},
{
"epoch": 3.52,
"grad_norm": 0.9763756394386292,
"learning_rate": 0.0003867333333333333,
"loss": 4.5136,
"step": 2200
},
{
"epoch": 3.68,
"grad_norm": 0.9771287441253662,
"learning_rate": 0.00038006666666666667,
"loss": 4.4895,
"step": 2300
},
{
"epoch": 3.84,
"grad_norm": 0.9347721338272095,
"learning_rate": 0.0003734,
"loss": 4.4725,
"step": 2400
},
{
"epoch": 4.0,
"grad_norm": 0.9864392876625061,
"learning_rate": 0.0003667333333333334,
"loss": 4.4327,
"step": 2500
},
{
"epoch": 4.16,
"grad_norm": 1.008345127105713,
"learning_rate": 0.00036006666666666667,
"loss": 4.1863,
"step": 2600
},
{
"epoch": 4.32,
"grad_norm": 1.0036407709121704,
"learning_rate": 0.0003534,
"loss": 4.1979,
"step": 2700
},
{
"epoch": 4.48,
"grad_norm": 1.0768797397613525,
"learning_rate": 0.0003467333333333333,
"loss": 4.2055,
"step": 2800
},
{
"epoch": 4.64,
"grad_norm": 1.0015532970428467,
"learning_rate": 0.0003400666666666667,
"loss": 4.2196,
"step": 2900
},
{
"epoch": 4.8,
"grad_norm": 0.9601279497146606,
"learning_rate": 0.00033339999999999997,
"loss": 4.2178,
"step": 3000
},
{
"epoch": 4.96,
"grad_norm": 0.9527409672737122,
"learning_rate": 0.0003267333333333333,
"loss": 4.2151,
"step": 3100
},
{
"epoch": 5.12,
"grad_norm": 0.9995954632759094,
"learning_rate": 0.0003200666666666667,
"loss": 4.0079,
"step": 3200
},
{
"epoch": 5.28,
"grad_norm": 1.1739501953125,
"learning_rate": 0.00031340000000000003,
"loss": 3.9542,
"step": 3300
},
{
"epoch": 5.44,
"grad_norm": 0.9967206120491028,
"learning_rate": 0.0003067333333333334,
"loss": 3.9829,
"step": 3400
},
{
"epoch": 5.6,
"grad_norm": 0.9680889844894409,
"learning_rate": 0.0003000666666666667,
"loss": 3.9858,
"step": 3500
},
{
"epoch": 5.76,
"grad_norm": 1.0430840253829956,
"learning_rate": 0.0002934,
"loss": 3.9846,
"step": 3600
},
{
"epoch": 5.92,
"grad_norm": 1.1499149799346924,
"learning_rate": 0.00028673333333333333,
"loss": 3.9907,
"step": 3700
},
{
"epoch": 6.08,
"grad_norm": 1.0168757438659668,
"learning_rate": 0.0002800666666666667,
"loss": 3.8551,
"step": 3800
},
{
"epoch": 6.24,
"grad_norm": 0.9971639513969421,
"learning_rate": 0.0002734,
"loss": 3.7603,
"step": 3900
},
{
"epoch": 6.4,
"grad_norm": 1.039982795715332,
"learning_rate": 0.00026673333333333333,
"loss": 3.7583,
"step": 4000
},
{
"epoch": 6.5600000000000005,
"grad_norm": 1.0709551572799683,
"learning_rate": 0.0002600666666666667,
"loss": 3.7758,
"step": 4100
},
{
"epoch": 6.72,
"grad_norm": 1.1284174919128418,
"learning_rate": 0.00025340000000000003,
"loss": 3.8042,
"step": 4200
},
{
"epoch": 6.88,
"grad_norm": 1.1228001117706299,
"learning_rate": 0.00024673333333333333,
"loss": 3.8001,
"step": 4300
},
{
"epoch": 7.04,
"grad_norm": 1.1634491682052612,
"learning_rate": 0.00024006666666666668,
"loss": 3.7497,
"step": 4400
},
{
"epoch": 7.2,
"grad_norm": 1.0996577739715576,
"learning_rate": 0.0002334,
"loss": 3.5699,
"step": 4500
},
{
"epoch": 7.36,
"grad_norm": 1.104560136795044,
"learning_rate": 0.00022673333333333336,
"loss": 3.5893,
"step": 4600
},
{
"epoch": 7.52,
"grad_norm": 1.1617519855499268,
"learning_rate": 0.00022006666666666666,
"loss": 3.6109,
"step": 4700
},
{
"epoch": 7.68,
"grad_norm": 1.185733675956726,
"learning_rate": 0.0002134,
"loss": 3.6237,
"step": 4800
},
{
"epoch": 7.84,
"grad_norm": 1.1075682640075684,
"learning_rate": 0.00020673333333333333,
"loss": 3.6226,
"step": 4900
},
{
"epoch": 8.0,
"grad_norm": 1.1072136163711548,
"learning_rate": 0.0002000666666666667,
"loss": 3.6264,
"step": 5000
},
{
"epoch": 8.16,
"grad_norm": 1.1280772686004639,
"learning_rate": 0.00019339999999999998,
"loss": 3.4084,
"step": 5100
},
{
"epoch": 8.32,
"grad_norm": 1.1734412908554077,
"learning_rate": 0.00018673333333333334,
"loss": 3.4317,
"step": 5200
},
{
"epoch": 8.48,
"grad_norm": 1.1547329425811768,
"learning_rate": 0.0001800666666666667,
"loss": 3.4479,
"step": 5300
},
{
"epoch": 8.64,
"grad_norm": 1.096431016921997,
"learning_rate": 0.0001734,
"loss": 3.4681,
"step": 5400
},
{
"epoch": 8.8,
"grad_norm": 1.1974066495895386,
"learning_rate": 0.00016673333333333334,
"loss": 3.4715,
"step": 5500
},
{
"epoch": 8.96,
"grad_norm": 1.143074631690979,
"learning_rate": 0.00016006666666666666,
"loss": 3.4956,
"step": 5600
},
{
"epoch": 9.12,
"grad_norm": 1.1572178602218628,
"learning_rate": 0.00015340000000000002,
"loss": 3.3488,
"step": 5700
},
{
"epoch": 9.28,
"grad_norm": 1.242713451385498,
"learning_rate": 0.00014673333333333334,
"loss": 3.3075,
"step": 5800
},
{
"epoch": 9.44,
"grad_norm": 1.339081883430481,
"learning_rate": 0.00014006666666666667,
"loss": 3.3196,
"step": 5900
},
{
"epoch": 9.6,
"grad_norm": 1.204089879989624,
"learning_rate": 0.0001334,
"loss": 3.3323,
"step": 6000
},
{
"epoch": 9.76,
"grad_norm": 1.2573519945144653,
"learning_rate": 0.00012673333333333334,
"loss": 3.3428,
"step": 6100
},
{
"epoch": 9.92,
"grad_norm": 1.2417329549789429,
"learning_rate": 0.00012006666666666667,
"loss": 3.3543,
"step": 6200
},
{
"epoch": 10.08,
"grad_norm": 1.2444055080413818,
"learning_rate": 0.0001134,
"loss": 3.2588,
"step": 6300
},
{
"epoch": 10.24,
"grad_norm": 1.3446458578109741,
"learning_rate": 0.00010673333333333333,
"loss": 3.1825,
"step": 6400
},
{
"epoch": 10.4,
"grad_norm": 1.2423597574234009,
"learning_rate": 0.00010006666666666667,
"loss": 3.2007,
"step": 6500
},
{
"epoch": 10.56,
"grad_norm": 1.2347921133041382,
"learning_rate": 9.34e-05,
"loss": 3.225,
"step": 6600
},
{
"epoch": 10.72,
"grad_norm": 1.2625757455825806,
"learning_rate": 8.673333333333333e-05,
"loss": 3.2249,
"step": 6700
},
{
"epoch": 10.88,
"grad_norm": 1.29766845703125,
"learning_rate": 8.006666666666666e-05,
"loss": 3.234,
"step": 6800
},
{
"epoch": 11.04,
"grad_norm": 1.3144736289978027,
"learning_rate": 7.340000000000001e-05,
"loss": 3.2187,
"step": 6900
},
{
"epoch": 11.2,
"grad_norm": 1.2403204441070557,
"learning_rate": 6.673333333333333e-05,
"loss": 3.109,
"step": 7000
},
{
"epoch": 11.36,
"grad_norm": 1.2787344455718994,
"learning_rate": 6.0066666666666665e-05,
"loss": 3.13,
"step": 7100
},
{
"epoch": 11.52,
"grad_norm": 1.2818809747695923,
"learning_rate": 5.3400000000000004e-05,
"loss": 3.1219,
"step": 7200
},
{
"epoch": 11.68,
"grad_norm": 1.39326810836792,
"learning_rate": 4.6733333333333335e-05,
"loss": 3.1198,
"step": 7300
},
{
"epoch": 11.84,
"grad_norm": 1.3103387355804443,
"learning_rate": 4.006666666666667e-05,
"loss": 3.1388,
"step": 7400
},
{
"epoch": 12.0,
"grad_norm": 1.3130543231964111,
"learning_rate": 3.34e-05,
"loss": 3.1268,
"step": 7500
},
{
"epoch": 12.16,
"grad_norm": 1.2466676235198975,
"learning_rate": 2.6733333333333334e-05,
"loss": 3.0555,
"step": 7600
},
{
"epoch": 12.32,
"grad_norm": 1.2399988174438477,
"learning_rate": 2.0066666666666665e-05,
"loss": 3.0536,
"step": 7700
},
{
"epoch": 12.48,
"grad_norm": 1.3190345764160156,
"learning_rate": 1.34e-05,
"loss": 3.0576,
"step": 7800
},
{
"epoch": 12.64,
"grad_norm": 1.2856481075286865,
"learning_rate": 6.733333333333334e-06,
"loss": 3.0583,
"step": 7900
},
{
"epoch": 12.8,
"grad_norm": 1.4591535329818726,
"learning_rate": 6.666666666666667e-08,
"loss": 3.0561,
"step": 8000
}
],
"logging_steps": 100,
"max_steps": 8000,
"num_input_tokens_seen": 0,
"num_train_epochs": 13,
"save_steps": 1000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 8373575614464000.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}

3
checkpoint-8000/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4c689aafec40f451565006e9e33219d9db6178a1f17b046866be648459b84f7b
size 5777

50259
checkpoint-8000/vocab.json Normal file
View File

File diff suppressed because it is too large Load Diff

31
config.json Normal file
View File

@@ -0,0 +1,31 @@
{
"activation_function": "gelu_new",
"architectures": [
"GPT2LMHeadModel"
],
"attn_pdrop": 0.1,
"bos_token_id": 50256,
"dtype": "float32",
"embd_pdrop": 0.1,
"eos_token_id": 50256,
"initializer_range": 0.02,
"layer_norm_epsilon": 1e-05,
"model_type": "gpt2",
"n_embd": 384,
"n_head": 6,
"n_inner": null,
"n_layer": 6,
"n_positions": 2048,
"reorder_and_upcast_attn": false,
"resid_pdrop": 0.1,
"scale_attn_by_inverse_layer_idx": false,
"scale_attn_weights": true,
"summary_activation": null,
"summary_first_dropout": 0.1,
"summary_proj_to_labels": true,
"summary_type": "cls_index",
"summary_use_proj": true,
"transformers_version": "4.57.1",
"use_cache": false,
"vocab_size": 50257
}

7
generation_config.json Normal file
View File

@@ -0,0 +1,7 @@
{
"_from_model_config": true,
"bos_token_id": 50256,
"eos_token_id": 50256,
"transformers_version": "4.57.1",
"use_cache": false
}

50001
merges.txt Normal file
View File

File diff suppressed because it is too large Load Diff

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:15e89bcb98594cce8ca16f6798eb3a902dfdc384105333a2e422d278e9f94452
size 122938232

24
special_tokens_map.json Normal file
View File

@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<|endoftext|>",
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

23
tokenizer_config.json Normal file
View File

@@ -0,0 +1,23 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"50256": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|endoftext|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"errors": "replace",
"extra_special_tokens": {},
"model_max_length": 1024,
"pad_token": "<|endoftext|>",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<|endoftext|>"
}

50259
vocab.json Normal file
View File

File diff suppressed because it is too large Load Diff