commit 9fec572eda11c3b11bd4435bc2ce1cf56580b65e
Author: ModelHub XC <noreply@modelhub.org.cn>
Date:   Mon Jul 27 22:42:09 2026 +0800

    初始化项目，由ModelHub XC社区提供模型
    
    Model: Omaratef3221/llama-3.1-8b-s1-none-s2-full-medarabench
    Source: Original Platform

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..2d37532
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,39 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1048/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2096/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3144/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..c0d052a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,58 @@
+---
+base_model: meta-llama/Llama-3.1-8B
+library_name: transformers
+model_name: stage2
+tags:
+- generated_from_trainer
+- sft
+- trl
+licence: license
+---
+
+# Model Card for stage2
+
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/omaratef3221/arabic-medical-llm-v3/runs/yctte084) 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 1.0.0
+- Transformers: 5.5.1
+- Pytorch: 2.6.0
+- Datasets: 4.8.4
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```
\ No newline at end of file
diff --git a/checkpoint-1048/config.json b/checkpoint-1048/config.json
new file mode 100644
index 0000000..7580fae
--- /dev/null
+++ b/checkpoint-1048/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128001,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-1048/generation_config.json b/checkpoint-1048/generation_config.json
new file mode 100644
index 0000000..c895560
--- /dev/null
+++ b/checkpoint-1048/generation_config.json
@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001
+  ],
+  "pad_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.5.1"
+}
diff --git a/checkpoint-1048/model.safetensors b/checkpoint-1048/model.safetensors
new file mode 100644
index 0000000..55ba81c
--- /dev/null
+++ b/checkpoint-1048/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33ee5d68157c16d9e49a35172b0a6e12b4f990003536c3eb92ac7642ed2b40fb
+size 16060556616
diff --git a/checkpoint-1048/optimizer.pt b/checkpoint-1048/optimizer.pt
new file mode 100644
index 0000000..69b256c
--- /dev/null
+++ b/checkpoint-1048/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d27fed11b4f010a0b2712a7ec61175c501558e94ed3b39317bd57f099d35867e
+size 19447450244
diff --git a/checkpoint-1048/rng_state.pth b/checkpoint-1048/rng_state.pth
new file mode 100644
index 0000000..ece6750
--- /dev/null
+++ b/checkpoint-1048/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b66e3cc7c452b707ddac5caf0aa17618afb9bc1a0333600a22c4afb353f3165
+size 14244
diff --git a/checkpoint-1048/scheduler.pt b/checkpoint-1048/scheduler.pt
new file mode 100644
index 0000000..61667fe
--- /dev/null
+++ b/checkpoint-1048/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e055c76c3e507738a50686f34ccf1e65e5c1708dac873ebc229ca797da4d02c9
+size 1064
diff --git a/checkpoint-1048/tokenizer.json b/checkpoint-1048/tokenizer.json
new file mode 100644
index 0000000..1c1d8d5
--- /dev/null
+++ b/checkpoint-1048/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-1048/tokenizer_config.json b/checkpoint-1048/tokenizer_config.json
new file mode 100644
index 0000000..f11592a
--- /dev/null
+++ b/checkpoint-1048/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-1048/trainer_state.json b/checkpoint-1048/trainer_state.json
new file mode 100644
index 0000000..605ea30
--- /dev/null
+++ b/checkpoint-1048/trainer_state.json
@@ -0,0 +1,245 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1048,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.0116909193992614,
+      "epoch": 0.047744091668656,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 2.0623187255859374,
+      "mean_token_accuracy": 0.589528747946024,
+      "num_tokens": 62415.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.7590599358081818,
+      "epoch": 0.095488183337312,
+      "grad_norm": 3.078125,
+      "learning_rate": 1.9999915067429385e-05,
+      "loss": 1.754022216796875,
+      "mean_token_accuracy": 0.6332476472854615,
+      "num_tokens": 126004.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.7224605268239974,
+      "epoch": 0.14323227500596802,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.9984525009996967e-05,
+      "loss": 1.7351203918457032,
+      "mean_token_accuracy": 0.6340418252348899,
+      "num_tokens": 188614.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.6983880323171616,
+      "epoch": 0.190976366674624,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.9942640420376764e-05,
+      "loss": 1.7146597290039063,
+      "mean_token_accuracy": 0.6401417210698128,
+      "num_tokens": 251062.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.6483143484592437,
+      "epoch": 0.23872045834328,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.9874372441823628e-05,
+      "loss": 1.658107147216797,
+      "mean_token_accuracy": 0.6478794780373573,
+      "num_tokens": 313957.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.6347757685184479,
+      "epoch": 0.28646455001193605,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.9779902227487397e-05,
+      "loss": 1.6550674438476562,
+      "mean_token_accuracy": 0.6487274625897408,
+      "num_tokens": 376779.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.5762060970067977,
+      "epoch": 0.334208641680592,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9659480459712203e-05,
+      "loss": 1.5960990905761718,
+      "mean_token_accuracy": 0.6587067684531211,
+      "num_tokens": 439472.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.534512039422989,
+      "epoch": 0.381952733349248,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.9513426684835922e-05,
+      "loss": 1.55486572265625,
+      "mean_token_accuracy": 0.6660523465275765,
+      "num_tokens": 501279.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.5402596753835678,
+      "epoch": 0.429696825017904,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.9342128465254943e-05,
+      "loss": 1.5535296630859374,
+      "mean_token_accuracy": 0.6673906499147415,
+      "num_tokens": 564442.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.5164723497629167,
+      "epoch": 0.47744091668656,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9146040351004225e-05,
+      "loss": 1.5373231506347655,
+      "mean_token_accuracy": 0.670901304781437,
+      "num_tokens": 627102.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.4891952139139175,
+      "epoch": 0.5251850083552161,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.8925682673581706e-05,
+      "loss": 1.5036320495605469,
+      "mean_token_accuracy": 0.6744937270879745,
+      "num_tokens": 689421.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.488253089785576,
+      "epoch": 0.5729291000238721,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.868164016521766e-05,
+      "loss": 1.5044355773925782,
+      "mean_token_accuracy": 0.6755094349384307,
+      "num_tokens": 752388.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.4446531629562378,
+      "epoch": 0.6206731916925281,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.8414560407252874e-05,
+      "loss": 1.4617152404785156,
+      "mean_token_accuracy": 0.6805451279878616,
+      "num_tokens": 815803.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.4555100780725478,
+      "epoch": 0.668417283361184,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.8125152111742944e-05,
+      "loss": 1.469644775390625,
+      "mean_token_accuracy": 0.6796707132458687,
+      "num_tokens": 878686.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.414481201171875,
+      "epoch": 0.71616137502984,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.7814183240848635e-05,
+      "loss": 1.4264332580566406,
+      "mean_token_accuracy": 0.6893567860126495,
+      "num_tokens": 940629.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3958445823192596,
+      "epoch": 0.763905466698496,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.7482478969002487e-05,
+      "loss": 1.4121397399902345,
+      "mean_token_accuracy": 0.6902168181538582,
+      "num_tokens": 1002810.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.3906721925735475,
+      "epoch": 0.811649558367152,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.7130919493259284e-05,
+      "loss": 1.3895584106445313,
+      "mean_token_accuracy": 0.6943912228941918,
+      "num_tokens": 1067041.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.3663516825437545,
+      "epoch": 0.859393650035808,
+      "grad_norm": 2.375,
+      "learning_rate": 1.676043769764072e-05,
+      "loss": 1.3753732299804688,
+      "mean_token_accuracy": 0.6982124423980713,
+      "num_tokens": 1130309.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.370076591670513,
+      "epoch": 0.907137741704464,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.637201667767204e-05,
+      "loss": 1.3839447021484375,
+      "mean_token_accuracy": 0.6986603051424026,
+      "num_tokens": 1192924.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.3369020774960518,
+      "epoch": 0.95488183337312,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.5966687131679463e-05,
+      "loss": 1.3427734375,
+      "mean_token_accuracy": 0.7060227099061013,
+      "num_tokens": 1254418.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.2727481353174803,
+      "eval_loss": 1.291359543800354,
+      "eval_mean_token_accuracy": 0.7115251171210456,
+      "eval_num_tokens": 1314012.0,
+      "eval_runtime": 52.7795,
+      "eval_samples_per_second": 16.711,
+      "eval_steps_per_second": 16.711,
+      "step": 1048
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3144,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.962196730295091e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-1048/training_args.bin b/checkpoint-1048/training_args.bin
new file mode 100644
index 0000000..49930dc
--- /dev/null
+++ b/checkpoint-1048/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a3ea84f94e96c81578a690800ff8f79d92cb1baf91ae820778832d6af60f008
+size 5304
diff --git a/checkpoint-2096/config.json b/checkpoint-2096/config.json
new file mode 100644
index 0000000..7580fae
--- /dev/null
+++ b/checkpoint-2096/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128001,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-2096/generation_config.json b/checkpoint-2096/generation_config.json
new file mode 100644
index 0000000..c895560
--- /dev/null
+++ b/checkpoint-2096/generation_config.json
@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001
+  ],
+  "pad_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.5.1"
+}
diff --git a/checkpoint-2096/model.safetensors b/checkpoint-2096/model.safetensors
new file mode 100644
index 0000000..26d4e70
--- /dev/null
+++ b/checkpoint-2096/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:844a8e251caaa84cdfb8e86ae9acbd329b8415f7a86e69abed6148d65f8e97f1
+size 16060556616
diff --git a/checkpoint-2096/optimizer.pt b/checkpoint-2096/optimizer.pt
new file mode 100644
index 0000000..044204d
--- /dev/null
+++ b/checkpoint-2096/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae03b75e4c92461e513d78ac37cc90727d5e4cb5bccc09170d0248242b208b97
+size 19447450244
diff --git a/checkpoint-2096/rng_state.pth b/checkpoint-2096/rng_state.pth
new file mode 100644
index 0000000..a8d9040
--- /dev/null
+++ b/checkpoint-2096/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f37c40ce327861a7ca13b719d3aa37510a143368b6e74358bdb14becb3899e1e
+size 14244
diff --git a/checkpoint-2096/scheduler.pt b/checkpoint-2096/scheduler.pt
new file mode 100644
index 0000000..dd128fd
--- /dev/null
+++ b/checkpoint-2096/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d76fed4e1336b28b263f3818f8a8cb0f32d2a137ecbb0cf42e66fa555f9c960
+size 1064
diff --git a/checkpoint-2096/tokenizer.json b/checkpoint-2096/tokenizer.json
new file mode 100644
index 0000000..1c1d8d5
--- /dev/null
+++ b/checkpoint-2096/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-2096/tokenizer_config.json b/checkpoint-2096/tokenizer_config.json
new file mode 100644
index 0000000..f11592a
--- /dev/null
+++ b/checkpoint-2096/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-2096/trainer_state.json b/checkpoint-2096/trainer_state.json
new file mode 100644
index 0000000..f4efc66
--- /dev/null
+++ b/checkpoint-2096/trainer_state.json
@@ -0,0 +1,466 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2096,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.0116909193992614,
+      "epoch": 0.047744091668656,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 2.0623187255859374,
+      "mean_token_accuracy": 0.589528747946024,
+      "num_tokens": 62415.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.7590599358081818,
+      "epoch": 0.095488183337312,
+      "grad_norm": 3.078125,
+      "learning_rate": 1.9999915067429385e-05,
+      "loss": 1.754022216796875,
+      "mean_token_accuracy": 0.6332476472854615,
+      "num_tokens": 126004.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.7224605268239974,
+      "epoch": 0.14323227500596802,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.9984525009996967e-05,
+      "loss": 1.7351203918457032,
+      "mean_token_accuracy": 0.6340418252348899,
+      "num_tokens": 188614.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.6983880323171616,
+      "epoch": 0.190976366674624,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.9942640420376764e-05,
+      "loss": 1.7146597290039063,
+      "mean_token_accuracy": 0.6401417210698128,
+      "num_tokens": 251062.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.6483143484592437,
+      "epoch": 0.23872045834328,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.9874372441823628e-05,
+      "loss": 1.658107147216797,
+      "mean_token_accuracy": 0.6478794780373573,
+      "num_tokens": 313957.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.6347757685184479,
+      "epoch": 0.28646455001193605,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.9779902227487397e-05,
+      "loss": 1.6550674438476562,
+      "mean_token_accuracy": 0.6487274625897408,
+      "num_tokens": 376779.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.5762060970067977,
+      "epoch": 0.334208641680592,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9659480459712203e-05,
+      "loss": 1.5960990905761718,
+      "mean_token_accuracy": 0.6587067684531211,
+      "num_tokens": 439472.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.534512039422989,
+      "epoch": 0.381952733349248,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.9513426684835922e-05,
+      "loss": 1.55486572265625,
+      "mean_token_accuracy": 0.6660523465275765,
+      "num_tokens": 501279.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.5402596753835678,
+      "epoch": 0.429696825017904,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.9342128465254943e-05,
+      "loss": 1.5535296630859374,
+      "mean_token_accuracy": 0.6673906499147415,
+      "num_tokens": 564442.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.5164723497629167,
+      "epoch": 0.47744091668656,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9146040351004225e-05,
+      "loss": 1.5373231506347655,
+      "mean_token_accuracy": 0.670901304781437,
+      "num_tokens": 627102.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.4891952139139175,
+      "epoch": 0.5251850083552161,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.8925682673581706e-05,
+      "loss": 1.5036320495605469,
+      "mean_token_accuracy": 0.6744937270879745,
+      "num_tokens": 689421.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.488253089785576,
+      "epoch": 0.5729291000238721,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.868164016521766e-05,
+      "loss": 1.5044355773925782,
+      "mean_token_accuracy": 0.6755094349384307,
+      "num_tokens": 752388.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.4446531629562378,
+      "epoch": 0.6206731916925281,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.8414560407252874e-05,
+      "loss": 1.4617152404785156,
+      "mean_token_accuracy": 0.6805451279878616,
+      "num_tokens": 815803.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.4555100780725478,
+      "epoch": 0.668417283361184,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.8125152111742944e-05,
+      "loss": 1.469644775390625,
+      "mean_token_accuracy": 0.6796707132458687,
+      "num_tokens": 878686.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.414481201171875,
+      "epoch": 0.71616137502984,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.7814183240848635e-05,
+      "loss": 1.4264332580566406,
+      "mean_token_accuracy": 0.6893567860126495,
+      "num_tokens": 940629.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3958445823192596,
+      "epoch": 0.763905466698496,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.7482478969002487e-05,
+      "loss": 1.4121397399902345,
+      "mean_token_accuracy": 0.6902168181538582,
+      "num_tokens": 1002810.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.3906721925735475,
+      "epoch": 0.811649558367152,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.7130919493259284e-05,
+      "loss": 1.3895584106445313,
+      "mean_token_accuracy": 0.6943912228941918,
+      "num_tokens": 1067041.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.3663516825437545,
+      "epoch": 0.859393650035808,
+      "grad_norm": 2.375,
+      "learning_rate": 1.676043769764072e-05,
+      "loss": 1.3753732299804688,
+      "mean_token_accuracy": 0.6982124423980713,
+      "num_tokens": 1130309.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.370076591670513,
+      "epoch": 0.907137741704464,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.637201667767204e-05,
+      "loss": 1.3839447021484375,
+      "mean_token_accuracy": 0.6986603051424026,
+      "num_tokens": 1192924.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.3369020774960518,
+      "epoch": 0.95488183337312,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.5966687131679463e-05,
+      "loss": 1.3427734375,
+      "mean_token_accuracy": 0.7060227099061013,
+      "num_tokens": 1254418.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.2727481353174803,
+      "eval_loss": 1.291359543800354,
+      "eval_mean_token_accuracy": 0.7115251171210456,
+      "eval_num_tokens": 1314012.0,
+      "eval_runtime": 52.7795,
+      "eval_samples_per_second": 16.711,
+      "eval_steps_per_second": 16.711,
+      "step": 1048
+    },
+    {
+      "entropy": 1.3094463989819367,
+      "epoch": 1.0019097636667462,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.554552462577077e-05,
+      "loss": 1.3102546691894532,
+      "mean_token_accuracy": 0.7078821743805397,
+      "num_tokens": 1316679.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9862651464343071,
+      "epoch": 1.0496538553354022,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.510964673975651e-05,
+      "loss": 0.94318603515625,
+      "mean_token_accuracy": 0.7765462607145309,
+      "num_tokens": 1379361.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9908152732253075,
+      "epoch": 1.0973979470040582,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.4660210101585367e-05,
+      "loss": 0.9593799591064454,
+      "mean_token_accuracy": 0.7724591562151909,
+      "num_tokens": 1442332.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.9940855634212494,
+      "epoch": 1.1451420386727142,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.4198407318162977e-05,
+      "loss": 0.9572904968261718,
+      "mean_token_accuracy": 0.7741717839241028,
+      "num_tokens": 1506048.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.997409186065197,
+      "epoch": 1.1928861303413703,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.3725463810698418e-05,
+      "loss": 0.962403564453125,
+      "mean_token_accuracy": 0.7725697284936905,
+      "num_tokens": 1569529.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.9752482569217682,
+      "epoch": 1.2406302220100263,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.3242634562975988e-05,
+      "loss": 0.9287136840820313,
+      "mean_token_accuracy": 0.7795223397016525,
+      "num_tokens": 1632744.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.972535012960434,
+      "epoch": 1.2883743136786823,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.2751200791180963e-05,
+      "loss": 0.9352114105224609,
+      "mean_token_accuracy": 0.7785275393724441,
+      "num_tokens": 1695454.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.9388299316167832,
+      "epoch": 1.3361184053473383,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.225246654411605e-05,
+      "loss": 0.9005762481689453,
+      "mean_token_accuracy": 0.7845277190208435,
+      "num_tokens": 1759868.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.9462847051024437,
+      "epoch": 1.3838624970159943,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.1747755242830204e-05,
+      "loss": 0.9043065643310547,
+      "mean_token_accuracy": 0.782291199862957,
+      "num_tokens": 1820879.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.9308868950605392,
+      "epoch": 1.4316065886846503,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.1238406168841981e-05,
+      "loss": 0.89322265625,
+      "mean_token_accuracy": 0.7880325165390968,
+      "num_tokens": 1883307.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.9272586759924889,
+      "epoch": 1.4793506803533063,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.0725770910276218e-05,
+      "loss": 0.8859436798095703,
+      "mean_token_accuracy": 0.7878314164280892,
+      "num_tokens": 1946095.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.9405490574240685,
+      "epoch": 1.5270947720219623,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.0211209775344377e-05,
+      "loss": 0.8991239929199218,
+      "mean_token_accuracy": 0.787108507156372,
+      "num_tokens": 2009039.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.9246599149703979,
+      "epoch": 1.5748388636906183,
+      "grad_norm": 2.1875,
+      "learning_rate": 9.696088182685638e-06,
+      "loss": 0.8893150329589844,
+      "mean_token_accuracy": 0.787010528743267,
+      "num_tokens": 2070866.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.9010607555508614,
+      "epoch": 1.6225829553592743,
+      "grad_norm": 2.5625,
+      "learning_rate": 9.18177303814717e-06,
+      "loss": 0.8644073486328125,
+      "mean_token_accuracy": 0.7935708364844323,
+      "num_tokens": 2133531.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.9290094000101089,
+      "epoch": 1.6703270470279303,
+      "grad_norm": 1.984375,
+      "learning_rate": 8.66962910761798e-06,
+      "loss": 0.8792234039306641,
+      "mean_token_accuracy": 0.7916052675247193,
+      "num_tokens": 2196212.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.9067334878444672,
+      "epoch": 1.7180711386965863,
+      "grad_norm": 2.0625,
+      "learning_rate": 8.161015395541311e-06,
+      "loss": 0.8666587829589844,
+      "mean_token_accuracy": 0.7931482782959938,
+      "num_tokens": 2259117.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.9192495954036712,
+      "epoch": 1.7658152303652423,
+      "grad_norm": 2.421875,
+      "learning_rate": 7.65728153871533e-06,
+      "loss": 0.8764252471923828,
+      "mean_token_accuracy": 0.792299057841301,
+      "num_tokens": 2320595.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.9041965058445931,
+      "epoch": 1.8135593220338984,
+      "grad_norm": 2.34375,
+      "learning_rate": 7.159764224951467e-06,
+      "loss": 0.8590383148193359,
+      "mean_token_accuracy": 0.7972308859229088,
+      "num_tokens": 2384167.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.9144112569093704,
+      "epoch": 1.8613034137025544,
+      "grad_norm": 2.21875,
+      "learning_rate": 6.669783646093635e-06,
+      "loss": 0.8813047790527344,
+      "mean_token_accuracy": 0.7919116005301475,
+      "num_tokens": 2444983.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.8830939248204231,
+      "epoch": 1.9090475053712104,
+      "grad_norm": 2.15625,
+      "learning_rate": 6.188639994810476e-06,
+      "loss": 0.8278516387939453,
+      "mean_token_accuracy": 0.8022209334373475,
+      "num_tokens": 2508413.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.9015234977006912,
+      "epoch": 1.9567915970398664,
+      "grad_norm": 2.1875,
+      "learning_rate": 5.717610014456644e-06,
+      "loss": 0.8635023498535156,
+      "mean_token_accuracy": 0.7982774981856346,
+      "num_tokens": 2570521.0,
+      "step": 2050
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 1.0000475248545746,
+      "eval_loss": 1.178954005241394,
+      "eval_mean_token_accuracy": 0.74215727841773,
+      "eval_num_tokens": 2628024.0,
+      "eval_runtime": 52.7405,
+      "eval_samples_per_second": 16.723,
+      "eval_steps_per_second": 16.723,
+      "step": 2096
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3144,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.5924789720612864e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-2096/training_args.bin b/checkpoint-2096/training_args.bin
new file mode 100644
index 0000000..49930dc
--- /dev/null
+++ b/checkpoint-2096/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a3ea84f94e96c81578a690800ff8f79d92cb1baf91ae820778832d6af60f008
+size 5304
diff --git a/checkpoint-3144/config.json b/checkpoint-3144/config.json
new file mode 100644
index 0000000..7580fae
--- /dev/null
+++ b/checkpoint-3144/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128001,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-3144/generation_config.json b/checkpoint-3144/generation_config.json
new file mode 100644
index 0000000..c895560
--- /dev/null
+++ b/checkpoint-3144/generation_config.json
@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001
+  ],
+  "pad_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.5.1"
+}
diff --git a/checkpoint-3144/model.safetensors b/checkpoint-3144/model.safetensors
new file mode 100644
index 0000000..14edbef
--- /dev/null
+++ b/checkpoint-3144/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31aeeebe856fb321b73f53a4d6587fa5e98aca83b451a9377728b509fd7c5c2a
+size 16060556616
diff --git a/checkpoint-3144/optimizer.pt b/checkpoint-3144/optimizer.pt
new file mode 100644
index 0000000..631c7f9
--- /dev/null
+++ b/checkpoint-3144/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce2d59fe944a5da110617f9058cb006c5c650f8d55680de8540fc0a43660b4f
+size 19447450244
diff --git a/checkpoint-3144/rng_state.pth b/checkpoint-3144/rng_state.pth
new file mode 100644
index 0000000..53b3a4c
--- /dev/null
+++ b/checkpoint-3144/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b6047bd8cc6f4cdf7c46dea47edb8e542435510070c6cd1e0a7d9ccf5fd7da
+size 14244
diff --git a/checkpoint-3144/scheduler.pt b/checkpoint-3144/scheduler.pt
new file mode 100644
index 0000000..8735e4a
--- /dev/null
+++ b/checkpoint-3144/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9259a74c2eaec0a1516c8c6e9f3df15c4d45979af94435ed693a2eab3ac86f28
+size 1064
diff --git a/checkpoint-3144/tokenizer.json b/checkpoint-3144/tokenizer.json
new file mode 100644
index 0000000..1c1d8d5
--- /dev/null
+++ b/checkpoint-3144/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-3144/tokenizer_config.json b/checkpoint-3144/tokenizer_config.json
new file mode 100644
index 0000000..f11592a
--- /dev/null
+++ b/checkpoint-3144/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-3144/trainer_state.json b/checkpoint-3144/trainer_state.json
new file mode 100644
index 0000000..5848918
--- /dev/null
+++ b/checkpoint-3144/trainer_state.json
@@ -0,0 +1,687 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 3144,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.0116909193992614,
+      "epoch": 0.047744091668656,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 2.0623187255859374,
+      "mean_token_accuracy": 0.589528747946024,
+      "num_tokens": 62415.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.7590599358081818,
+      "epoch": 0.095488183337312,
+      "grad_norm": 3.078125,
+      "learning_rate": 1.9999915067429385e-05,
+      "loss": 1.754022216796875,
+      "mean_token_accuracy": 0.6332476472854615,
+      "num_tokens": 126004.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.7224605268239974,
+      "epoch": 0.14323227500596802,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.9984525009996967e-05,
+      "loss": 1.7351203918457032,
+      "mean_token_accuracy": 0.6340418252348899,
+      "num_tokens": 188614.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.6983880323171616,
+      "epoch": 0.190976366674624,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.9942640420376764e-05,
+      "loss": 1.7146597290039063,
+      "mean_token_accuracy": 0.6401417210698128,
+      "num_tokens": 251062.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.6483143484592437,
+      "epoch": 0.23872045834328,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.9874372441823628e-05,
+      "loss": 1.658107147216797,
+      "mean_token_accuracy": 0.6478794780373573,
+      "num_tokens": 313957.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.6347757685184479,
+      "epoch": 0.28646455001193605,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.9779902227487397e-05,
+      "loss": 1.6550674438476562,
+      "mean_token_accuracy": 0.6487274625897408,
+      "num_tokens": 376779.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.5762060970067977,
+      "epoch": 0.334208641680592,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9659480459712203e-05,
+      "loss": 1.5960990905761718,
+      "mean_token_accuracy": 0.6587067684531211,
+      "num_tokens": 439472.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.534512039422989,
+      "epoch": 0.381952733349248,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.9513426684835922e-05,
+      "loss": 1.55486572265625,
+      "mean_token_accuracy": 0.6660523465275765,
+      "num_tokens": 501279.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.5402596753835678,
+      "epoch": 0.429696825017904,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.9342128465254943e-05,
+      "loss": 1.5535296630859374,
+      "mean_token_accuracy": 0.6673906499147415,
+      "num_tokens": 564442.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.5164723497629167,
+      "epoch": 0.47744091668656,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9146040351004225e-05,
+      "loss": 1.5373231506347655,
+      "mean_token_accuracy": 0.670901304781437,
+      "num_tokens": 627102.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.4891952139139175,
+      "epoch": 0.5251850083552161,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.8925682673581706e-05,
+      "loss": 1.5036320495605469,
+      "mean_token_accuracy": 0.6744937270879745,
+      "num_tokens": 689421.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.488253089785576,
+      "epoch": 0.5729291000238721,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.868164016521766e-05,
+      "loss": 1.5044355773925782,
+      "mean_token_accuracy": 0.6755094349384307,
+      "num_tokens": 752388.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.4446531629562378,
+      "epoch": 0.6206731916925281,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.8414560407252874e-05,
+      "loss": 1.4617152404785156,
+      "mean_token_accuracy": 0.6805451279878616,
+      "num_tokens": 815803.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.4555100780725478,
+      "epoch": 0.668417283361184,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.8125152111742944e-05,
+      "loss": 1.469644775390625,
+      "mean_token_accuracy": 0.6796707132458687,
+      "num_tokens": 878686.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.414481201171875,
+      "epoch": 0.71616137502984,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.7814183240848635e-05,
+      "loss": 1.4264332580566406,
+      "mean_token_accuracy": 0.6893567860126495,
+      "num_tokens": 940629.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3958445823192596,
+      "epoch": 0.763905466698496,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.7482478969002487e-05,
+      "loss": 1.4121397399902345,
+      "mean_token_accuracy": 0.6902168181538582,
+      "num_tokens": 1002810.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.3906721925735475,
+      "epoch": 0.811649558367152,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.7130919493259284e-05,
+      "loss": 1.3895584106445313,
+      "mean_token_accuracy": 0.6943912228941918,
+      "num_tokens": 1067041.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.3663516825437545,
+      "epoch": 0.859393650035808,
+      "grad_norm": 2.375,
+      "learning_rate": 1.676043769764072e-05,
+      "loss": 1.3753732299804688,
+      "mean_token_accuracy": 0.6982124423980713,
+      "num_tokens": 1130309.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.370076591670513,
+      "epoch": 0.907137741704464,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.637201667767204e-05,
+      "loss": 1.3839447021484375,
+      "mean_token_accuracy": 0.6986603051424026,
+      "num_tokens": 1192924.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.3369020774960518,
+      "epoch": 0.95488183337312,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.5966687131679463e-05,
+      "loss": 1.3427734375,
+      "mean_token_accuracy": 0.7060227099061013,
+      "num_tokens": 1254418.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.2727481353174803,
+      "eval_loss": 1.291359543800354,
+      "eval_mean_token_accuracy": 0.7115251171210456,
+      "eval_num_tokens": 1314012.0,
+      "eval_runtime": 52.7795,
+      "eval_samples_per_second": 16.711,
+      "eval_steps_per_second": 16.711,
+      "step": 1048
+    },
+    {
+      "entropy": 1.3094463989819367,
+      "epoch": 1.0019097636667462,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.554552462577077e-05,
+      "loss": 1.3102546691894532,
+      "mean_token_accuracy": 0.7078821743805397,
+      "num_tokens": 1316679.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9862651464343071,
+      "epoch": 1.0496538553354022,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.510964673975651e-05,
+      "loss": 0.94318603515625,
+      "mean_token_accuracy": 0.7765462607145309,
+      "num_tokens": 1379361.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9908152732253075,
+      "epoch": 1.0973979470040582,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.4660210101585367e-05,
+      "loss": 0.9593799591064454,
+      "mean_token_accuracy": 0.7724591562151909,
+      "num_tokens": 1442332.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.9940855634212494,
+      "epoch": 1.1451420386727142,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.4198407318162977e-05,
+      "loss": 0.9572904968261718,
+      "mean_token_accuracy": 0.7741717839241028,
+      "num_tokens": 1506048.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.997409186065197,
+      "epoch": 1.1928861303413703,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.3725463810698418e-05,
+      "loss": 0.962403564453125,
+      "mean_token_accuracy": 0.7725697284936905,
+      "num_tokens": 1569529.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.9752482569217682,
+      "epoch": 1.2406302220100263,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.3242634562975988e-05,
+      "loss": 0.9287136840820313,
+      "mean_token_accuracy": 0.7795223397016525,
+      "num_tokens": 1632744.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.972535012960434,
+      "epoch": 1.2883743136786823,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.2751200791180963e-05,
+      "loss": 0.9352114105224609,
+      "mean_token_accuracy": 0.7785275393724441,
+      "num_tokens": 1695454.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.9388299316167832,
+      "epoch": 1.3361184053473383,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.225246654411605e-05,
+      "loss": 0.9005762481689453,
+      "mean_token_accuracy": 0.7845277190208435,
+      "num_tokens": 1759868.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.9462847051024437,
+      "epoch": 1.3838624970159943,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.1747755242830204e-05,
+      "loss": 0.9043065643310547,
+      "mean_token_accuracy": 0.782291199862957,
+      "num_tokens": 1820879.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.9308868950605392,
+      "epoch": 1.4316065886846503,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.1238406168841981e-05,
+      "loss": 0.89322265625,
+      "mean_token_accuracy": 0.7880325165390968,
+      "num_tokens": 1883307.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.9272586759924889,
+      "epoch": 1.4793506803533063,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.0725770910276218e-05,
+      "loss": 0.8859436798095703,
+      "mean_token_accuracy": 0.7878314164280892,
+      "num_tokens": 1946095.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.9405490574240685,
+      "epoch": 1.5270947720219623,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.0211209775344377e-05,
+      "loss": 0.8991239929199218,
+      "mean_token_accuracy": 0.787108507156372,
+      "num_tokens": 2009039.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.9246599149703979,
+      "epoch": 1.5748388636906183,
+      "grad_norm": 2.1875,
+      "learning_rate": 9.696088182685638e-06,
+      "loss": 0.8893150329589844,
+      "mean_token_accuracy": 0.787010528743267,
+      "num_tokens": 2070866.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.9010607555508614,
+      "epoch": 1.6225829553592743,
+      "grad_norm": 2.5625,
+      "learning_rate": 9.18177303814717e-06,
+      "loss": 0.8644073486328125,
+      "mean_token_accuracy": 0.7935708364844323,
+      "num_tokens": 2133531.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.9290094000101089,
+      "epoch": 1.6703270470279303,
+      "grad_norm": 1.984375,
+      "learning_rate": 8.66962910761798e-06,
+      "loss": 0.8792234039306641,
+      "mean_token_accuracy": 0.7916052675247193,
+      "num_tokens": 2196212.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.9067334878444672,
+      "epoch": 1.7180711386965863,
+      "grad_norm": 2.0625,
+      "learning_rate": 8.161015395541311e-06,
+      "loss": 0.8666587829589844,
+      "mean_token_accuracy": 0.7931482782959938,
+      "num_tokens": 2259117.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.9192495954036712,
+      "epoch": 1.7658152303652423,
+      "grad_norm": 2.421875,
+      "learning_rate": 7.65728153871533e-06,
+      "loss": 0.8764252471923828,
+      "mean_token_accuracy": 0.792299057841301,
+      "num_tokens": 2320595.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.9041965058445931,
+      "epoch": 1.8135593220338984,
+      "grad_norm": 2.34375,
+      "learning_rate": 7.159764224951467e-06,
+      "loss": 0.8590383148193359,
+      "mean_token_accuracy": 0.7972308859229088,
+      "num_tokens": 2384167.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.9144112569093704,
+      "epoch": 1.8613034137025544,
+      "grad_norm": 2.21875,
+      "learning_rate": 6.669783646093635e-06,
+      "loss": 0.8813047790527344,
+      "mean_token_accuracy": 0.7919116005301475,
+      "num_tokens": 2444983.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.8830939248204231,
+      "epoch": 1.9090475053712104,
+      "grad_norm": 2.15625,
+      "learning_rate": 6.188639994810476e-06,
+      "loss": 0.8278516387939453,
+      "mean_token_accuracy": 0.8022209334373475,
+      "num_tokens": 2508413.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.9015234977006912,
+      "epoch": 1.9567915970398664,
+      "grad_norm": 2.1875,
+      "learning_rate": 5.717610014456644e-06,
+      "loss": 0.8635023498535156,
+      "mean_token_accuracy": 0.7982774981856346,
+      "num_tokens": 2570521.0,
+      "step": 2050
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 1.0000475248545746,
+      "eval_loss": 1.178954005241394,
+      "eval_mean_token_accuracy": 0.74215727841773,
+      "eval_num_tokens": 2628024.0,
+      "eval_runtime": 52.7405,
+      "eval_samples_per_second": 16.723,
+      "eval_steps_per_second": 16.723,
+      "step": 2096
+    },
+    {
+      "entropy": 0.8756030080887267,
+      "epoch": 2.0038195273334924,
+      "grad_norm": 2.03125,
+      "learning_rate": 5.257943611158213e-06,
+      "loss": 0.8093524169921875,
+      "mean_token_accuracy": 0.8077045040687328,
+      "num_tokens": 2632866.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.6753400918841362,
+      "epoch": 2.0515636190021485,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.810860537112266e-06,
+      "loss": 0.5168679809570312,
+      "mean_token_accuracy": 0.8748944187164307,
+      "num_tokens": 2695322.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.663670559078455,
+      "epoch": 2.0993077106708045,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.377547153901771e-06,
+      "loss": 0.5165143203735352,
+      "mean_token_accuracy": 0.873924298286438,
+      "num_tokens": 2758736.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.6488903272151947,
+      "epoch": 2.1470518023394605,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.959153284414369e-06,
+      "loss": 0.5097105789184571,
+      "mean_token_accuracy": 0.8761234775185585,
+      "num_tokens": 2820842.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.6527856409549713,
+      "epoch": 2.1947958940081165,
+      "grad_norm": 2.125,
+      "learning_rate": 3.5567891617188666e-06,
+      "loss": 0.5143218231201172,
+      "mean_token_accuracy": 0.8756609866023064,
+      "num_tokens": 2883956.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.6608934430778026,
+      "epoch": 2.2425399856767725,
+      "grad_norm": 2.15625,
+      "learning_rate": 3.171522482995576e-06,
+      "loss": 0.5194900894165039,
+      "mean_token_accuracy": 0.8734953963756561,
+      "num_tokens": 2947262.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.6422708696126938,
+      "epoch": 2.2902840773454285,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.8043755763382284e-06,
+      "loss": 0.5032471084594726,
+      "mean_token_accuracy": 0.8760477969050408,
+      "num_tokens": 3010412.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.6539460071921348,
+      "epoch": 2.3380281690140845,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.4563226879454594e-06,
+      "loss": 0.5098763275146484,
+      "mean_token_accuracy": 0.8765698638558388,
+      "num_tokens": 3073608.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.6659840613603591,
+      "epoch": 2.3857722606827405,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.1282873969004236e-06,
+      "loss": 0.5180015563964844,
+      "mean_token_accuracy": 0.8748002126812935,
+      "num_tokens": 3135626.0,
+      "step": 2500
+    },
+    {
+      "entropy": 0.6446484234929085,
+      "epoch": 2.4335163523513965,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.8211401643986893e-06,
+      "loss": 0.4983504867553711,
+      "mean_token_accuracy": 0.8782649829983711,
+      "num_tokens": 3199381.0,
+      "step": 2550
+    },
+    {
+      "entropy": 0.6465041987597943,
+      "epoch": 2.4812604440200525,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.5356960239275587e-06,
+      "loss": 0.5081784057617188,
+      "mean_token_accuracy": 0.8756673774123191,
+      "num_tokens": 3261575.0,
+      "step": 2600
+    },
+    {
+      "entropy": 0.6383489421010018,
+      "epoch": 2.5290045356887085,
+      "grad_norm": 1.984375,
+      "learning_rate": 1.2727124185261197e-06,
+      "loss": 0.5109892654418945,
+      "mean_token_accuracy": 0.8760039383172988,
+      "num_tokens": 3323813.0,
+      "step": 2650
+    },
+    {
+      "entropy": 0.6562608055770397,
+      "epoch": 2.5767486273573645,
+      "grad_norm": 2.125,
+      "learning_rate": 1.0328871908650328e-06,
+      "loss": 0.5123030853271484,
+      "mean_token_accuracy": 0.8748046290874482,
+      "num_tokens": 3386995.0,
+      "step": 2700
+    },
+    {
+      "entropy": 0.6482995609939098,
+      "epoch": 2.6244927190260205,
+      "grad_norm": 2.265625,
+      "learning_rate": 8.16856731479384e-07,
+      "loss": 0.5052508926391601,
+      "mean_token_accuracy": 0.8763853028416634,
+      "num_tokens": 3449788.0,
+      "step": 2750
+    },
+    {
+      "entropy": 0.6505758769810199,
+      "epoch": 2.6722368106946766,
+      "grad_norm": 2.640625,
+      "learning_rate": 6.251942900684804e-07,
+      "loss": 0.5092948150634765,
+      "mean_token_accuracy": 0.87547658264637,
+      "num_tokens": 3511961.0,
+      "step": 2800
+    },
+    {
+      "entropy": 0.6379988174140453,
+      "epoch": 2.7199809023633326,
+      "grad_norm": 2.234375,
+      "learning_rate": 4.584084543436174e-07,
+      "loss": 0.4954957962036133,
+      "mean_token_accuracy": 0.877953137755394,
+      "num_tokens": 3574785.0,
+      "step": 2850
+    },
+    {
+      "entropy": 0.6535128343105316,
+      "epoch": 2.7677249940319886,
+      "grad_norm": 2.078125,
+      "learning_rate": 3.1694180046025647e-07,
+      "loss": 0.5142677688598633,
+      "mean_token_accuracy": 0.872634556889534,
+      "num_tokens": 3636998.0,
+      "step": 2900
+    },
+    {
+      "entropy": 0.6504488955438137,
+      "epoch": 2.8154690857006446,
+      "grad_norm": 2.125,
+      "learning_rate": 2.0116971861583568e-07,
+      "loss": 0.5068624114990234,
+      "mean_token_accuracy": 0.8755693709850312,
+      "num_tokens": 3700205.0,
+      "step": 2950
+    },
+    {
+      "entropy": 0.6478478516638279,
+      "epoch": 2.8632131773693006,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.1139941692950074e-07,
+      "loss": 0.5064026641845704,
+      "mean_token_accuracy": 0.8756004160642624,
+      "num_tokens": 3763095.0,
+      "step": 3000
+    },
+    {
+      "entropy": 0.6406606796383858,
+      "epoch": 2.9109572690379566,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.7869106247049326e-08,
+      "loss": 0.5050786209106445,
+      "mean_token_accuracy": 0.8759026023745536,
+      "num_tokens": 3826241.0,
+      "step": 3050
+    },
+    {
+      "entropy": 0.6304298985004425,
+      "epoch": 2.9587013607066126,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.0747368034260686e-08,
+      "loss": 0.49738613128662107,
+      "mean_token_accuracy": 0.8762941983342171,
+      "num_tokens": 3887718.0,
+      "step": 3100
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.9061206575491532,
+      "eval_loss": 1.2480237483978271,
+      "eval_mean_token_accuracy": 0.7400713412940096,
+      "eval_num_tokens": 3942036.0,
+      "eval_runtime": 52.7004,
+      "eval_samples_per_second": 16.736,
+      "eval_steps_per_second": 16.736,
+      "step": 3144
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3144,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.3875422862973338e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-3144/training_args.bin b/checkpoint-3144/training_args.bin
new file mode 100644
index 0000000..49930dc
--- /dev/null
+++ b/checkpoint-3144/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a3ea84f94e96c81578a690800ff8f79d92cb1baf91ae820778832d6af60f008
+size 5304
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..7580fae
--- /dev/null
+++ b/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128001,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000..c895560
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001
+  ],
+  "pad_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.5.1"
+}
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000..14edbef
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31aeeebe856fb321b73f53a4d6587fa5e98aca83b451a9377728b509fd7c5c2a
+size 16060556616
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000..1c1d8d5
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..f11592a
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000..49930dc
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a3ea84f94e96c81578a690800ff8f79d92cb1baf91ae820778832d6af60f008
+size 5304
diff --git a/training_args.json b/training_args.json
new file mode 100644
index 0000000..cfd056b
--- /dev/null
+++ b/training_args.json
@@ -0,0 +1,11 @@
+{
+  "model_name": "meta-llama/Llama-3.1-8B",
+  "stage1_method": null,
+  "stage2_method": "full",
+  "stage": "task_specific",
+  "num_epochs": 3,
+  "config_path": "/home/oelgendy/arabic-medical-llm/script/configs/full_ft.yaml",
+  "stage1_checkpoint": null,
+  "train_samples": 16756,
+  "val_samples": 882
+}
\ No newline at end of file