commit 7cbf69a62604edcc8c4e845b926b72ed0d023983
Author: ModelHub XC <noreply@modelhub.org.cn>
Date:   Tue Aug 18 00:51:16 2026 +0800

    初始化项目，由ModelHub XC社区提供模型
    
    Model: lorinet3/gpt-sw3-126m-exercise-scenario-generator
    Source: Original Platform

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..ad055bd
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,40 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+gpt-sw3-126m-q6_k.gguf filter=lfs diff=lfs merge=lfs -text
+gpt-sw3-126m-q8_0.gguf filter=lfs diff=lfs merge=lfs -text
+gpt-sw3-126m-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+gpt-sw3-126m-fp16.gguf filter=lfs diff=lfs merge=lfs -text
+gpt-sw3-126m-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..bc5f30d
--- /dev/null
+++ b/README.md
@@ -0,0 +1,199 @@
+---
+library_name: transformers
+tags: []
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated.
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
\ No newline at end of file
diff --git a/added_tokens.json b/added_tokens.json
new file mode 100644
index 0000000..c045a99
--- /dev/null
+++ b/added_tokens.json
@@ -0,0 +1,18 @@
+{
+  "[A1]": 64012,
+  "[A2]": 64013,
+  "[B1]": 64014,
+  "[B2]": 64015,
+  "[EXERCISE_VARIATIONS]": 64011,
+  "[LESSON]": 64009,
+  "[PRACTICE_TEXT]": 64010,
+  "antonym_focus": 64007,
+  "definition_only": 64008,
+  "fill_in_the_blanks": 64003,
+  "free_writing": 64000,
+  "general_vocab": 64004,
+  "matching_pairs": 64001,
+  "morphology_focus": 64005,
+  "reorder_tokens": 64002,
+  "synonym_focus": 64006
+}
diff --git a/checkpoint-5000/added_tokens.json b/checkpoint-5000/added_tokens.json
new file mode 100644
index 0000000..c045a99
--- /dev/null
+++ b/checkpoint-5000/added_tokens.json
@@ -0,0 +1,18 @@
+{
+  "[A1]": 64012,
+  "[A2]": 64013,
+  "[B1]": 64014,
+  "[B2]": 64015,
+  "[EXERCISE_VARIATIONS]": 64011,
+  "[LESSON]": 64009,
+  "[PRACTICE_TEXT]": 64010,
+  "antonym_focus": 64007,
+  "definition_only": 64008,
+  "fill_in_the_blanks": 64003,
+  "free_writing": 64000,
+  "general_vocab": 64004,
+  "matching_pairs": 64001,
+  "morphology_focus": 64005,
+  "reorder_tokens": 64002,
+  "synonym_focus": 64006
+}
diff --git a/checkpoint-5000/config.json b/checkpoint-5000/config.json
new file mode 100644
index 0000000..152460c
--- /dev/null
+++ b/checkpoint-5000/config.json
@@ -0,0 +1,36 @@
+{
+  "activation_function": "gelu",
+  "apply_query_key_layer_scaling": true,
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 3,
+  "initializer_range": 0.023,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 2048,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": 3072,
+  "n_layer": 12,
+  "n_positions": 4096,
+  "normalize_attention_scores": true,
+  "pad_token_id": 0,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "tokenizer_class": "GPTSw3Tokenizer",
+  "transformers_version": "4.57.1",
+  "use_cache": false,
+  "vocab_size": 64016
+}
diff --git a/checkpoint-5000/generation_config.json b/checkpoint-5000/generation_config.json
new file mode 100644
index 0000000..aca1c6f
--- /dev/null
+++ b/checkpoint-5000/generation_config.json
@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "transformers_version": "4.57.1",
+  "use_cache": false
+}
diff --git a/checkpoint-5000/model.safetensors b/checkpoint-5000/model.safetensors
new file mode 100644
index 0000000..15a8254
--- /dev/null
+++ b/checkpoint-5000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c8409674d4f5f7842bdbe90eafd6f4ee47fd5a324ad1335a27ee50dd3d1464c
+size 274779928
diff --git a/checkpoint-5000/optimizer.pt b/checkpoint-5000/optimizer.pt
new file mode 100644
index 0000000..8cba9c1
--- /dev/null
+++ b/checkpoint-5000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebd558a9a8e6f927612096fcee1a8f7eb45ef97e36a8768896f17cb53316d9a8
+size 549590475
diff --git a/checkpoint-5000/rng_state.pth b/checkpoint-5000/rng_state.pth
new file mode 100644
index 0000000..ce541ba
--- /dev/null
+++ b/checkpoint-5000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fefcda1eb027efef7338951cd8da153385aed51c0764a3508e3386a88297f27
+size 14645
diff --git a/checkpoint-5000/scheduler.pt b/checkpoint-5000/scheduler.pt
new file mode 100644
index 0000000..4b2aa1e
--- /dev/null
+++ b/checkpoint-5000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12b1feb292d5e79121d1de1f6c7915c97f017d43ae37c65efa1038db2ec6c3d5
+size 1465
diff --git a/checkpoint-5000/special_tokens_map.json b/checkpoint-5000/special_tokens_map.json
new file mode 100644
index 0000000..fac732c
--- /dev/null
+++ b/checkpoint-5000/special_tokens_map.json
@@ -0,0 +1,127 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "--",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "free_writing",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "matching_pairs",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "reorder_tokens",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "fill_in_the_blanks",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "general_vocab",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "morphology_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "synonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "antonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "definition_only",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[LESSON]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[PRACTICE_TEXT]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[EXERCISE_VARIATIONS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[A1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[A2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[B1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[B2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<s>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}
diff --git a/checkpoint-5000/spiece.model b/checkpoint-5000/spiece.model
new file mode 100644
index 0000000..491f2fd
--- /dev/null
+++ b/checkpoint-5000/spiece.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4bda2cd84ff0ac659cda40e746c55f47ee3e57cf18471670ad26998c28be52d
+size 1071955
diff --git a/checkpoint-5000/tokenizer_config.json b/checkpoint-5000/tokenizer_config.json
new file mode 100644
index 0000000..67102bb
--- /dev/null
+++ b/checkpoint-5000/tokenizer_config.json
@@ -0,0 +1,203 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "603": {
+      "content": "--",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "free_writing",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "matching_pairs",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64002": {
+      "content": "reorder_tokens",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64003": {
+      "content": "fill_in_the_blanks",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64004": {
+      "content": "general_vocab",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64005": {
+      "content": "morphology_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64006": {
+      "content": "synonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64007": {
+      "content": "antonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64008": {
+      "content": "definition_only",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64009": {
+      "content": "[LESSON]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64010": {
+      "content": "[PRACTICE_TEXT]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64011": {
+      "content": "[EXERCISE_VARIATIONS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64012": {
+      "content": "[A1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64013": {
+      "content": "[A2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64014": {
+      "content": "[B1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64015": {
+      "content": "[B2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "--",
+    "free_writing",
+    "matching_pairs",
+    "reorder_tokens",
+    "fill_in_the_blanks",
+    "general_vocab",
+    "morphology_focus",
+    "synonym_focus",
+    "antonym_focus",
+    "definition_only",
+    "[LESSON]",
+    "[PRACTICE_TEXT]",
+    "[EXERCISE_VARIATIONS]",
+    "[A1]",
+    "[A2]",
+    "[B1]",
+    "[B2]"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "keep_accents": false,
+  "model_max_length": 4096,
+  "pad_token": "<pad>",
+  "remove_space": false,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "GPTSw3Tokenizer",
+  "unk_token": "<unk>"
+}
diff --git a/checkpoint-5000/trainer_state.json b/checkpoint-5000/trainer_state.json
new file mode 100644
index 0000000..66cad2b
--- /dev/null
+++ b/checkpoint-5000/trainer_state.json
@@ -0,0 +1,3614 @@
+{
+  "best_global_step": 5000,
+  "best_metric": 0.47889795899391174,
+  "best_model_checkpoint": "/home/unsloth/szakdolgozat/stage2_exercise_scenario_generator_model/gpt-sw3-finetuned/checkpoint-5000",
+  "epoch": 3.791275486012328,
+  "eval_steps": 500,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.007586533902323376,
+      "grad_norm": 18.25,
+      "learning_rate": 8.522727272727273e-07,
+      "loss": 2.8483,
+      "step": 10
+    },
+    {
+      "epoch": 0.015173067804646752,
+      "grad_norm": 20.5,
+      "learning_rate": 1.7992424242424244e-06,
+      "loss": 2.8471,
+      "step": 20
+    },
+    {
+      "epoch": 0.02275960170697013,
+      "grad_norm": 12.1875,
+      "learning_rate": 2.7462121212121215e-06,
+      "loss": 2.7292,
+      "step": 30
+    },
+    {
+      "epoch": 0.030346135609293504,
+      "grad_norm": 11.1875,
+      "learning_rate": 3.6931818181818186e-06,
+      "loss": 2.794,
+      "step": 40
+    },
+    {
+      "epoch": 0.03793266951161688,
+      "grad_norm": 14.8125,
+      "learning_rate": 4.640151515151515e-06,
+      "loss": 2.6682,
+      "step": 50
+    },
+    {
+      "epoch": 0.04551920341394026,
+      "grad_norm": 11.125,
+      "learning_rate": 5.587121212121213e-06,
+      "loss": 2.5886,
+      "step": 60
+    },
+    {
+      "epoch": 0.05310573731626363,
+      "grad_norm": 7.375,
+      "learning_rate": 6.534090909090909e-06,
+      "loss": 2.4229,
+      "step": 70
+    },
+    {
+      "epoch": 0.06069227121858701,
+      "grad_norm": 6.21875,
+      "learning_rate": 7.481060606060607e-06,
+      "loss": 2.3076,
+      "step": 80
+    },
+    {
+      "epoch": 0.06827880512091039,
+      "grad_norm": 6.96875,
+      "learning_rate": 8.428030303030303e-06,
+      "loss": 2.2223,
+      "step": 90
+    },
+    {
+      "epoch": 0.07586533902323377,
+      "grad_norm": 5.25,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 2.112,
+      "step": 100
+    },
+    {
+      "epoch": 0.08345187292555714,
+      "grad_norm": 6.0,
+      "learning_rate": 1.0321969696969698e-05,
+      "loss": 2.0566,
+      "step": 110
+    },
+    {
+      "epoch": 0.09103840682788052,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.1268939393939394e-05,
+      "loss": 1.8943,
+      "step": 120
+    },
+    {
+      "epoch": 0.09862494073020389,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.2215909090909092e-05,
+      "loss": 1.8111,
+      "step": 130
+    },
+    {
+      "epoch": 0.10621147463252727,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.3162878787878787e-05,
+      "loss": 1.753,
+      "step": 140
+    },
+    {
+      "epoch": 0.11379800853485064,
+      "grad_norm": 4.125,
+      "learning_rate": 1.4109848484848487e-05,
+      "loss": 1.7014,
+      "step": 150
+    },
+    {
+      "epoch": 0.12138454243717402,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.5056818181818182e-05,
+      "loss": 1.6254,
+      "step": 160
+    },
+    {
+      "epoch": 0.1289710763394974,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.600378787878788e-05,
+      "loss": 1.5866,
+      "step": 170
+    },
+    {
+      "epoch": 0.13655761024182078,
+      "grad_norm": 5.875,
+      "learning_rate": 1.6950757575757574e-05,
+      "loss": 1.4858,
+      "step": 180
+    },
+    {
+      "epoch": 0.14414414414414414,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.7897727272727276e-05,
+      "loss": 1.4044,
+      "step": 190
+    },
+    {
+      "epoch": 0.15173067804646753,
+      "grad_norm": 5.3125,
+      "learning_rate": 1.884469696969697e-05,
+      "loss": 1.3114,
+      "step": 200
+    },
+    {
+      "epoch": 0.1593172119487909,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.9791666666666665e-05,
+      "loss": 1.2421,
+      "step": 210
+    },
+    {
+      "epoch": 0.16690374585111428,
+      "grad_norm": 3.65625,
+      "learning_rate": 2.0738636363636367e-05,
+      "loss": 1.1665,
+      "step": 220
+    },
+    {
+      "epoch": 0.17449027975343764,
+      "grad_norm": 3.390625,
+      "learning_rate": 2.1685606060606062e-05,
+      "loss": 1.1339,
+      "step": 230
+    },
+    {
+      "epoch": 0.18207681365576103,
+      "grad_norm": 3.0625,
+      "learning_rate": 2.2632575757575757e-05,
+      "loss": 1.1219,
+      "step": 240
+    },
+    {
+      "epoch": 0.1896633475580844,
+      "grad_norm": 3.1875,
+      "learning_rate": 2.3579545454545455e-05,
+      "loss": 1.0914,
+      "step": 250
+    },
+    {
+      "epoch": 0.19724988146040778,
+      "grad_norm": 3.171875,
+      "learning_rate": 2.4526515151515154e-05,
+      "loss": 1.0727,
+      "step": 260
+    },
+    {
+      "epoch": 0.20483641536273114,
+      "grad_norm": 3.625,
+      "learning_rate": 2.5473484848484852e-05,
+      "loss": 1.0931,
+      "step": 270
+    },
+    {
+      "epoch": 0.21242294926505453,
+      "grad_norm": 3.375,
+      "learning_rate": 2.6420454545454547e-05,
+      "loss": 1.041,
+      "step": 280
+    },
+    {
+      "epoch": 0.2200094831673779,
+      "grad_norm": 3.1875,
+      "learning_rate": 2.7367424242424245e-05,
+      "loss": 1.0206,
+      "step": 290
+    },
+    {
+      "epoch": 0.22759601706970128,
+      "grad_norm": 3.140625,
+      "learning_rate": 2.831439393939394e-05,
+      "loss": 1.0103,
+      "step": 300
+    },
+    {
+      "epoch": 0.23518255097202465,
+      "grad_norm": 13.5625,
+      "learning_rate": 2.9261363636363635e-05,
+      "loss": 0.9865,
+      "step": 310
+    },
+    {
+      "epoch": 0.24276908487434803,
+      "grad_norm": 3.40625,
+      "learning_rate": 3.0208333333333334e-05,
+      "loss": 0.9431,
+      "step": 320
+    },
+    {
+      "epoch": 0.2503556187766714,
+      "grad_norm": 3.46875,
+      "learning_rate": 3.115530303030303e-05,
+      "loss": 0.9698,
+      "step": 330
+    },
+    {
+      "epoch": 0.2579421526789948,
+      "grad_norm": 3.0,
+      "learning_rate": 3.210227272727273e-05,
+      "loss": 0.9284,
+      "step": 340
+    },
+    {
+      "epoch": 0.2655286865813182,
+      "grad_norm": 3.0625,
+      "learning_rate": 3.3049242424242425e-05,
+      "loss": 0.8814,
+      "step": 350
+    },
+    {
+      "epoch": 0.27311522048364156,
+      "grad_norm": 3.609375,
+      "learning_rate": 3.399621212121212e-05,
+      "loss": 0.8822,
+      "step": 360
+    },
+    {
+      "epoch": 0.2807017543859649,
+      "grad_norm": 3.796875,
+      "learning_rate": 3.494318181818182e-05,
+      "loss": 0.8931,
+      "step": 370
+    },
+    {
+      "epoch": 0.2882882882882883,
+      "grad_norm": 3.203125,
+      "learning_rate": 3.589015151515152e-05,
+      "loss": 0.8948,
+      "step": 380
+    },
+    {
+      "epoch": 0.2958748221906117,
+      "grad_norm": 3.1875,
+      "learning_rate": 3.683712121212121e-05,
+      "loss": 0.8459,
+      "step": 390
+    },
+    {
+      "epoch": 0.30346135609293506,
+      "grad_norm": 3.03125,
+      "learning_rate": 3.778409090909091e-05,
+      "loss": 0.8503,
+      "step": 400
+    },
+    {
+      "epoch": 0.3110478899952584,
+      "grad_norm": 3.078125,
+      "learning_rate": 3.873106060606061e-05,
+      "loss": 0.8365,
+      "step": 410
+    },
+    {
+      "epoch": 0.3186344238975818,
+      "grad_norm": 3.171875,
+      "learning_rate": 3.96780303030303e-05,
+      "loss": 0.8607,
+      "step": 420
+    },
+    {
+      "epoch": 0.3262209577999052,
+      "grad_norm": 3.5625,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 0.8253,
+      "step": 430
+    },
+    {
+      "epoch": 0.33380749170222856,
+      "grad_norm": 3.46875,
+      "learning_rate": 4.15719696969697e-05,
+      "loss": 0.8143,
+      "step": 440
+    },
+    {
+      "epoch": 0.3413940256045519,
+      "grad_norm": 3.859375,
+      "learning_rate": 4.2518939393939395e-05,
+      "loss": 0.7962,
+      "step": 450
+    },
+    {
+      "epoch": 0.3489805595068753,
+      "grad_norm": 3.203125,
+      "learning_rate": 4.346590909090909e-05,
+      "loss": 0.8083,
+      "step": 460
+    },
+    {
+      "epoch": 0.3565670934091987,
+      "grad_norm": 3.125,
+      "learning_rate": 4.441287878787879e-05,
+      "loss": 0.7884,
+      "step": 470
+    },
+    {
+      "epoch": 0.36415362731152207,
+      "grad_norm": 3.1875,
+      "learning_rate": 4.5359848484848486e-05,
+      "loss": 0.7643,
+      "step": 480
+    },
+    {
+      "epoch": 0.3717401612138454,
+      "grad_norm": 2.828125,
+      "learning_rate": 4.630681818181818e-05,
+      "loss": 0.7865,
+      "step": 490
+    },
+    {
+      "epoch": 0.3793266951161688,
+      "grad_norm": 3.03125,
+      "learning_rate": 4.725378787878788e-05,
+      "loss": 0.7825,
+      "step": 500
+    },
+    {
+      "epoch": 0.3793266951161688,
+      "eval_loss": 0.72025066614151,
+      "eval_runtime": 9.0957,
+      "eval_samples_per_second": 122.036,
+      "eval_steps_per_second": 15.282,
+      "step": 500
+    },
+    {
+      "epoch": 0.3869132290184922,
+      "grad_norm": 2.703125,
+      "learning_rate": 4.820075757575758e-05,
+      "loss": 0.7693,
+      "step": 510
+    },
+    {
+      "epoch": 0.39449976292081557,
+      "grad_norm": 3.9375,
+      "learning_rate": 4.914772727272727e-05,
+      "loss": 0.7902,
+      "step": 520
+    },
+    {
+      "epoch": 0.40208629682313896,
+      "grad_norm": 2.90625,
+      "learning_rate": 4.9999994527467846e-05,
+      "loss": 0.7622,
+      "step": 530
+    },
+    {
+      "epoch": 0.4096728307254623,
+      "grad_norm": 2.96875,
+      "learning_rate": 4.999933782650834e-05,
+      "loss": 0.747,
+      "step": 540
+    },
+    {
+      "epoch": 0.4172593646277857,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.999758665206119e-05,
+      "loss": 0.7543,
+      "step": 550
+    },
+    {
+      "epoch": 0.42484589853010907,
+      "grad_norm": 2.96875,
+      "learning_rate": 4.999474108079299e-05,
+      "loss": 0.7418,
+      "step": 560
+    },
+    {
+      "epoch": 0.43243243243243246,
+      "grad_norm": 2.90625,
+      "learning_rate": 4.9990801237283106e-05,
+      "loss": 0.7405,
+      "step": 570
+    },
+    {
+      "epoch": 0.4400189663347558,
+      "grad_norm": 3.09375,
+      "learning_rate": 4.998576729401832e-05,
+      "loss": 0.717,
+      "step": 580
+    },
+    {
+      "epoch": 0.4476055002370792,
+      "grad_norm": 2.765625,
+      "learning_rate": 4.997963947138513e-05,
+      "loss": 0.7181,
+      "step": 590
+    },
+    {
+      "epoch": 0.45519203413940257,
+      "grad_norm": 3.09375,
+      "learning_rate": 4.9972418037660236e-05,
+      "loss": 0.7132,
+      "step": 600
+    },
+    {
+      "epoch": 0.46277856804172596,
+      "grad_norm": 2.84375,
+      "learning_rate": 4.9964103308998723e-05,
+      "loss": 0.7236,
+      "step": 610
+    },
+    {
+      "epoch": 0.4703651019440493,
+      "grad_norm": 3.125,
+      "learning_rate": 4.995469564942023e-05,
+      "loss": 0.6841,
+      "step": 620
+    },
+    {
+      "epoch": 0.4779516358463727,
+      "grad_norm": 2.84375,
+      "learning_rate": 4.994419547079303e-05,
+      "loss": 0.682,
+      "step": 630
+    },
+    {
+      "epoch": 0.48553816974869607,
+      "grad_norm": 2.5625,
+      "learning_rate": 4.9932603232815976e-05,
+      "loss": 0.6747,
+      "step": 640
+    },
+    {
+      "epoch": 0.49312470365101946,
+      "grad_norm": 2.90625,
+      "learning_rate": 4.991991944299841e-05,
+      "loss": 0.6993,
+      "step": 650
+    },
+    {
+      "epoch": 0.5007112375533428,
+      "grad_norm": 3.125,
+      "learning_rate": 4.990614465663789e-05,
+      "loss": 0.6905,
+      "step": 660
+    },
+    {
+      "epoch": 0.5082977714556662,
+      "grad_norm": 2.640625,
+      "learning_rate": 4.989127947679595e-05,
+      "loss": 0.6803,
+      "step": 670
+    },
+    {
+      "epoch": 0.5158843053579896,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.987532455427161e-05,
+      "loss": 0.6721,
+      "step": 680
+    },
+    {
+      "epoch": 0.5234708392603129,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.985828058757298e-05,
+      "loss": 0.67,
+      "step": 690
+    },
+    {
+      "epoch": 0.5310573731626363,
+      "grad_norm": 2.5,
+      "learning_rate": 4.98401483228866e-05,
+      "loss": 0.6955,
+      "step": 700
+    },
+    {
+      "epoch": 0.5386439070649597,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.982092855404482e-05,
+      "loss": 0.678,
+      "step": 710
+    },
+    {
+      "epoch": 0.5462304409672831,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.9800622122491026e-05,
+      "loss": 0.6769,
+      "step": 720
+    },
+    {
+      "epoch": 0.5538169748696065,
+      "grad_norm": 2.875,
+      "learning_rate": 4.9779229917242795e-05,
+      "loss": 0.6803,
+      "step": 730
+    },
+    {
+      "epoch": 0.5614035087719298,
+      "grad_norm": 2.9375,
+      "learning_rate": 4.9756752874853006e-05,
+      "loss": 0.6512,
+      "step": 740
+    },
+    {
+      "epoch": 0.5689900426742532,
+      "grad_norm": 3.3125,
+      "learning_rate": 4.973319197936879e-05,
+      "loss": 0.6642,
+      "step": 750
+    },
+    {
+      "epoch": 0.5765765765765766,
+      "grad_norm": 2.640625,
+      "learning_rate": 4.97085482622885e-05,
+      "loss": 0.6701,
+      "step": 760
+    },
+    {
+      "epoch": 0.5841631104788999,
+      "grad_norm": 2.84375,
+      "learning_rate": 4.9682822802516506e-05,
+      "loss": 0.6523,
+      "step": 770
+    },
+    {
+      "epoch": 0.5917496443812233,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.965601672631597e-05,
+      "loss": 0.6535,
+      "step": 780
+    },
+    {
+      "epoch": 0.5993361782835467,
+      "grad_norm": 2.515625,
+      "learning_rate": 4.9628131207259596e-05,
+      "loss": 0.6452,
+      "step": 790
+    },
+    {
+      "epoch": 0.6069227121858701,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.959916746617814e-05,
+      "loss": 0.6686,
+      "step": 800
+    },
+    {
+      "epoch": 0.6145092460881935,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.9569126771107086e-05,
+      "loss": 0.6433,
+      "step": 810
+    },
+    {
+      "epoch": 0.6220957799905168,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.953801043723102e-05,
+      "loss": 0.6608,
+      "step": 820
+    },
+    {
+      "epoch": 0.6296823138928402,
+      "grad_norm": 2.53125,
+      "learning_rate": 4.950581982682614e-05,
+      "loss": 0.6492,
+      "step": 830
+    },
+    {
+      "epoch": 0.6372688477951636,
+      "grad_norm": 2.5625,
+      "learning_rate": 4.947255634920054e-05,
+      "loss": 0.6523,
+      "step": 840
+    },
+    {
+      "epoch": 0.6448553816974869,
+      "grad_norm": 2.671875,
+      "learning_rate": 4.9438221460632584e-05,
+      "loss": 0.6343,
+      "step": 850
+    },
+    {
+      "epoch": 0.6524419155998104,
+      "grad_norm": 2.671875,
+      "learning_rate": 4.940281666430708e-05,
+      "loss": 0.6565,
+      "step": 860
+    },
+    {
+      "epoch": 0.6600284495021337,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.9366343510249543e-05,
+      "loss": 0.6736,
+      "step": 870
+    },
+    {
+      "epoch": 0.6676149834044571,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.9328803595258254e-05,
+      "loss": 0.6229,
+      "step": 880
+    },
+    {
+      "epoch": 0.6752015173067805,
+      "grad_norm": 2.484375,
+      "learning_rate": 4.929019856283442e-05,
+      "loss": 0.6358,
+      "step": 890
+    },
+    {
+      "epoch": 0.6827880512091038,
+      "grad_norm": 2.421875,
+      "learning_rate": 4.925053010311018e-05,
+      "loss": 0.6187,
+      "step": 900
+    },
+    {
+      "epoch": 0.6903745851114272,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.920979995277464e-05,
+      "loss": 0.6408,
+      "step": 910
+    },
+    {
+      "epoch": 0.6979611190137506,
+      "grad_norm": 2.59375,
+      "learning_rate": 4.916800989499781e-05,
+      "loss": 0.6346,
+      "step": 920
+    },
+    {
+      "epoch": 0.705547652916074,
+      "grad_norm": 2.625,
+      "learning_rate": 4.9125161759352564e-05,
+      "loss": 0.6052,
+      "step": 930
+    },
+    {
+      "epoch": 0.7131341868183974,
+      "grad_norm": 2.5,
+      "learning_rate": 4.9081257421734536e-05,
+      "loss": 0.6474,
+      "step": 940
+    },
+    {
+      "epoch": 0.7207207207207207,
+      "grad_norm": 2.421875,
+      "learning_rate": 4.903629880427997e-05,
+      "loss": 0.6145,
+      "step": 950
+    },
+    {
+      "epoch": 0.7283072546230441,
+      "grad_norm": 2.4375,
+      "learning_rate": 4.899028787528158e-05,
+      "loss": 0.6219,
+      "step": 960
+    },
+    {
+      "epoch": 0.7358937885253675,
+      "grad_norm": 2.4375,
+      "learning_rate": 4.894322664910242e-05,
+      "loss": 0.6285,
+      "step": 970
+    },
+    {
+      "epoch": 0.7434803224276908,
+      "grad_norm": 2.203125,
+      "learning_rate": 4.8895117186087635e-05,
+      "loss": 0.6245,
+      "step": 980
+    },
+    {
+      "epoch": 0.7510668563300142,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.884596159247428e-05,
+      "loss": 0.619,
+      "step": 990
+    },
+    {
+      "epoch": 0.7586533902323376,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.87957620202991e-05,
+      "loss": 0.6051,
+      "step": 1000
+    },
+    {
+      "epoch": 0.7586533902323376,
+      "eval_loss": 0.571657657623291,
+      "eval_runtime": 9.0866,
+      "eval_samples_per_second": 122.157,
+      "eval_steps_per_second": 15.297,
+      "step": 1000
+    },
+    {
+      "epoch": 0.766239924134661,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.874452066730436e-05,
+      "loss": 0.6233,
+      "step": 1010
+    },
+    {
+      "epoch": 0.7738264580369844,
+      "grad_norm": 2.515625,
+      "learning_rate": 4.869223977684156e-05,
+      "loss": 0.6276,
+      "step": 1020
+    },
+    {
+      "epoch": 0.7814129919393077,
+      "grad_norm": 2.375,
+      "learning_rate": 4.863892163777327e-05,
+      "loss": 0.595,
+      "step": 1030
+    },
+    {
+      "epoch": 0.7889995258416311,
+      "grad_norm": 3.203125,
+      "learning_rate": 4.8584568584372904e-05,
+      "loss": 0.6067,
+      "step": 1040
+    },
+    {
+      "epoch": 0.7965860597439545,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.852918299622252e-05,
+      "loss": 0.5986,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8041725936462779,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.847276729810867e-05,
+      "loss": 0.5928,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8117591275486012,
+      "grad_norm": 2.375,
+      "learning_rate": 4.8415323959916195e-05,
+      "loss": 0.5857,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8193456614509246,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.835685549652015e-05,
+      "loss": 0.6214,
+      "step": 1080
+    },
+    {
+      "epoch": 0.826932195353248,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.829736446767563e-05,
+      "loss": 0.6083,
+      "step": 1090
+    },
+    {
+      "epoch": 0.8345187292555714,
+      "grad_norm": 2.390625,
+      "learning_rate": 4.8236853477905805e-05,
+      "loss": 0.5877,
+      "step": 1100
+    },
+    {
+      "epoch": 0.8421052631578947,
+      "grad_norm": 2.53125,
+      "learning_rate": 4.817532517638778e-05,
+      "loss": 0.6098,
+      "step": 1110
+    },
+    {
+      "epoch": 0.8496917970602181,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.811278225683671e-05,
+      "loss": 0.5788,
+      "step": 1120
+    },
+    {
+      "epoch": 0.8572783309625415,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.80492274573878e-05,
+      "loss": 0.6283,
+      "step": 1130
+    },
+    {
+      "epoch": 0.8648648648648649,
+      "grad_norm": 2.515625,
+      "learning_rate": 4.798466356047647e-05,
+      "loss": 0.6017,
+      "step": 1140
+    },
+    {
+      "epoch": 0.8724513987671882,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.791909339271653e-05,
+      "loss": 0.5914,
+      "step": 1150
+    },
+    {
+      "epoch": 0.8800379326695116,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.785251982477642e-05,
+      "loss": 0.6226,
+      "step": 1160
+    },
+    {
+      "epoch": 0.887624466571835,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.778494577125355e-05,
+      "loss": 0.5898,
+      "step": 1170
+    },
+    {
+      "epoch": 0.8952110004741584,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.7716374190546667e-05,
+      "loss": 0.6042,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9027975343764817,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.764680808472638e-05,
+      "loss": 0.5892,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9103840682788051,
+      "grad_norm": 2.375,
+      "learning_rate": 4.75762504994037e-05,
+      "loss": 0.594,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9179706021811285,
+      "grad_norm": 2.59375,
+      "learning_rate": 4.7504704523596686e-05,
+      "loss": 0.5787,
+      "step": 1210
+    },
+    {
+      "epoch": 0.9255571360834519,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.7432173289595275e-05,
+      "loss": 0.5833,
+      "step": 1220
+    },
+    {
+      "epoch": 0.9331436699857752,
+      "grad_norm": 2.375,
+      "learning_rate": 4.735865997282405e-05,
+      "loss": 0.596,
+      "step": 1230
+    },
+    {
+      "epoch": 0.9407302038880986,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.728416779170334e-05,
+      "loss": 0.5865,
+      "step": 1240
+    },
+    {
+      "epoch": 0.948316737790422,
+      "grad_norm": 2.5,
+      "learning_rate": 4.720870000750818e-05,
+      "loss": 0.6107,
+      "step": 1250
+    },
+    {
+      "epoch": 0.9559032716927454,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.713225992422567e-05,
+      "loss": 0.5709,
+      "step": 1260
+    },
+    {
+      "epoch": 0.9634898055950688,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.705485088841023e-05,
+      "loss": 0.5834,
+      "step": 1270
+    },
+    {
+      "epoch": 0.9710763394973921,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.6976476289037106e-05,
+      "loss": 0.5948,
+      "step": 1280
+    },
+    {
+      "epoch": 0.9786628733997155,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.689713955735403e-05,
+      "loss": 0.5863,
+      "step": 1290
+    },
+    {
+      "epoch": 0.9862494073020389,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.681684416673098e-05,
+      "loss": 0.5805,
+      "step": 1300
+    },
+    {
+      "epoch": 0.9938359412043622,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.673559363250811e-05,
+      "loss": 0.5796,
+      "step": 1310
+    },
+    {
+      "epoch": 1.0007586533902324,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.6653391511841876e-05,
+      "loss": 0.5691,
+      "step": 1320
+    },
+    {
+      "epoch": 1.0083451872925557,
+      "grad_norm": 2.375,
+      "learning_rate": 4.657024140354925e-05,
+      "loss": 0.5521,
+      "step": 1330
+    },
+    {
+      "epoch": 1.015931721194879,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.648614694795023e-05,
+      "loss": 0.5593,
+      "step": 1340
+    },
+    {
+      "epoch": 1.0235182550972024,
+      "grad_norm": 2.375,
+      "learning_rate": 4.64011118267084e-05,
+      "loss": 0.5742,
+      "step": 1350
+    },
+    {
+      "epoch": 1.0311047889995257,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.63151397626698e-05,
+      "loss": 0.5513,
+      "step": 1360
+    },
+    {
+      "epoch": 1.0386913229018493,
+      "grad_norm": 2.703125,
+      "learning_rate": 4.622823451969992e-05,
+      "loss": 0.5454,
+      "step": 1370
+    },
+    {
+      "epoch": 1.0462778568041726,
+      "grad_norm": 2.375,
+      "learning_rate": 4.614039990251892e-05,
+      "loss": 0.5678,
+      "step": 1380
+    },
+    {
+      "epoch": 1.053864390706496,
+      "grad_norm": 2.40625,
+      "learning_rate": 4.605163975653502e-05,
+      "loss": 0.5541,
+      "step": 1390
+    },
+    {
+      "epoch": 1.0614509246088193,
+      "grad_norm": 2.390625,
+      "learning_rate": 4.5961957967676216e-05,
+      "loss": 0.5626,
+      "step": 1400
+    },
+    {
+      "epoch": 1.0690374585111426,
+      "grad_norm": 2.4375,
+      "learning_rate": 4.587135846222013e-05,
+      "loss": 0.576,
+      "step": 1410
+    },
+    {
+      "epoch": 1.0766239924134662,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.577984520662207e-05,
+      "loss": 0.5547,
+      "step": 1420
+    },
+    {
+      "epoch": 1.0842105263157895,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.5687422207341434e-05,
+      "loss": 0.5738,
+      "step": 1430
+    },
+    {
+      "epoch": 1.0917970602181128,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.5594093510666317e-05,
+      "loss": 0.5329,
+      "step": 1440
+    },
+    {
+      "epoch": 1.0993835941204362,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.549986320253628e-05,
+      "loss": 0.5444,
+      "step": 1450
+    },
+    {
+      "epoch": 1.1069701280227595,
+      "grad_norm": 2.484375,
+      "learning_rate": 4.540473540836356e-05,
+      "loss": 0.544,
+      "step": 1460
+    },
+    {
+      "epoch": 1.114556661925083,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.530871429285242e-05,
+      "loss": 0.5646,
+      "step": 1470
+    },
+    {
+      "epoch": 1.1221431958274064,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.521180405981681e-05,
+      "loss": 0.5635,
+      "step": 1480
+    },
+    {
+      "epoch": 1.1297297297297297,
+      "grad_norm": 2.640625,
+      "learning_rate": 4.5114008951996325e-05,
+      "loss": 0.5342,
+      "step": 1490
+    },
+    {
+      "epoch": 1.137316263632053,
+      "grad_norm": 2.40625,
+      "learning_rate": 4.50153332508705e-05,
+      "loss": 0.547,
+      "step": 1500
+    },
+    {
+      "epoch": 1.137316263632053,
+      "eval_loss": 0.5373980402946472,
+      "eval_runtime": 9.1273,
+      "eval_samples_per_second": 121.614,
+      "eval_steps_per_second": 15.229,
+      "step": 1500
+    },
+    {
+      "epoch": 1.1449027975343764,
+      "grad_norm": 2.171875,
+      "learning_rate": 4.491578127647129e-05,
+      "loss": 0.5425,
+      "step": 1510
+    },
+    {
+      "epoch": 1.1524893314367,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.481535738719402e-05,
+      "loss": 0.5678,
+      "step": 1520
+    },
+    {
+      "epoch": 1.1600758653390233,
+      "grad_norm": 2.5,
+      "learning_rate": 4.47140659796065e-05,
+      "loss": 0.5589,
+      "step": 1530
+    },
+    {
+      "epoch": 1.1676623992413466,
+      "grad_norm": 2.25,
+      "learning_rate": 4.46119114882566e-05,
+      "loss": 0.5392,
+      "step": 1540
+    },
+    {
+      "epoch": 1.17524893314367,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.450889838547806e-05,
+      "loss": 0.5318,
+      "step": 1550
+    },
+    {
+      "epoch": 1.1828354670459933,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.4405031181194744e-05,
+      "loss": 0.5512,
+      "step": 1560
+    },
+    {
+      "epoch": 1.1904220009483168,
+      "grad_norm": 2.234375,
+      "learning_rate": 4.4300314422723154e-05,
+      "loss": 0.5483,
+      "step": 1570
+    },
+    {
+      "epoch": 1.1980085348506402,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.419475269457335e-05,
+      "loss": 0.5712,
+      "step": 1580
+    },
+    {
+      "epoch": 1.2055950687529635,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.408835061824824e-05,
+      "loss": 0.528,
+      "step": 1590
+    },
+    {
+      "epoch": 1.2131816026552869,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.398111285204128e-05,
+      "loss": 0.5626,
+      "step": 1600
+    },
+    {
+      "epoch": 1.2207681365576102,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.387304409083251e-05,
+      "loss": 0.5391,
+      "step": 1610
+    },
+    {
+      "epoch": 1.2283546704599337,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.376414906588299e-05,
+      "loss": 0.5384,
+      "step": 1620
+    },
+    {
+      "epoch": 1.235941204362257,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.365443254462772e-05,
+      "loss": 0.534,
+      "step": 1630
+    },
+    {
+      "epoch": 1.2435277382645804,
+      "grad_norm": 2.53125,
+      "learning_rate": 4.354389933046687e-05,
+      "loss": 0.5645,
+      "step": 1640
+    },
+    {
+      "epoch": 1.2511142721669037,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.343255426255549e-05,
+      "loss": 0.551,
+      "step": 1650
+    },
+    {
+      "epoch": 1.258700806069227,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.332040221559172e-05,
+      "loss": 0.537,
+      "step": 1660
+    },
+    {
+      "epoch": 1.2662873399715506,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.3207448099603275e-05,
+      "loss": 0.5427,
+      "step": 1670
+    },
+    {
+      "epoch": 1.2738738738738737,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.3093696859732546e-05,
+      "loss": 0.5327,
+      "step": 1680
+    },
+    {
+      "epoch": 1.2814604077761973,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.29791534760201e-05,
+      "loss": 0.5518,
+      "step": 1690
+    },
+    {
+      "epoch": 1.2890469416785206,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.2863822963186604e-05,
+      "loss": 0.5531,
+      "step": 1700
+    },
+    {
+      "epoch": 1.296633475580844,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.274771037041336e-05,
+      "loss": 0.5352,
+      "step": 1710
+    },
+    {
+      "epoch": 1.3042200094831673,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.263082078112119e-05,
+      "loss": 0.5569,
+      "step": 1720
+    },
+    {
+      "epoch": 1.3118065433854906,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.251315931274789e-05,
+      "loss": 0.5288,
+      "step": 1730
+    },
+    {
+      "epoch": 1.3193930772878142,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.23947311165242e-05,
+      "loss": 0.5396,
+      "step": 1740
+    },
+    {
+      "epoch": 1.3269796111901375,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.227554137724829e-05,
+      "loss": 0.5304,
+      "step": 1750
+    },
+    {
+      "epoch": 1.3345661450924609,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.2155595313058776e-05,
+      "loss": 0.5438,
+      "step": 1760
+    },
+    {
+      "epoch": 1.3421526789947842,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.203489817520622e-05,
+      "loss": 0.5312,
+      "step": 1770
+    },
+    {
+      "epoch": 1.3497392128971075,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.191345524782327e-05,
+      "loss": 0.5271,
+      "step": 1780
+    },
+    {
+      "epoch": 1.357325746799431,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.1791271847693336e-05,
+      "loss": 0.5361,
+      "step": 1790
+    },
+    {
+      "epoch": 1.3649122807017544,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.166835332401779e-05,
+      "loss": 0.5469,
+      "step": 1800
+    },
+    {
+      "epoch": 1.3724988146040777,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.154470505818177e-05,
+      "loss": 0.5259,
+      "step": 1810
+    },
+    {
+      "epoch": 1.380085348506401,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.142033246351861e-05,
+      "loss": 0.5232,
+      "step": 1820
+    },
+    {
+      "epoch": 1.3876718824087244,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.129524098507283e-05,
+      "loss": 0.526,
+      "step": 1830
+    },
+    {
+      "epoch": 1.395258416311048,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.116943609936176e-05,
+      "loss": 0.5437,
+      "step": 1840
+    },
+    {
+      "epoch": 1.4028449502133713,
+      "grad_norm": 2.25,
+      "learning_rate": 4.1042923314135764e-05,
+      "loss": 0.5103,
+      "step": 1850
+    },
+    {
+      "epoch": 1.4104314841156946,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.0915708168137104e-05,
+      "loss": 0.5345,
+      "step": 1860
+    },
+    {
+      "epoch": 1.418018018018018,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.07877962308575e-05,
+      "loss": 0.5547,
+      "step": 1870
+    },
+    {
+      "epoch": 1.4256045519203413,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.0659193102294224e-05,
+      "loss": 0.5374,
+      "step": 1880
+    },
+    {
+      "epoch": 1.4331910858226649,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.0529904412704997e-05,
+      "loss": 0.5271,
+      "step": 1890
+    },
+    {
+      "epoch": 1.4407776197249882,
+      "grad_norm": 2.375,
+      "learning_rate": 4.039993582236148e-05,
+      "loss": 0.5453,
+      "step": 1900
+    },
+    {
+      "epoch": 1.4483641536273115,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.0269293021301406e-05,
+      "loss": 0.5063,
+      "step": 1910
+    },
+    {
+      "epoch": 1.4559506875296349,
+      "grad_norm": 2.390625,
+      "learning_rate": 4.013798172907958e-05,
+      "loss": 0.5453,
+      "step": 1920
+    },
+    {
+      "epoch": 1.4635372214319582,
+      "grad_norm": 2.203125,
+      "learning_rate": 4.0006007694517376e-05,
+      "loss": 0.5403,
+      "step": 1930
+    },
+    {
+      "epoch": 1.4711237553342817,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.9873376695451114e-05,
+      "loss": 0.5216,
+      "step": 1940
+    },
+    {
+      "epoch": 1.478710289236605,
+      "grad_norm": 2.25,
+      "learning_rate": 3.9740094538479064e-05,
+      "loss": 0.5573,
+      "step": 1950
+    },
+    {
+      "epoch": 1.4862968231389284,
+      "grad_norm": 2.375,
+      "learning_rate": 3.960616705870729e-05,
+      "loss": 0.5403,
+      "step": 1960
+    },
+    {
+      "epoch": 1.4938833570412517,
+      "grad_norm": 2.625,
+      "learning_rate": 3.947160011949411e-05,
+      "loss": 0.5385,
+      "step": 1970
+    },
+    {
+      "epoch": 1.501469890943575,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.9336399612193465e-05,
+      "loss": 0.5354,
+      "step": 1980
+    },
+    {
+      "epoch": 1.5090564248458986,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.920057145589697e-05,
+      "loss": 0.523,
+      "step": 1990
+    },
+    {
+      "epoch": 1.5166429587482217,
+      "grad_norm": 2.46875,
+      "learning_rate": 3.9064121597174764e-05,
+      "loss": 0.5451,
+      "step": 2000
+    },
+    {
+      "epoch": 1.5166429587482217,
+      "eval_loss": 0.5127466917037964,
+      "eval_runtime": 9.1468,
+      "eval_samples_per_second": 121.354,
+      "eval_steps_per_second": 15.197,
+      "step": 2000
+    },
+    {
+      "epoch": 1.5242294926505453,
+      "grad_norm": 2.375,
+      "learning_rate": 3.8927056009815176e-05,
+      "loss": 0.5269,
+      "step": 2010
+    },
+    {
+      "epoch": 1.5318160265528686,
+      "grad_norm": 2.390625,
+      "learning_rate": 3.878938069456322e-05,
+      "loss": 0.5322,
+      "step": 2020
+    },
+    {
+      "epoch": 1.539402560455192,
+      "grad_norm": 2.328125,
+      "learning_rate": 3.865110167885784e-05,
+      "loss": 0.5322,
+      "step": 2030
+    },
+    {
+      "epoch": 1.5469890943575155,
+      "grad_norm": 2.28125,
+      "learning_rate": 3.8512225016568034e-05,
+      "loss": 0.5373,
+      "step": 2040
+    },
+    {
+      "epoch": 1.5545756282598386,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.837275678772786e-05,
+      "loss": 0.5194,
+      "step": 2050
+    },
+    {
+      "epoch": 1.5621621621621622,
+      "grad_norm": 2.15625,
+      "learning_rate": 3.8232703098270186e-05,
+      "loss": 0.5304,
+      "step": 2060
+    },
+    {
+      "epoch": 1.5697486960644855,
+      "grad_norm": 2.421875,
+      "learning_rate": 3.8092070079759426e-05,
+      "loss": 0.5245,
+      "step": 2070
+    },
+    {
+      "epoch": 1.5773352299668089,
+      "grad_norm": 2.359375,
+      "learning_rate": 3.795086388912305e-05,
+      "loss": 0.5351,
+      "step": 2080
+    },
+    {
+      "epoch": 1.5849217638691324,
+      "grad_norm": 2.125,
+      "learning_rate": 3.78090907083821e-05,
+      "loss": 0.5214,
+      "step": 2090
+    },
+    {
+      "epoch": 1.5925082977714555,
+      "grad_norm": 2.09375,
+      "learning_rate": 3.766675674438047e-05,
+      "loss": 0.5308,
+      "step": 2100
+    },
+    {
+      "epoch": 1.600094831673779,
+      "grad_norm": 2.25,
+      "learning_rate": 3.7523868228513207e-05,
+      "loss": 0.5231,
+      "step": 2110
+    },
+    {
+      "epoch": 1.6076813655761024,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.7380431416453706e-05,
+      "loss": 0.5366,
+      "step": 2120
+    },
+    {
+      "epoch": 1.6152678994784258,
+      "grad_norm": 2.359375,
+      "learning_rate": 3.72364525878798e-05,
+      "loss": 0.5273,
+      "step": 2130
+    },
+    {
+      "epoch": 1.6228544333807493,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.709193804619889e-05,
+      "loss": 0.5111,
+      "step": 2140
+    },
+    {
+      "epoch": 1.6304409672830724,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.694689411827192e-05,
+      "loss": 0.5231,
+      "step": 2150
+    },
+    {
+      "epoch": 1.638027501185396,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.680132715413644e-05,
+      "loss": 0.5219,
+      "step": 2160
+    },
+    {
+      "epoch": 1.6456140350877193,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.6655243526728555e-05,
+      "loss": 0.5249,
+      "step": 2170
+    },
+    {
+      "epoch": 1.6532005689900426,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.6508649631603967e-05,
+      "loss": 0.5177,
+      "step": 2180
+    },
+    {
+      "epoch": 1.6607871028923662,
+      "grad_norm": 2.21875,
+      "learning_rate": 3.6361551886657915e-05,
+      "loss": 0.5548,
+      "step": 2190
+    },
+    {
+      "epoch": 1.6683736367946893,
+      "grad_norm": 2.25,
+      "learning_rate": 3.621395673184426e-05,
+      "loss": 0.5304,
+      "step": 2200
+    },
+    {
+      "epoch": 1.6759601706970129,
+      "grad_norm": 2.015625,
+      "learning_rate": 3.6065870628893504e-05,
+      "loss": 0.5168,
+      "step": 2210
+    },
+    {
+      "epoch": 1.6835467045993362,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.591730006102992e-05,
+      "loss": 0.5051,
+      "step": 2220
+    },
+    {
+      "epoch": 1.6911332385016595,
+      "grad_norm": 2.0625,
+      "learning_rate": 3.576825153268768e-05,
+      "loss": 0.5134,
+      "step": 2230
+    },
+    {
+      "epoch": 1.698719772403983,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.561873156922614e-05,
+      "loss": 0.526,
+      "step": 2240
+    },
+    {
+      "epoch": 1.7063063063063062,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.546874671664411e-05,
+      "loss": 0.5313,
+      "step": 2250
+    },
+    {
+      "epoch": 1.7138928402086298,
+      "grad_norm": 2.375,
+      "learning_rate": 3.531830354129329e-05,
+      "loss": 0.5292,
+      "step": 2260
+    },
+    {
+      "epoch": 1.721479374110953,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.5167408629590835e-05,
+      "loss": 0.5319,
+      "step": 2270
+    },
+    {
+      "epoch": 1.7290659080132764,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.501606858773092e-05,
+      "loss": 0.5208,
+      "step": 2280
+    },
+    {
+      "epoch": 1.7366524419156,
+      "grad_norm": 2.53125,
+      "learning_rate": 3.486429004139556e-05,
+      "loss": 0.514,
+      "step": 2290
+    },
+    {
+      "epoch": 1.744238975817923,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.4712079635464587e-05,
+      "loss": 0.5272,
+      "step": 2300
+    },
+    {
+      "epoch": 1.7518255097202466,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.455944403372463e-05,
+      "loss": 0.5414,
+      "step": 2310
+    },
+    {
+      "epoch": 1.75941204362257,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.440638991857747e-05,
+      "loss": 0.5363,
+      "step": 2320
+    },
+    {
+      "epoch": 1.7669985775248933,
+      "grad_norm": 2.25,
+      "learning_rate": 3.425292399074744e-05,
+      "loss": 0.5209,
+      "step": 2330
+    },
+    {
+      "epoch": 1.7745851114272166,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.4099052968988076e-05,
+      "loss": 0.5093,
+      "step": 2340
+    },
+    {
+      "epoch": 1.78217164532954,
+      "grad_norm": 2.28125,
+      "learning_rate": 3.394478358978795e-05,
+      "loss": 0.5118,
+      "step": 2350
+    },
+    {
+      "epoch": 1.7897581792318635,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.379012260707577e-05,
+      "loss": 0.5321,
+      "step": 2360
+    },
+    {
+      "epoch": 1.7973447131341869,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.363507679192469e-05,
+      "loss": 0.5126,
+      "step": 2370
+    },
+    {
+      "epoch": 1.8049312470365102,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.347965293225587e-05,
+      "loss": 0.5208,
+      "step": 2380
+    },
+    {
+      "epoch": 1.8125177809388335,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.3323857832541295e-05,
+      "loss": 0.5232,
+      "step": 2390
+    },
+    {
+      "epoch": 1.8201043148411569,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.316769831350585e-05,
+      "loss": 0.5189,
+      "step": 2400
+    },
+    {
+      "epoch": 1.8276908487434804,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.3011181211828766e-05,
+      "loss": 0.5144,
+      "step": 2410
+    },
+    {
+      "epoch": 1.8352773826458035,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.285431337984427e-05,
+      "loss": 0.513,
+      "step": 2420
+    },
+    {
+      "epoch": 1.842863916548127,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.269710168524159e-05,
+      "loss": 0.5085,
+      "step": 2430
+    },
+    {
+      "epoch": 1.8504504504504504,
+      "grad_norm": 2.21875,
+      "learning_rate": 3.253955301076429e-05,
+      "loss": 0.4978,
+      "step": 2440
+    },
+    {
+      "epoch": 1.8580369843527738,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.2381674253908965e-05,
+      "loss": 0.5428,
+      "step": 2450
+    },
+    {
+      "epoch": 1.8656235182550973,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.222347232662323e-05,
+      "loss": 0.5292,
+      "step": 2460
+    },
+    {
+      "epoch": 1.8732100521574204,
+      "grad_norm": 2.15625,
+      "learning_rate": 3.206495415500314e-05,
+      "loss": 0.5366,
+      "step": 2470
+    },
+    {
+      "epoch": 1.880796586059744,
+      "grad_norm": 2.328125,
+      "learning_rate": 3.190612667898996e-05,
+      "loss": 0.5114,
+      "step": 2480
+    },
+    {
+      "epoch": 1.8883831199620673,
+      "grad_norm": 2.078125,
+      "learning_rate": 3.1746996852066324e-05,
+      "loss": 0.5093,
+      "step": 2490
+    },
+    {
+      "epoch": 1.8959696538643906,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.158757164095183e-05,
+      "loss": 0.5014,
+      "step": 2500
+    },
+    {
+      "epoch": 1.8959696538643906,
+      "eval_loss": 0.49517810344696045,
+      "eval_runtime": 9.1582,
+      "eval_samples_per_second": 121.203,
+      "eval_steps_per_second": 15.178,
+      "step": 2500
+    },
+    {
+      "epoch": 1.9035561877667142,
+      "grad_norm": 2.234375,
+      "learning_rate": 3.142785802529804e-05,
+      "loss": 0.5117,
+      "step": 2510
+    },
+    {
+      "epoch": 1.9111427216690373,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.126786299738285e-05,
+      "loss": 0.5346,
+      "step": 2520
+    },
+    {
+      "epoch": 1.9187292555713609,
+      "grad_norm": 2.21875,
+      "learning_rate": 3.1107593561804436e-05,
+      "loss": 0.5203,
+      "step": 2530
+    },
+    {
+      "epoch": 1.9263157894736842,
+      "grad_norm": 2.421875,
+      "learning_rate": 3.0947056735174596e-05,
+      "loss": 0.5311,
+      "step": 2540
+    },
+    {
+      "epoch": 1.9339023233760075,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.078625954581149e-05,
+      "loss": 0.5109,
+      "step": 2550
+    },
+    {
+      "epoch": 1.941488857278331,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.0625209033432015e-05,
+      "loss": 0.5256,
+      "step": 2560
+    },
+    {
+      "epoch": 1.9490753911806542,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.0463912248843563e-05,
+      "loss": 0.5216,
+      "step": 2570
+    },
+    {
+      "epoch": 1.9566619250829778,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.0302376253635347e-05,
+      "loss": 0.5134,
+      "step": 2580
+    },
+    {
+      "epoch": 1.964248458985301,
+      "grad_norm": 2.234375,
+      "learning_rate": 3.014060811986924e-05,
+      "loss": 0.5234,
+      "step": 2590
+    },
+    {
+      "epoch": 1.9718349928876244,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.9978614929770176e-05,
+      "loss": 0.5114,
+      "step": 2600
+    },
+    {
+      "epoch": 1.979421526789948,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.981640377541608e-05,
+      "loss": 0.5085,
+      "step": 2610
+    },
+    {
+      "epoch": 1.987008060692271,
+      "grad_norm": 2.25,
+      "learning_rate": 2.965398175842735e-05,
+      "loss": 0.5002,
+      "step": 2620
+    },
+    {
+      "epoch": 1.9945945945945946,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.9491355989656e-05,
+      "loss": 0.5161,
+      "step": 2630
+    },
+    {
+      "epoch": 2.001517306780465,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.9328533588874295e-05,
+      "loss": 0.52,
+      "step": 2640
+    },
+    {
+      "epoch": 2.009103840682788,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.9165521684463087e-05,
+      "loss": 0.487,
+      "step": 2650
+    },
+    {
+      "epoch": 2.0166903745851115,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.9002327413099705e-05,
+      "loss": 0.4851,
+      "step": 2660
+    },
+    {
+      "epoch": 2.024276908487435,
+      "grad_norm": 2.421875,
+      "learning_rate": 2.8838957919445515e-05,
+      "loss": 0.498,
+      "step": 2670
+    },
+    {
+      "epoch": 2.031863442389758,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.867542035583315e-05,
+      "loss": 0.4904,
+      "step": 2680
+    },
+    {
+      "epoch": 2.0394499762920817,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.8511721881953335e-05,
+      "loss": 0.4733,
+      "step": 2690
+    },
+    {
+      "epoch": 2.047036510194405,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.8347869664541503e-05,
+      "loss": 0.4997,
+      "step": 2700
+    },
+    {
+      "epoch": 2.0546230440967284,
+      "grad_norm": 2.203125,
+      "learning_rate": 2.8183870877063963e-05,
+      "loss": 0.4996,
+      "step": 2710
+    },
+    {
+      "epoch": 2.0622095779990515,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.801973269940388e-05,
+      "loss": 0.5077,
+      "step": 2720
+    },
+    {
+      "epoch": 2.069796111901375,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.7855462317546944e-05,
+      "loss": 0.4922,
+      "step": 2730
+    },
+    {
+      "epoch": 2.0773826458036986,
+      "grad_norm": 2.25,
+      "learning_rate": 2.7691066923266746e-05,
+      "loss": 0.4942,
+      "step": 2740
+    },
+    {
+      "epoch": 2.0849691797060217,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.7526553713809938e-05,
+      "loss": 0.4947,
+      "step": 2750
+    },
+    {
+      "epoch": 2.0925557136083452,
+      "grad_norm": 2.203125,
+      "learning_rate": 2.736192989158115e-05,
+      "loss": 0.4903,
+      "step": 2760
+    },
+    {
+      "epoch": 2.1001422475106684,
+      "grad_norm": 2.125,
+      "learning_rate": 2.7197202663827625e-05,
+      "loss": 0.4803,
+      "step": 2770
+    },
+    {
+      "epoch": 2.107728781412992,
+      "grad_norm": 2.015625,
+      "learning_rate": 2.7032379242323724e-05,
+      "loss": 0.491,
+      "step": 2780
+    },
+    {
+      "epoch": 2.1153153153153155,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.6867466843055166e-05,
+      "loss": 0.4782,
+      "step": 2790
+    },
+    {
+      "epoch": 2.1229018492176386,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.6702472685903134e-05,
+      "loss": 0.4956,
+      "step": 2800
+    },
+    {
+      "epoch": 2.130488383119962,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.6537403994328198e-05,
+      "loss": 0.4951,
+      "step": 2810
+    },
+    {
+      "epoch": 2.1380749170222852,
+      "grad_norm": 2.015625,
+      "learning_rate": 2.6372267995054023e-05,
+      "loss": 0.4723,
+      "step": 2820
+    },
+    {
+      "epoch": 2.145661450924609,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.620707191775102e-05,
+      "loss": 0.4914,
+      "step": 2830
+    },
+    {
+      "epoch": 2.1532479848269324,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.604182299471983e-05,
+      "loss": 0.4747,
+      "step": 2840
+    },
+    {
+      "epoch": 2.1608345187292555,
+      "grad_norm": 2.34375,
+      "learning_rate": 2.5876528460574672e-05,
+      "loss": 0.4858,
+      "step": 2850
+    },
+    {
+      "epoch": 2.168421052631579,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.5711195551926646e-05,
+      "loss": 0.4798,
+      "step": 2860
+    },
+    {
+      "epoch": 2.176007586533902,
+      "grad_norm": 2.4375,
+      "learning_rate": 2.5545831507066875e-05,
+      "loss": 0.4857,
+      "step": 2870
+    },
+    {
+      "epoch": 2.1835941204362257,
+      "grad_norm": 2.0625,
+      "learning_rate": 2.538044356564963e-05,
+      "loss": 0.4845,
+      "step": 2880
+    },
+    {
+      "epoch": 2.1911806543385492,
+      "grad_norm": 2.25,
+      "learning_rate": 2.5215038968375376e-05,
+      "loss": 0.4827,
+      "step": 2890
+    },
+    {
+      "epoch": 2.1987671882408724,
+      "grad_norm": 1.96875,
+      "learning_rate": 2.5049624956673773e-05,
+      "loss": 0.4728,
+      "step": 2900
+    },
+    {
+      "epoch": 2.206353722143196,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.4884208772386644e-05,
+      "loss": 0.4836,
+      "step": 2910
+    },
+    {
+      "epoch": 2.213940256045519,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.4718797657450944e-05,
+      "loss": 0.4866,
+      "step": 2920
+    },
+    {
+      "epoch": 2.2215267899478426,
+      "grad_norm": 2.375,
+      "learning_rate": 2.455339885358166e-05,
+      "loss": 0.4871,
+      "step": 2930
+    },
+    {
+      "epoch": 2.229113323850166,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.4388019601954843e-05,
+      "loss": 0.4844,
+      "step": 2940
+    },
+    {
+      "epoch": 2.2366998577524893,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.4222667142890488e-05,
+      "loss": 0.4649,
+      "step": 2950
+    },
+    {
+      "epoch": 2.244286391654813,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.4057348715535662e-05,
+      "loss": 0.4851,
+      "step": 2960
+    },
+    {
+      "epoch": 2.251872925557136,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.389207155754749e-05,
+      "loss": 0.4977,
+      "step": 2970
+    },
+    {
+      "epoch": 2.2594594594594595,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.3726842904776338e-05,
+      "loss": 0.4798,
+      "step": 2980
+    },
+    {
+      "epoch": 2.267045993361783,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.356166999094899e-05,
+      "loss": 0.4967,
+      "step": 2990
+    },
+    {
+      "epoch": 2.274632527264106,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.3396560047351984e-05,
+      "loss": 0.4817,
+      "step": 3000
+    },
+    {
+      "epoch": 2.274632527264106,
+      "eval_loss": 0.4872964918613434,
+      "eval_runtime": 9.1611,
+      "eval_samples_per_second": 121.164,
+      "eval_steps_per_second": 15.173,
+      "step": 3000
+    },
+    {
+      "epoch": 2.2822190611664297,
+      "grad_norm": 2.34375,
+      "learning_rate": 2.323152030251502e-05,
+      "loss": 0.4819,
+      "step": 3010
+    },
+    {
+      "epoch": 2.289805595068753,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.3066557981894476e-05,
+      "loss": 0.488,
+      "step": 3020
+    },
+    {
+      "epoch": 2.2973921289710764,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.2901680307557104e-05,
+      "loss": 0.481,
+      "step": 3030
+    },
+    {
+      "epoch": 2.3049786628734,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.2736894497863828e-05,
+      "loss": 0.4896,
+      "step": 3040
+    },
+    {
+      "epoch": 2.312565196775723,
+      "grad_norm": 2.0,
+      "learning_rate": 2.2572207767153718e-05,
+      "loss": 0.4781,
+      "step": 3050
+    },
+    {
+      "epoch": 2.3201517306780466,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.2407627325428177e-05,
+      "loss": 0.4711,
+      "step": 3060
+    },
+    {
+      "epoch": 2.3277382645803697,
+      "grad_norm": 2.25,
+      "learning_rate": 2.2243160378035244e-05,
+      "loss": 0.489,
+      "step": 3070
+    },
+    {
+      "epoch": 2.3353247984826933,
+      "grad_norm": 2.46875,
+      "learning_rate": 2.2078814125354192e-05,
+      "loss": 0.4967,
+      "step": 3080
+    },
+    {
+      "epoch": 2.3429113323850164,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.1914595762480233e-05,
+      "loss": 0.4815,
+      "step": 3090
+    },
+    {
+      "epoch": 2.35049786628734,
+      "grad_norm": 1.9765625,
+      "learning_rate": 2.1750512478909586e-05,
+      "loss": 0.4705,
+      "step": 3100
+    },
+    {
+      "epoch": 2.3580844001896635,
+      "grad_norm": 2.0625,
+      "learning_rate": 2.158657145822465e-05,
+      "loss": 0.4862,
+      "step": 3110
+    },
+    {
+      "epoch": 2.3656709340919866,
+      "grad_norm": 2.03125,
+      "learning_rate": 2.142277987777957e-05,
+      "loss": 0.4628,
+      "step": 3120
+    },
+    {
+      "epoch": 2.37325746799431,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.125914490838596e-05,
+      "loss": 0.4713,
+      "step": 3130
+    },
+    {
+      "epoch": 2.3808440018966337,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.1095673713999018e-05,
+      "loss": 0.4837,
+      "step": 3140
+    },
+    {
+      "epoch": 2.388430535798957,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.093237345140381e-05,
+      "loss": 0.4907,
+      "step": 3150
+    },
+    {
+      "epoch": 2.3960170697012804,
+      "grad_norm": 2.25,
+      "learning_rate": 2.076925126990204e-05,
+      "loss": 0.4757,
+      "step": 3160
+    },
+    {
+      "epoch": 2.4036036036036035,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.060631431099896e-05,
+      "loss": 0.4785,
+      "step": 3170
+    },
+    {
+      "epoch": 2.411190137505927,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.044356970809079e-05,
+      "loss": 0.4726,
+      "step": 3180
+    },
+    {
+      "epoch": 2.41877667140825,
+      "grad_norm": 2.0,
+      "learning_rate": 2.028102458615237e-05,
+      "loss": 0.4744,
+      "step": 3190
+    },
+    {
+      "epoch": 2.4263632053105737,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.0118686061425257e-05,
+      "loss": 0.4813,
+      "step": 3200
+    },
+    {
+      "epoch": 2.4339497392128973,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.995656124110614e-05,
+      "loss": 0.4929,
+      "step": 3210
+    },
+    {
+      "epoch": 2.4415362731152204,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.979465722303573e-05,
+      "loss": 0.4874,
+      "step": 3220
+    },
+    {
+      "epoch": 2.449122807017544,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.9632981095387973e-05,
+      "loss": 0.4786,
+      "step": 3230
+    },
+    {
+      "epoch": 2.4567093409198675,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.947153993635978e-05,
+      "loss": 0.4864,
+      "step": 3240
+    },
+    {
+      "epoch": 2.4642958748221906,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.9310340813861067e-05,
+      "loss": 0.4957,
+      "step": 3250
+    },
+    {
+      "epoch": 2.471882408724514,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9149390785205416e-05,
+      "loss": 0.501,
+      "step": 3260
+    },
+    {
+      "epoch": 2.4794689426268373,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.8988696896801018e-05,
+      "loss": 0.4826,
+      "step": 3270
+    },
+    {
+      "epoch": 2.487055476529161,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.882826618384224e-05,
+      "loss": 0.4844,
+      "step": 3280
+    },
+    {
+      "epoch": 2.494642010431484,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.866810567000159e-05,
+      "loss": 0.4929,
+      "step": 3290
+    },
+    {
+      "epoch": 2.5022285443338075,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.8508222367122254e-05,
+      "loss": 0.469,
+      "step": 3300
+    },
+    {
+      "epoch": 2.509815078236131,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.8348623274911057e-05,
+      "loss": 0.4721,
+      "step": 3310
+    },
+    {
+      "epoch": 2.517401612138454,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.8189315380632073e-05,
+      "loss": 0.476,
+      "step": 3320
+    },
+    {
+      "epoch": 2.5249881460407777,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.8030305658800718e-05,
+      "loss": 0.4621,
+      "step": 3330
+    },
+    {
+      "epoch": 2.5325746799431013,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.787160107087835e-05,
+      "loss": 0.4824,
+      "step": 3340
+    },
+    {
+      "epoch": 2.5401612138454244,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.771320856496759e-05,
+      "loss": 0.4868,
+      "step": 3350
+    },
+    {
+      "epoch": 2.5477477477477475,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.7555135075508013e-05,
+      "loss": 0.4811,
+      "step": 3360
+    },
+    {
+      "epoch": 2.555334281650071,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.739738752297267e-05,
+      "loss": 0.4813,
+      "step": 3370
+    },
+    {
+      "epoch": 2.5629208155523946,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.7239972813565037e-05,
+      "loss": 0.4694,
+      "step": 3380
+    },
+    {
+      "epoch": 2.5705073494547177,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.708289783891671e-05,
+      "loss": 0.4821,
+      "step": 3390
+    },
+    {
+      "epoch": 2.5780938833570413,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.692616947578562e-05,
+      "loss": 0.4894,
+      "step": 3400
+    },
+    {
+      "epoch": 2.585680417259365,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6769794585755057e-05,
+      "loss": 0.4787,
+      "step": 3410
+    },
+    {
+      "epoch": 2.593266951161688,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.6613780014933193e-05,
+      "loss": 0.4758,
+      "step": 3420
+    },
+    {
+      "epoch": 2.6008534850640115,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6458132593653407e-05,
+      "loss": 0.4642,
+      "step": 3430
+    },
+    {
+      "epoch": 2.6084400189663346,
+      "grad_norm": 2.125,
+      "learning_rate": 1.6302859136175228e-05,
+      "loss": 0.4835,
+      "step": 3440
+    },
+    {
+      "epoch": 2.616026552868658,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.614796644038602e-05,
+      "loss": 0.4801,
+      "step": 3450
+    },
+    {
+      "epoch": 2.6236130867709813,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.5993461287503337e-05,
+      "loss": 0.4723,
+      "step": 3460
+    },
+    {
+      "epoch": 2.631199620673305,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.5839350441778104e-05,
+      "loss": 0.4958,
+      "step": 3470
+    },
+    {
+      "epoch": 2.6387861545756284,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.56856406501984e-05,
+      "loss": 0.4782,
+      "step": 3480
+    },
+    {
+      "epoch": 2.6463726884779515,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.5532338642194154e-05,
+      "loss": 0.4745,
+      "step": 3490
+    },
+    {
+      "epoch": 2.653959222380275,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.5379451129342456e-05,
+      "loss": 0.4852,
+      "step": 3500
+    },
+    {
+      "epoch": 2.653959222380275,
+      "eval_loss": 0.48158586025238037,
+      "eval_runtime": 9.1707,
+      "eval_samples_per_second": 121.038,
+      "eval_steps_per_second": 15.157,
+      "step": 3500
+    },
+    {
+      "epoch": 2.6615457562825986,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.5226984805073786e-05,
+      "loss": 0.4557,
+      "step": 3510
+    },
+    {
+      "epoch": 2.6691322901849217,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.507494634437891e-05,
+      "loss": 0.4689,
+      "step": 3520
+    },
+    {
+      "epoch": 2.6767188240872453,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.4923342403516721e-05,
+      "loss": 0.4804,
+      "step": 3530
+    },
+    {
+      "epoch": 2.6843053579895684,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.477217961972277e-05,
+      "loss": 0.4719,
+      "step": 3540
+    },
+    {
+      "epoch": 2.691891891891892,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.4621464610918717e-05,
+      "loss": 0.4818,
+      "step": 3550
+    },
+    {
+      "epoch": 2.699478425794215,
+      "grad_norm": 2.375,
+      "learning_rate": 1.4471203975422576e-05,
+      "loss": 0.4962,
+      "step": 3560
+    },
+    {
+      "epoch": 2.7070649596965386,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.432140429165987e-05,
+      "loss": 0.4808,
+      "step": 3570
+    },
+    {
+      "epoch": 2.714651493598862,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.417207211787559e-05,
+      "loss": 0.4773,
+      "step": 3580
+    },
+    {
+      "epoch": 2.7222380275011853,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.4023213991847118e-05,
+      "loss": 0.4613,
+      "step": 3590
+    },
+    {
+      "epoch": 2.729824561403509,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.3874836430597943e-05,
+      "loss": 0.4942,
+      "step": 3600
+    },
+    {
+      "epoch": 2.7374110953058324,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.3726945930112418e-05,
+      "loss": 0.4747,
+      "step": 3610
+    },
+    {
+      "epoch": 2.7449976292081555,
+      "grad_norm": 2.25,
+      "learning_rate": 1.3579548965051305e-05,
+      "loss": 0.4835,
+      "step": 3620
+    },
+    {
+      "epoch": 2.752584163110479,
+      "grad_norm": 2.125,
+      "learning_rate": 1.343265198846836e-05,
+      "loss": 0.4849,
+      "step": 3630
+    },
+    {
+      "epoch": 2.760170697012802,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.3286261431527747e-05,
+      "loss": 0.4773,
+      "step": 3640
+    },
+    {
+      "epoch": 2.7677572309151257,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.3140383703222583e-05,
+      "loss": 0.4891,
+      "step": 3650
+    },
+    {
+      "epoch": 2.775343764817449,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.2995025190094251e-05,
+      "loss": 0.4867,
+      "step": 3660
+    },
+    {
+      "epoch": 2.7829302987197724,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.2850192255952875e-05,
+      "loss": 0.4754,
+      "step": 3670
+    },
+    {
+      "epoch": 2.790516832622096,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.2705891241598655e-05,
+      "loss": 0.4634,
+      "step": 3680
+    },
+    {
+      "epoch": 2.798103366524419,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.2562128464544315e-05,
+      "loss": 0.4776,
+      "step": 3690
+    },
+    {
+      "epoch": 2.8056899004267426,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.2418910218738467e-05,
+      "loss": 0.4802,
+      "step": 3700
+    },
+    {
+      "epoch": 2.813276434329066,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.227624277429012e-05,
+      "loss": 0.4902,
+      "step": 3710
+    },
+    {
+      "epoch": 2.8208629682313893,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.2134132377194112e-05,
+      "loss": 0.4655,
+      "step": 3720
+    },
+    {
+      "epoch": 2.828449502133713,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.1992585249057727e-05,
+      "loss": 0.4918,
+      "step": 3730
+    },
+    {
+      "epoch": 2.836036036036036,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.1851607586828262e-05,
+      "loss": 0.4683,
+      "step": 3740
+    },
+    {
+      "epoch": 2.8436225699383595,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.1711205562521745e-05,
+      "loss": 0.4792,
+      "step": 3750
+    },
+    {
+      "epoch": 2.8512091038406826,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.1571385322952702e-05,
+      "loss": 0.4848,
+      "step": 3760
+    },
+    {
+      "epoch": 2.858795637743006,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.1432152989465094e-05,
+      "loss": 0.4882,
+      "step": 3770
+    },
+    {
+      "epoch": 2.8663821716453297,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.1293514657664264e-05,
+      "loss": 0.4813,
+      "step": 3780
+    },
+    {
+      "epoch": 2.873968705547653,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.1155476397150133e-05,
+      "loss": 0.4697,
+      "step": 3790
+    },
+    {
+      "epoch": 2.8815552394499764,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.1018044251251415e-05,
+      "loss": 0.4685,
+      "step": 3800
+    },
+    {
+      "epoch": 2.8891417733523,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.0881224236761087e-05,
+      "loss": 0.4607,
+      "step": 3810
+    },
+    {
+      "epoch": 2.896728307254623,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.074502234367293e-05,
+      "loss": 0.4671,
+      "step": 3820
+    },
+    {
+      "epoch": 2.904314841156946,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.0609444534919334e-05,
+      "loss": 0.478,
+      "step": 3830
+    },
+    {
+      "epoch": 2.9119013750592697,
+      "grad_norm": 2.375,
+      "learning_rate": 1.0474496746110197e-05,
+      "loss": 0.4618,
+      "step": 3840
+    },
+    {
+      "epoch": 2.9194879089615933,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.03401848852731e-05,
+      "loss": 0.48,
+      "step": 3850
+    },
+    {
+      "epoch": 2.9270744428639164,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.0206514832594603e-05,
+      "loss": 0.4669,
+      "step": 3860
+    },
+    {
+      "epoch": 2.93466097676624,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.0073492440162876e-05,
+      "loss": 0.4728,
+      "step": 3870
+    },
+    {
+      "epoch": 2.9422475106685635,
+      "grad_norm": 2.171875,
+      "learning_rate": 9.941123531711428e-06,
+      "loss": 0.4852,
+      "step": 3880
+    },
+    {
+      "epoch": 2.9498340445708866,
+      "grad_norm": 2.28125,
+      "learning_rate": 9.809413902364206e-06,
+      "loss": 0.4902,
+      "step": 3890
+    },
+    {
+      "epoch": 2.95742057847321,
+      "grad_norm": 2.328125,
+      "learning_rate": 9.678369318381816e-06,
+      "loss": 0.4816,
+      "step": 3900
+    },
+    {
+      "epoch": 2.9650071123755333,
+      "grad_norm": 2.109375,
+      "learning_rate": 9.547995516909156e-06,
+      "loss": 0.4821,
+      "step": 3910
+    },
+    {
+      "epoch": 2.972593646277857,
+      "grad_norm": 2.015625,
+      "learning_rate": 9.418298205724152e-06,
+      "loss": 0.4784,
+      "step": 3920
+    },
+    {
+      "epoch": 2.98018018018018,
+      "grad_norm": 2.265625,
+      "learning_rate": 9.289283062987953e-06,
+      "loss": 0.4788,
+      "step": 3930
+    },
+    {
+      "epoch": 2.9877667140825035,
+      "grad_norm": 2.015625,
+      "learning_rate": 9.160955736996301e-06,
+      "loss": 0.4642,
+      "step": 3940
+    },
+    {
+      "epoch": 2.995353247984827,
+      "grad_norm": 2.125,
+      "learning_rate": 9.033321845932233e-06,
+      "loss": 0.4777,
+      "step": 3950
+    },
+    {
+      "epoch": 3.002275960170697,
+      "grad_norm": 2.0,
+      "learning_rate": 8.90638697762015e-06,
+      "loss": 0.475,
+      "step": 3960
+    },
+    {
+      "epoch": 3.0098624940730203,
+      "grad_norm": 2.03125,
+      "learning_rate": 8.780156689281185e-06,
+      "loss": 0.4568,
+      "step": 3970
+    },
+    {
+      "epoch": 3.017449027975344,
+      "grad_norm": 2.1875,
+      "learning_rate": 8.654636507289848e-06,
+      "loss": 0.4599,
+      "step": 3980
+    },
+    {
+      "epoch": 3.025035561877667,
+      "grad_norm": 2.21875,
+      "learning_rate": 8.529831926932163e-06,
+      "loss": 0.4807,
+      "step": 3990
+    },
+    {
+      "epoch": 3.0326220957799905,
+      "grad_norm": 2.171875,
+      "learning_rate": 8.405748412165015e-06,
+      "loss": 0.4625,
+      "step": 4000
+    },
+    {
+      "epoch": 3.0326220957799905,
+      "eval_loss": 0.47956109046936035,
+      "eval_runtime": 9.1701,
+      "eval_samples_per_second": 121.046,
+      "eval_steps_per_second": 15.158,
+      "step": 4000
+    },
+    {
+      "epoch": 3.040208629682314,
+      "grad_norm": 2.1875,
+      "learning_rate": 8.282391395376993e-06,
+      "loss": 0.4836,
+      "step": 4010
+    },
+    {
+      "epoch": 3.047795163584637,
+      "grad_norm": 2.203125,
+      "learning_rate": 8.1597662771505e-06,
+      "loss": 0.4608,
+      "step": 4020
+    },
+    {
+      "epoch": 3.0553816974869608,
+      "grad_norm": 1.9453125,
+      "learning_rate": 8.03787842602538e-06,
+      "loss": 0.4588,
+      "step": 4030
+    },
+    {
+      "epoch": 3.062968231389284,
+      "grad_norm": 2.265625,
+      "learning_rate": 7.916733178263822e-06,
+      "loss": 0.4852,
+      "step": 4040
+    },
+    {
+      "epoch": 3.0705547652916074,
+      "grad_norm": 2.125,
+      "learning_rate": 7.796335837616783e-06,
+      "loss": 0.4564,
+      "step": 4050
+    },
+    {
+      "epoch": 3.078141299193931,
+      "grad_norm": 2.375,
+      "learning_rate": 7.676691675091774e-06,
+      "loss": 0.4792,
+      "step": 4060
+    },
+    {
+      "epoch": 3.085727833096254,
+      "grad_norm": 2.109375,
+      "learning_rate": 7.557805928722092e-06,
+      "loss": 0.4707,
+      "step": 4070
+    },
+    {
+      "epoch": 3.0933143669985776,
+      "grad_norm": 2.078125,
+      "learning_rate": 7.439683803337482e-06,
+      "loss": 0.4518,
+      "step": 4080
+    },
+    {
+      "epoch": 3.1009009009009008,
+      "grad_norm": 2.078125,
+      "learning_rate": 7.3223304703363135e-06,
+      "loss": 0.4711,
+      "step": 4090
+    },
+    {
+      "epoch": 3.1084874348032243,
+      "grad_norm": 2.140625,
+      "learning_rate": 7.205751067459129e-06,
+      "loss": 0.4719,
+      "step": 4100
+    },
+    {
+      "epoch": 3.116073968705548,
+      "grad_norm": 2.296875,
+      "learning_rate": 7.0899506985637356e-06,
+      "loss": 0.477,
+      "step": 4110
+    },
+    {
+      "epoch": 3.123660502607871,
+      "grad_norm": 2.1875,
+      "learning_rate": 6.9749344334017695e-06,
+      "loss": 0.4608,
+      "step": 4120
+    },
+    {
+      "epoch": 3.1312470365101945,
+      "grad_norm": 2.09375,
+      "learning_rate": 6.860707307396708e-06,
+      "loss": 0.4653,
+      "step": 4130
+    },
+    {
+      "epoch": 3.1388335704125176,
+      "grad_norm": 2.265625,
+      "learning_rate": 6.747274321423452e-06,
+      "loss": 0.4609,
+      "step": 4140
+    },
+    {
+      "epoch": 3.146420104314841,
+      "grad_norm": 2.0625,
+      "learning_rate": 6.634640441589371e-06,
+      "loss": 0.4459,
+      "step": 4150
+    },
+    {
+      "epoch": 3.1540066382171643,
+      "grad_norm": 2.15625,
+      "learning_rate": 6.522810599016896e-06,
+      "loss": 0.4437,
+      "step": 4160
+    },
+    {
+      "epoch": 3.161593172119488,
+      "grad_norm": 2.296875,
+      "learning_rate": 6.4117896896276035e-06,
+      "loss": 0.4579,
+      "step": 4170
+    },
+    {
+      "epoch": 3.1691797060218114,
+      "grad_norm": 2.046875,
+      "learning_rate": 6.301582573927922e-06,
+      "loss": 0.475,
+      "step": 4180
+    },
+    {
+      "epoch": 3.1767662399241345,
+      "grad_norm": 2.21875,
+      "learning_rate": 6.1921940767962845e-06,
+      "loss": 0.4634,
+      "step": 4190
+    },
+    {
+      "epoch": 3.184352773826458,
+      "grad_norm": 2.0625,
+      "learning_rate": 6.08362898727195e-06,
+      "loss": 0.4633,
+      "step": 4200
+    },
+    {
+      "epoch": 3.191939307728781,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.975892058345287e-06,
+      "loss": 0.4737,
+      "step": 4210
+    },
+    {
+      "epoch": 3.1995258416311048,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.868988006749732e-06,
+      "loss": 0.4664,
+      "step": 4220
+    },
+    {
+      "epoch": 3.2071123755334283,
+      "grad_norm": 2.109375,
+      "learning_rate": 5.7629215127552455e-06,
+      "loss": 0.4622,
+      "step": 4230
+    },
+    {
+      "epoch": 3.2146989094357514,
+      "grad_norm": 2.046875,
+      "learning_rate": 5.657697219963454e-06,
+      "loss": 0.4819,
+      "step": 4240
+    },
+    {
+      "epoch": 3.222285443338075,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.5533197351043315e-06,
+      "loss": 0.4655,
+      "step": 4250
+    },
+    {
+      "epoch": 3.229871977240398,
+      "grad_norm": 2.015625,
+      "learning_rate": 5.449793627834496e-06,
+      "loss": 0.4568,
+      "step": 4260
+    },
+    {
+      "epoch": 3.2374585111427217,
+      "grad_norm": 2.140625,
+      "learning_rate": 5.3471234305371905e-06,
+      "loss": 0.467,
+      "step": 4270
+    },
+    {
+      "epoch": 3.245045045045045,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.245313638123833e-06,
+      "loss": 0.4608,
+      "step": 4280
+    },
+    {
+      "epoch": 3.2526315789473683,
+      "grad_norm": 2.15625,
+      "learning_rate": 5.144368707837216e-06,
+      "loss": 0.4596,
+      "step": 4290
+    },
+    {
+      "epoch": 3.260218112849692,
+      "grad_norm": 2.234375,
+      "learning_rate": 5.044293059056401e-06,
+      "loss": 0.483,
+      "step": 4300
+    },
+    {
+      "epoch": 3.267804646752015,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.945091073103197e-06,
+      "loss": 0.4564,
+      "step": 4310
+    },
+    {
+      "epoch": 3.2753911806543385,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.8467670930503975e-06,
+      "loss": 0.466,
+      "step": 4320
+    },
+    {
+      "epoch": 3.282977714556662,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.749325423531575e-06,
+      "loss": 0.4776,
+      "step": 4330
+    },
+    {
+      "epoch": 3.290564248458985,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.652770330552697e-06,
+      "loss": 0.47,
+      "step": 4340
+    },
+    {
+      "epoch": 3.2981507823613088,
+      "grad_norm": 2.171875,
+      "learning_rate": 4.557106041305287e-06,
+      "loss": 0.4547,
+      "step": 4350
+    },
+    {
+      "epoch": 3.305737316263632,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.462336743981416e-06,
+      "loss": 0.4742,
+      "step": 4360
+    },
+    {
+      "epoch": 3.3133238501659554,
+      "grad_norm": 2.25,
+      "learning_rate": 4.368466587590314e-06,
+      "loss": 0.4789,
+      "step": 4370
+    },
+    {
+      "epoch": 3.320910384068279,
+      "grad_norm": 2.03125,
+      "learning_rate": 4.275499681776734e-06,
+      "loss": 0.4747,
+      "step": 4380
+    },
+    {
+      "epoch": 3.328496917970602,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.1834400966410195e-06,
+      "loss": 0.481,
+      "step": 4390
+    },
+    {
+      "epoch": 3.3360834518729257,
+      "grad_norm": 2.09375,
+      "learning_rate": 4.09229186256094e-06,
+      "loss": 0.4673,
+      "step": 4400
+    },
+    {
+      "epoch": 3.3436699857752488,
+      "grad_norm": 2.625,
+      "learning_rate": 4.0020589700152045e-06,
+      "loss": 0.4583,
+      "step": 4410
+    },
+    {
+      "epoch": 3.3512565196775723,
+      "grad_norm": 2.125,
+      "learning_rate": 3.912745369408805e-06,
+      "loss": 0.4554,
+      "step": 4420
+    },
+    {
+      "epoch": 3.358843053579896,
+      "grad_norm": 1.984375,
+      "learning_rate": 3.824354970900015e-06,
+      "loss": 0.4734,
+      "step": 4430
+    },
+    {
+      "epoch": 3.366429587482219,
+      "grad_norm": 2.078125,
+      "learning_rate": 3.736891644229254e-06,
+      "loss": 0.4521,
+      "step": 4440
+    },
+    {
+      "epoch": 3.3740161213845425,
+      "grad_norm": 2.125,
+      "learning_rate": 3.6503592185496176e-06,
+      "loss": 0.4604,
+      "step": 4450
+    },
+    {
+      "epoch": 3.3816026552868657,
+      "grad_norm": 2.09375,
+      "learning_rate": 3.5647614822592878e-06,
+      "loss": 0.4626,
+      "step": 4460
+    },
+    {
+      "epoch": 3.389189189189189,
+      "grad_norm": 2.078125,
+      "learning_rate": 3.4801021828356374e-06,
+      "loss": 0.4652,
+      "step": 4470
+    },
+    {
+      "epoch": 3.3967757230915128,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.396385026671195e-06,
+      "loss": 0.4709,
+      "step": 4480
+    },
+    {
+      "epoch": 3.404362256993836,
+      "grad_norm": 2.03125,
+      "learning_rate": 3.3136136789113454e-06,
+      "loss": 0.4796,
+      "step": 4490
+    },
+    {
+      "epoch": 3.4119487908961594,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.231791763293901e-06,
+      "loss": 0.4644,
+      "step": 4500
+    },
+    {
+      "epoch": 3.4119487908961594,
+      "eval_loss": 0.4789316952228546,
+      "eval_runtime": 9.1801,
+      "eval_samples_per_second": 120.914,
+      "eval_steps_per_second": 15.141,
+      "step": 4500
+    },
+    {
+      "epoch": 3.4195353247984825,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.1509228619904297e-06,
+      "loss": 0.4787,
+      "step": 4510
+    },
+    {
+      "epoch": 3.427121858700806,
+      "grad_norm": 2.03125,
+      "learning_rate": 3.0710105154494573e-06,
+      "loss": 0.4501,
+      "step": 4520
+    },
+    {
+      "epoch": 3.434708392603129,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.992058222241423e-06,
+      "loss": 0.4632,
+      "step": 4530
+    },
+    {
+      "epoch": 3.4422949265054528,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.9140694389055562e-06,
+      "loss": 0.46,
+      "step": 4540
+    },
+    {
+      "epoch": 3.4498814604077763,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.8370475797985158e-06,
+      "loss": 0.4616,
+      "step": 4550
+    },
+    {
+      "epoch": 3.4574679943100994,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.7609960169449308e-06,
+      "loss": 0.4772,
+      "step": 4560
+    },
+    {
+      "epoch": 3.465054528212423,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.6859180798897543e-06,
+      "loss": 0.4608,
+      "step": 4570
+    },
+    {
+      "epoch": 3.4726410621147465,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.6118170555525177e-06,
+      "loss": 0.4634,
+      "step": 4580
+    },
+    {
+      "epoch": 3.4802275960170697,
+      "grad_norm": 2.125,
+      "learning_rate": 2.5386961880834057e-06,
+      "loss": 0.4667,
+      "step": 4590
+    },
+    {
+      "epoch": 3.487814129919393,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.466558678721248e-06,
+      "loss": 0.4686,
+      "step": 4600
+    },
+    {
+      "epoch": 3.4954006638217163,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.39540768565335e-06,
+      "loss": 0.4621,
+      "step": 4610
+    },
+    {
+      "epoch": 3.50298719772404,
+      "grad_norm": 2.125,
+      "learning_rate": 2.325246323877242e-06,
+      "loss": 0.4803,
+      "step": 4620
+    },
+    {
+      "epoch": 3.510573731626363,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.2560776650642907e-06,
+      "loss": 0.4703,
+      "step": 4630
+    },
+    {
+      "epoch": 3.5181602655286865,
+      "grad_norm": 2.140625,
+      "learning_rate": 2.187904737425245e-06,
+      "loss": 0.468,
+      "step": 4640
+    },
+    {
+      "epoch": 3.52574679943101,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.120730525577619e-06,
+      "loss": 0.4779,
+      "step": 4650
+    },
+    {
+      "epoch": 3.533333333333333,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.0545579704150775e-06,
+      "loss": 0.4869,
+      "step": 4660
+    },
+    {
+      "epoch": 3.5409198672356568,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.9893899689786335e-06,
+      "loss": 0.467,
+      "step": 4670
+    },
+    {
+      "epoch": 3.5485064011379803,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.925229374329854e-06,
+      "loss": 0.4692,
+      "step": 4680
+    },
+    {
+      "epoch": 3.5560929350403034,
+      "grad_norm": 2.125,
+      "learning_rate": 1.8620789954259283e-06,
+      "loss": 0.4634,
+      "step": 4690
+    },
+    {
+      "epoch": 3.563679468942627,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.7999415969967126e-06,
+      "loss": 0.4698,
+      "step": 4700
+    },
+    {
+      "epoch": 3.57126600284495,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.7388198994236576e-06,
+      "loss": 0.4591,
+      "step": 4710
+    },
+    {
+      "epoch": 3.5788525367472737,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.6787165786207455e-06,
+      "loss": 0.4726,
+      "step": 4720
+    },
+    {
+      "epoch": 3.5864390706495968,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6196342659173164e-06,
+      "loss": 0.4791,
+      "step": 4730
+    },
+    {
+      "epoch": 3.5940256045519203,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.5615755479428785e-06,
+      "loss": 0.4741,
+      "step": 4740
+    },
+    {
+      "epoch": 3.601612138454244,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.5045429665138527e-06,
+      "loss": 0.4566,
+      "step": 4750
+    },
+    {
+      "epoch": 3.609198672356567,
+      "grad_norm": 2.125,
+      "learning_rate": 1.448539018522313e-06,
+      "loss": 0.4703,
+      "step": 4760
+    },
+    {
+      "epoch": 3.6167852062588906,
+      "grad_norm": 2.375,
+      "learning_rate": 1.3935661558266476e-06,
+      "loss": 0.4731,
+      "step": 4770
+    },
+    {
+      "epoch": 3.624371740161214,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.339626785144235e-06,
+      "loss": 0.4647,
+      "step": 4780
+    },
+    {
+      "epoch": 3.631958274063537,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.2867232679460718e-06,
+      "loss": 0.4615,
+      "step": 4790
+    },
+    {
+      "epoch": 3.6395448079658603,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.2348579203533828e-06,
+      "loss": 0.4793,
+      "step": 4800
+    },
+    {
+      "epoch": 3.647131341868184,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.184033013036226e-06,
+      "loss": 0.462,
+      "step": 4810
+    },
+    {
+      "epoch": 3.6547178757705074,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.1342507711140815e-06,
+      "loss": 0.4645,
+      "step": 4820
+    },
+    {
+      "epoch": 3.6623044096728306,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.0855133740584305e-06,
+      "loss": 0.4829,
+      "step": 4830
+    },
+    {
+      "epoch": 3.669890943575154,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.0378229555973507e-06,
+      "loss": 0.4745,
+      "step": 4840
+    },
+    {
+      "epoch": 3.6774774774774777,
+      "grad_norm": 2.109375,
+      "learning_rate": 9.91181603622085e-07,
+      "loss": 0.4582,
+      "step": 4850
+    },
+    {
+      "epoch": 3.6850640113798008,
+      "grad_norm": 2.125,
+      "learning_rate": 9.455913600956517e-07,
+      "loss": 0.4748,
+      "step": 4860
+    },
+    {
+      "epoch": 3.6926505452821243,
+      "grad_norm": 2.296875,
+      "learning_rate": 9.01054220963421e-07,
+      "loss": 0.47,
+      "step": 4870
+    },
+    {
+      "epoch": 3.7002370791844474,
+      "grad_norm": 2.28125,
+      "learning_rate": 8.57572136065768e-07,
+      "loss": 0.4639,
+      "step": 4880
+    },
+    {
+      "epoch": 3.707823613086771,
+      "grad_norm": 2.21875,
+      "learning_rate": 8.151470090526691e-07,
+      "loss": 0.4628,
+      "step": 4890
+    },
+    {
+      "epoch": 3.715410146989094,
+      "grad_norm": 1.9921875,
+      "learning_rate": 7.737806973003969e-07,
+      "loss": 0.4655,
+      "step": 4900
+    },
+    {
+      "epoch": 3.7229966808914177,
+      "grad_norm": 2.171875,
+      "learning_rate": 7.334750118301814e-07,
+      "loss": 0.4495,
+      "step": 4910
+    },
+    {
+      "epoch": 3.730583214793741,
+      "grad_norm": 2.234375,
+      "learning_rate": 6.942317172289186e-07,
+      "loss": 0.4735,
+      "step": 4920
+    },
+    {
+      "epoch": 3.7381697486960643,
+      "grad_norm": 2.21875,
+      "learning_rate": 6.560525315719435e-07,
+      "loss": 0.4895,
+      "step": 4930
+    },
+    {
+      "epoch": 3.745756282598388,
+      "grad_norm": 2.203125,
+      "learning_rate": 6.189391263477812e-07,
+      "loss": 0.4722,
+      "step": 4940
+    },
+    {
+      "epoch": 3.7533428165007114,
+      "grad_norm": 2.171875,
+      "learning_rate": 5.82893126384998e-07,
+      "loss": 0.4525,
+      "step": 4950
+    },
+    {
+      "epoch": 3.7609293504030346,
+      "grad_norm": 2.296875,
+      "learning_rate": 5.479161097810354e-07,
+      "loss": 0.4751,
+      "step": 4960
+    },
+    {
+      "epoch": 3.768515884305358,
+      "grad_norm": 2.140625,
+      "learning_rate": 5.140096078331546e-07,
+      "loss": 0.4792,
+      "step": 4970
+    },
+    {
+      "epoch": 3.776102418207681,
+      "grad_norm": 2.15625,
+      "learning_rate": 4.811751049713681e-07,
+      "loss": 0.4706,
+      "step": 4980
+    },
+    {
+      "epoch": 3.7836889521100048,
+      "grad_norm": 2.015625,
+      "learning_rate": 4.494140386934631e-07,
+      "loss": 0.4626,
+      "step": 4990
+    },
+    {
+      "epoch": 3.791275486012328,
+      "grad_norm": 2.0625,
+      "learning_rate": 4.1872779950207497e-07,
+      "loss": 0.4536,
+      "step": 5000
+    },
+    {
+      "epoch": 3.791275486012328,
+      "eval_loss": 0.47889795899391174,
+      "eval_runtime": 9.1858,
+      "eval_samples_per_second": 120.838,
+      "eval_steps_per_second": 15.132,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5276,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.6700647783936e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-5000/training_args.bin b/checkpoint-5000/training_args.bin
new file mode 100644
index 0000000..17f56c1
--- /dev/null
+++ b/checkpoint-5000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a647ef5efc6ef28f13ecc4d3c99da28526fcc6c25fb933bc470139f20fd302
+size 5969
diff --git a/checkpoint-5276/added_tokens.json b/checkpoint-5276/added_tokens.json
new file mode 100644
index 0000000..c045a99
--- /dev/null
+++ b/checkpoint-5276/added_tokens.json
@@ -0,0 +1,18 @@
+{
+  "[A1]": 64012,
+  "[A2]": 64013,
+  "[B1]": 64014,
+  "[B2]": 64015,
+  "[EXERCISE_VARIATIONS]": 64011,
+  "[LESSON]": 64009,
+  "[PRACTICE_TEXT]": 64010,
+  "antonym_focus": 64007,
+  "definition_only": 64008,
+  "fill_in_the_blanks": 64003,
+  "free_writing": 64000,
+  "general_vocab": 64004,
+  "matching_pairs": 64001,
+  "morphology_focus": 64005,
+  "reorder_tokens": 64002,
+  "synonym_focus": 64006
+}
diff --git a/checkpoint-5276/config.json b/checkpoint-5276/config.json
new file mode 100644
index 0000000..152460c
--- /dev/null
+++ b/checkpoint-5276/config.json
@@ -0,0 +1,36 @@
+{
+  "activation_function": "gelu",
+  "apply_query_key_layer_scaling": true,
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 3,
+  "initializer_range": 0.023,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 2048,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": 3072,
+  "n_layer": 12,
+  "n_positions": 4096,
+  "normalize_attention_scores": true,
+  "pad_token_id": 0,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "tokenizer_class": "GPTSw3Tokenizer",
+  "transformers_version": "4.57.1",
+  "use_cache": false,
+  "vocab_size": 64016
+}
diff --git a/checkpoint-5276/generation_config.json b/checkpoint-5276/generation_config.json
new file mode 100644
index 0000000..aca1c6f
--- /dev/null
+++ b/checkpoint-5276/generation_config.json
@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "transformers_version": "4.57.1",
+  "use_cache": false
+}
diff --git a/checkpoint-5276/model.safetensors b/checkpoint-5276/model.safetensors
new file mode 100644
index 0000000..1644cf1
--- /dev/null
+++ b/checkpoint-5276/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c6cf256fc4400f09066715e1b9be14c69ff8a0bc4f8f23bd3c998fdc6395a7c
+size 274779928
diff --git a/checkpoint-5276/optimizer.pt b/checkpoint-5276/optimizer.pt
new file mode 100644
index 0000000..1a2d1ab
--- /dev/null
+++ b/checkpoint-5276/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1011c6e5aa36d52d8e337faeefe71ba7399ab8c52536e16d508cb396e0b458f
+size 549590475
diff --git a/checkpoint-5276/rng_state.pth b/checkpoint-5276/rng_state.pth
new file mode 100644
index 0000000..759e31e
--- /dev/null
+++ b/checkpoint-5276/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11feebee6c482d74473263fd2bc20e2034e2ba91a39748e8c922699d428633bc
+size 14645
diff --git a/checkpoint-5276/scheduler.pt b/checkpoint-5276/scheduler.pt
new file mode 100644
index 0000000..17d6817
--- /dev/null
+++ b/checkpoint-5276/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4172d8c96ddc50581f82e3368358fca4280c85996ea501cbed64bfa7d37652b9
+size 1465
diff --git a/checkpoint-5276/special_tokens_map.json b/checkpoint-5276/special_tokens_map.json
new file mode 100644
index 0000000..fac732c
--- /dev/null
+++ b/checkpoint-5276/special_tokens_map.json
@@ -0,0 +1,127 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "--",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "free_writing",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "matching_pairs",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "reorder_tokens",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "fill_in_the_blanks",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "general_vocab",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "morphology_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "synonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "antonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "definition_only",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[LESSON]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[PRACTICE_TEXT]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[EXERCISE_VARIATIONS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[A1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[A2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[B1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[B2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<s>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}
diff --git a/checkpoint-5276/spiece.model b/checkpoint-5276/spiece.model
new file mode 100644
index 0000000..491f2fd
--- /dev/null
+++ b/checkpoint-5276/spiece.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4bda2cd84ff0ac659cda40e746c55f47ee3e57cf18471670ad26998c28be52d
+size 1071955
diff --git a/checkpoint-5276/tokenizer_config.json b/checkpoint-5276/tokenizer_config.json
new file mode 100644
index 0000000..67102bb
--- /dev/null
+++ b/checkpoint-5276/tokenizer_config.json
@@ -0,0 +1,203 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "603": {
+      "content": "--",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "free_writing",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "matching_pairs",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64002": {
+      "content": "reorder_tokens",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64003": {
+      "content": "fill_in_the_blanks",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64004": {
+      "content": "general_vocab",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64005": {
+      "content": "morphology_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64006": {
+      "content": "synonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64007": {
+      "content": "antonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64008": {
+      "content": "definition_only",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64009": {
+      "content": "[LESSON]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64010": {
+      "content": "[PRACTICE_TEXT]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64011": {
+      "content": "[EXERCISE_VARIATIONS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64012": {
+      "content": "[A1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64013": {
+      "content": "[A2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64014": {
+      "content": "[B1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64015": {
+      "content": "[B2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "--",
+    "free_writing",
+    "matching_pairs",
+    "reorder_tokens",
+    "fill_in_the_blanks",
+    "general_vocab",
+    "morphology_focus",
+    "synonym_focus",
+    "antonym_focus",
+    "definition_only",
+    "[LESSON]",
+    "[PRACTICE_TEXT]",
+    "[EXERCISE_VARIATIONS]",
+    "[A1]",
+    "[A2]",
+    "[B1]",
+    "[B2]"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "keep_accents": false,
+  "model_max_length": 4096,
+  "pad_token": "<pad>",
+  "remove_space": false,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "GPTSw3Tokenizer",
+  "unk_token": "<unk>"
+}
diff --git a/checkpoint-5276/trainer_state.json b/checkpoint-5276/trainer_state.json
new file mode 100644
index 0000000..a07474c
--- /dev/null
+++ b/checkpoint-5276/trainer_state.json
@@ -0,0 +1,3803 @@
+{
+  "best_global_step": 5000,
+  "best_metric": 0.47889795899391174,
+  "best_model_checkpoint": "/home/unsloth/szakdolgozat/stage2_exercise_scenario_generator_model/gpt-sw3-finetuned/checkpoint-5000",
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 5276,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.007586533902323376,
+      "grad_norm": 18.25,
+      "learning_rate": 8.522727272727273e-07,
+      "loss": 2.8483,
+      "step": 10
+    },
+    {
+      "epoch": 0.015173067804646752,
+      "grad_norm": 20.5,
+      "learning_rate": 1.7992424242424244e-06,
+      "loss": 2.8471,
+      "step": 20
+    },
+    {
+      "epoch": 0.02275960170697013,
+      "grad_norm": 12.1875,
+      "learning_rate": 2.7462121212121215e-06,
+      "loss": 2.7292,
+      "step": 30
+    },
+    {
+      "epoch": 0.030346135609293504,
+      "grad_norm": 11.1875,
+      "learning_rate": 3.6931818181818186e-06,
+      "loss": 2.794,
+      "step": 40
+    },
+    {
+      "epoch": 0.03793266951161688,
+      "grad_norm": 14.8125,
+      "learning_rate": 4.640151515151515e-06,
+      "loss": 2.6682,
+      "step": 50
+    },
+    {
+      "epoch": 0.04551920341394026,
+      "grad_norm": 11.125,
+      "learning_rate": 5.587121212121213e-06,
+      "loss": 2.5886,
+      "step": 60
+    },
+    {
+      "epoch": 0.05310573731626363,
+      "grad_norm": 7.375,
+      "learning_rate": 6.534090909090909e-06,
+      "loss": 2.4229,
+      "step": 70
+    },
+    {
+      "epoch": 0.06069227121858701,
+      "grad_norm": 6.21875,
+      "learning_rate": 7.481060606060607e-06,
+      "loss": 2.3076,
+      "step": 80
+    },
+    {
+      "epoch": 0.06827880512091039,
+      "grad_norm": 6.96875,
+      "learning_rate": 8.428030303030303e-06,
+      "loss": 2.2223,
+      "step": 90
+    },
+    {
+      "epoch": 0.07586533902323377,
+      "grad_norm": 5.25,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 2.112,
+      "step": 100
+    },
+    {
+      "epoch": 0.08345187292555714,
+      "grad_norm": 6.0,
+      "learning_rate": 1.0321969696969698e-05,
+      "loss": 2.0566,
+      "step": 110
+    },
+    {
+      "epoch": 0.09103840682788052,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.1268939393939394e-05,
+      "loss": 1.8943,
+      "step": 120
+    },
+    {
+      "epoch": 0.09862494073020389,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.2215909090909092e-05,
+      "loss": 1.8111,
+      "step": 130
+    },
+    {
+      "epoch": 0.10621147463252727,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.3162878787878787e-05,
+      "loss": 1.753,
+      "step": 140
+    },
+    {
+      "epoch": 0.11379800853485064,
+      "grad_norm": 4.125,
+      "learning_rate": 1.4109848484848487e-05,
+      "loss": 1.7014,
+      "step": 150
+    },
+    {
+      "epoch": 0.12138454243717402,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.5056818181818182e-05,
+      "loss": 1.6254,
+      "step": 160
+    },
+    {
+      "epoch": 0.1289710763394974,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.600378787878788e-05,
+      "loss": 1.5866,
+      "step": 170
+    },
+    {
+      "epoch": 0.13655761024182078,
+      "grad_norm": 5.875,
+      "learning_rate": 1.6950757575757574e-05,
+      "loss": 1.4858,
+      "step": 180
+    },
+    {
+      "epoch": 0.14414414414414414,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.7897727272727276e-05,
+      "loss": 1.4044,
+      "step": 190
+    },
+    {
+      "epoch": 0.15173067804646753,
+      "grad_norm": 5.3125,
+      "learning_rate": 1.884469696969697e-05,
+      "loss": 1.3114,
+      "step": 200
+    },
+    {
+      "epoch": 0.1593172119487909,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.9791666666666665e-05,
+      "loss": 1.2421,
+      "step": 210
+    },
+    {
+      "epoch": 0.16690374585111428,
+      "grad_norm": 3.65625,
+      "learning_rate": 2.0738636363636367e-05,
+      "loss": 1.1665,
+      "step": 220
+    },
+    {
+      "epoch": 0.17449027975343764,
+      "grad_norm": 3.390625,
+      "learning_rate": 2.1685606060606062e-05,
+      "loss": 1.1339,
+      "step": 230
+    },
+    {
+      "epoch": 0.18207681365576103,
+      "grad_norm": 3.0625,
+      "learning_rate": 2.2632575757575757e-05,
+      "loss": 1.1219,
+      "step": 240
+    },
+    {
+      "epoch": 0.1896633475580844,
+      "grad_norm": 3.1875,
+      "learning_rate": 2.3579545454545455e-05,
+      "loss": 1.0914,
+      "step": 250
+    },
+    {
+      "epoch": 0.19724988146040778,
+      "grad_norm": 3.171875,
+      "learning_rate": 2.4526515151515154e-05,
+      "loss": 1.0727,
+      "step": 260
+    },
+    {
+      "epoch": 0.20483641536273114,
+      "grad_norm": 3.625,
+      "learning_rate": 2.5473484848484852e-05,
+      "loss": 1.0931,
+      "step": 270
+    },
+    {
+      "epoch": 0.21242294926505453,
+      "grad_norm": 3.375,
+      "learning_rate": 2.6420454545454547e-05,
+      "loss": 1.041,
+      "step": 280
+    },
+    {
+      "epoch": 0.2200094831673779,
+      "grad_norm": 3.1875,
+      "learning_rate": 2.7367424242424245e-05,
+      "loss": 1.0206,
+      "step": 290
+    },
+    {
+      "epoch": 0.22759601706970128,
+      "grad_norm": 3.140625,
+      "learning_rate": 2.831439393939394e-05,
+      "loss": 1.0103,
+      "step": 300
+    },
+    {
+      "epoch": 0.23518255097202465,
+      "grad_norm": 13.5625,
+      "learning_rate": 2.9261363636363635e-05,
+      "loss": 0.9865,
+      "step": 310
+    },
+    {
+      "epoch": 0.24276908487434803,
+      "grad_norm": 3.40625,
+      "learning_rate": 3.0208333333333334e-05,
+      "loss": 0.9431,
+      "step": 320
+    },
+    {
+      "epoch": 0.2503556187766714,
+      "grad_norm": 3.46875,
+      "learning_rate": 3.115530303030303e-05,
+      "loss": 0.9698,
+      "step": 330
+    },
+    {
+      "epoch": 0.2579421526789948,
+      "grad_norm": 3.0,
+      "learning_rate": 3.210227272727273e-05,
+      "loss": 0.9284,
+      "step": 340
+    },
+    {
+      "epoch": 0.2655286865813182,
+      "grad_norm": 3.0625,
+      "learning_rate": 3.3049242424242425e-05,
+      "loss": 0.8814,
+      "step": 350
+    },
+    {
+      "epoch": 0.27311522048364156,
+      "grad_norm": 3.609375,
+      "learning_rate": 3.399621212121212e-05,
+      "loss": 0.8822,
+      "step": 360
+    },
+    {
+      "epoch": 0.2807017543859649,
+      "grad_norm": 3.796875,
+      "learning_rate": 3.494318181818182e-05,
+      "loss": 0.8931,
+      "step": 370
+    },
+    {
+      "epoch": 0.2882882882882883,
+      "grad_norm": 3.203125,
+      "learning_rate": 3.589015151515152e-05,
+      "loss": 0.8948,
+      "step": 380
+    },
+    {
+      "epoch": 0.2958748221906117,
+      "grad_norm": 3.1875,
+      "learning_rate": 3.683712121212121e-05,
+      "loss": 0.8459,
+      "step": 390
+    },
+    {
+      "epoch": 0.30346135609293506,
+      "grad_norm": 3.03125,
+      "learning_rate": 3.778409090909091e-05,
+      "loss": 0.8503,
+      "step": 400
+    },
+    {
+      "epoch": 0.3110478899952584,
+      "grad_norm": 3.078125,
+      "learning_rate": 3.873106060606061e-05,
+      "loss": 0.8365,
+      "step": 410
+    },
+    {
+      "epoch": 0.3186344238975818,
+      "grad_norm": 3.171875,
+      "learning_rate": 3.96780303030303e-05,
+      "loss": 0.8607,
+      "step": 420
+    },
+    {
+      "epoch": 0.3262209577999052,
+      "grad_norm": 3.5625,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 0.8253,
+      "step": 430
+    },
+    {
+      "epoch": 0.33380749170222856,
+      "grad_norm": 3.46875,
+      "learning_rate": 4.15719696969697e-05,
+      "loss": 0.8143,
+      "step": 440
+    },
+    {
+      "epoch": 0.3413940256045519,
+      "grad_norm": 3.859375,
+      "learning_rate": 4.2518939393939395e-05,
+      "loss": 0.7962,
+      "step": 450
+    },
+    {
+      "epoch": 0.3489805595068753,
+      "grad_norm": 3.203125,
+      "learning_rate": 4.346590909090909e-05,
+      "loss": 0.8083,
+      "step": 460
+    },
+    {
+      "epoch": 0.3565670934091987,
+      "grad_norm": 3.125,
+      "learning_rate": 4.441287878787879e-05,
+      "loss": 0.7884,
+      "step": 470
+    },
+    {
+      "epoch": 0.36415362731152207,
+      "grad_norm": 3.1875,
+      "learning_rate": 4.5359848484848486e-05,
+      "loss": 0.7643,
+      "step": 480
+    },
+    {
+      "epoch": 0.3717401612138454,
+      "grad_norm": 2.828125,
+      "learning_rate": 4.630681818181818e-05,
+      "loss": 0.7865,
+      "step": 490
+    },
+    {
+      "epoch": 0.3793266951161688,
+      "grad_norm": 3.03125,
+      "learning_rate": 4.725378787878788e-05,
+      "loss": 0.7825,
+      "step": 500
+    },
+    {
+      "epoch": 0.3793266951161688,
+      "eval_loss": 0.72025066614151,
+      "eval_runtime": 9.0957,
+      "eval_samples_per_second": 122.036,
+      "eval_steps_per_second": 15.282,
+      "step": 500
+    },
+    {
+      "epoch": 0.3869132290184922,
+      "grad_norm": 2.703125,
+      "learning_rate": 4.820075757575758e-05,
+      "loss": 0.7693,
+      "step": 510
+    },
+    {
+      "epoch": 0.39449976292081557,
+      "grad_norm": 3.9375,
+      "learning_rate": 4.914772727272727e-05,
+      "loss": 0.7902,
+      "step": 520
+    },
+    {
+      "epoch": 0.40208629682313896,
+      "grad_norm": 2.90625,
+      "learning_rate": 4.9999994527467846e-05,
+      "loss": 0.7622,
+      "step": 530
+    },
+    {
+      "epoch": 0.4096728307254623,
+      "grad_norm": 2.96875,
+      "learning_rate": 4.999933782650834e-05,
+      "loss": 0.747,
+      "step": 540
+    },
+    {
+      "epoch": 0.4172593646277857,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.999758665206119e-05,
+      "loss": 0.7543,
+      "step": 550
+    },
+    {
+      "epoch": 0.42484589853010907,
+      "grad_norm": 2.96875,
+      "learning_rate": 4.999474108079299e-05,
+      "loss": 0.7418,
+      "step": 560
+    },
+    {
+      "epoch": 0.43243243243243246,
+      "grad_norm": 2.90625,
+      "learning_rate": 4.9990801237283106e-05,
+      "loss": 0.7405,
+      "step": 570
+    },
+    {
+      "epoch": 0.4400189663347558,
+      "grad_norm": 3.09375,
+      "learning_rate": 4.998576729401832e-05,
+      "loss": 0.717,
+      "step": 580
+    },
+    {
+      "epoch": 0.4476055002370792,
+      "grad_norm": 2.765625,
+      "learning_rate": 4.997963947138513e-05,
+      "loss": 0.7181,
+      "step": 590
+    },
+    {
+      "epoch": 0.45519203413940257,
+      "grad_norm": 3.09375,
+      "learning_rate": 4.9972418037660236e-05,
+      "loss": 0.7132,
+      "step": 600
+    },
+    {
+      "epoch": 0.46277856804172596,
+      "grad_norm": 2.84375,
+      "learning_rate": 4.9964103308998723e-05,
+      "loss": 0.7236,
+      "step": 610
+    },
+    {
+      "epoch": 0.4703651019440493,
+      "grad_norm": 3.125,
+      "learning_rate": 4.995469564942023e-05,
+      "loss": 0.6841,
+      "step": 620
+    },
+    {
+      "epoch": 0.4779516358463727,
+      "grad_norm": 2.84375,
+      "learning_rate": 4.994419547079303e-05,
+      "loss": 0.682,
+      "step": 630
+    },
+    {
+      "epoch": 0.48553816974869607,
+      "grad_norm": 2.5625,
+      "learning_rate": 4.9932603232815976e-05,
+      "loss": 0.6747,
+      "step": 640
+    },
+    {
+      "epoch": 0.49312470365101946,
+      "grad_norm": 2.90625,
+      "learning_rate": 4.991991944299841e-05,
+      "loss": 0.6993,
+      "step": 650
+    },
+    {
+      "epoch": 0.5007112375533428,
+      "grad_norm": 3.125,
+      "learning_rate": 4.990614465663789e-05,
+      "loss": 0.6905,
+      "step": 660
+    },
+    {
+      "epoch": 0.5082977714556662,
+      "grad_norm": 2.640625,
+      "learning_rate": 4.989127947679595e-05,
+      "loss": 0.6803,
+      "step": 670
+    },
+    {
+      "epoch": 0.5158843053579896,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.987532455427161e-05,
+      "loss": 0.6721,
+      "step": 680
+    },
+    {
+      "epoch": 0.5234708392603129,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.985828058757298e-05,
+      "loss": 0.67,
+      "step": 690
+    },
+    {
+      "epoch": 0.5310573731626363,
+      "grad_norm": 2.5,
+      "learning_rate": 4.98401483228866e-05,
+      "loss": 0.6955,
+      "step": 700
+    },
+    {
+      "epoch": 0.5386439070649597,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.982092855404482e-05,
+      "loss": 0.678,
+      "step": 710
+    },
+    {
+      "epoch": 0.5462304409672831,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.9800622122491026e-05,
+      "loss": 0.6769,
+      "step": 720
+    },
+    {
+      "epoch": 0.5538169748696065,
+      "grad_norm": 2.875,
+      "learning_rate": 4.9779229917242795e-05,
+      "loss": 0.6803,
+      "step": 730
+    },
+    {
+      "epoch": 0.5614035087719298,
+      "grad_norm": 2.9375,
+      "learning_rate": 4.9756752874853006e-05,
+      "loss": 0.6512,
+      "step": 740
+    },
+    {
+      "epoch": 0.5689900426742532,
+      "grad_norm": 3.3125,
+      "learning_rate": 4.973319197936879e-05,
+      "loss": 0.6642,
+      "step": 750
+    },
+    {
+      "epoch": 0.5765765765765766,
+      "grad_norm": 2.640625,
+      "learning_rate": 4.97085482622885e-05,
+      "loss": 0.6701,
+      "step": 760
+    },
+    {
+      "epoch": 0.5841631104788999,
+      "grad_norm": 2.84375,
+      "learning_rate": 4.9682822802516506e-05,
+      "loss": 0.6523,
+      "step": 770
+    },
+    {
+      "epoch": 0.5917496443812233,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.965601672631597e-05,
+      "loss": 0.6535,
+      "step": 780
+    },
+    {
+      "epoch": 0.5993361782835467,
+      "grad_norm": 2.515625,
+      "learning_rate": 4.9628131207259596e-05,
+      "loss": 0.6452,
+      "step": 790
+    },
+    {
+      "epoch": 0.6069227121858701,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.959916746617814e-05,
+      "loss": 0.6686,
+      "step": 800
+    },
+    {
+      "epoch": 0.6145092460881935,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.9569126771107086e-05,
+      "loss": 0.6433,
+      "step": 810
+    },
+    {
+      "epoch": 0.6220957799905168,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.953801043723102e-05,
+      "loss": 0.6608,
+      "step": 820
+    },
+    {
+      "epoch": 0.6296823138928402,
+      "grad_norm": 2.53125,
+      "learning_rate": 4.950581982682614e-05,
+      "loss": 0.6492,
+      "step": 830
+    },
+    {
+      "epoch": 0.6372688477951636,
+      "grad_norm": 2.5625,
+      "learning_rate": 4.947255634920054e-05,
+      "loss": 0.6523,
+      "step": 840
+    },
+    {
+      "epoch": 0.6448553816974869,
+      "grad_norm": 2.671875,
+      "learning_rate": 4.9438221460632584e-05,
+      "loss": 0.6343,
+      "step": 850
+    },
+    {
+      "epoch": 0.6524419155998104,
+      "grad_norm": 2.671875,
+      "learning_rate": 4.940281666430708e-05,
+      "loss": 0.6565,
+      "step": 860
+    },
+    {
+      "epoch": 0.6600284495021337,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.9366343510249543e-05,
+      "loss": 0.6736,
+      "step": 870
+    },
+    {
+      "epoch": 0.6676149834044571,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.9328803595258254e-05,
+      "loss": 0.6229,
+      "step": 880
+    },
+    {
+      "epoch": 0.6752015173067805,
+      "grad_norm": 2.484375,
+      "learning_rate": 4.929019856283442e-05,
+      "loss": 0.6358,
+      "step": 890
+    },
+    {
+      "epoch": 0.6827880512091038,
+      "grad_norm": 2.421875,
+      "learning_rate": 4.925053010311018e-05,
+      "loss": 0.6187,
+      "step": 900
+    },
+    {
+      "epoch": 0.6903745851114272,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.920979995277464e-05,
+      "loss": 0.6408,
+      "step": 910
+    },
+    {
+      "epoch": 0.6979611190137506,
+      "grad_norm": 2.59375,
+      "learning_rate": 4.916800989499781e-05,
+      "loss": 0.6346,
+      "step": 920
+    },
+    {
+      "epoch": 0.705547652916074,
+      "grad_norm": 2.625,
+      "learning_rate": 4.9125161759352564e-05,
+      "loss": 0.6052,
+      "step": 930
+    },
+    {
+      "epoch": 0.7131341868183974,
+      "grad_norm": 2.5,
+      "learning_rate": 4.9081257421734536e-05,
+      "loss": 0.6474,
+      "step": 940
+    },
+    {
+      "epoch": 0.7207207207207207,
+      "grad_norm": 2.421875,
+      "learning_rate": 4.903629880427997e-05,
+      "loss": 0.6145,
+      "step": 950
+    },
+    {
+      "epoch": 0.7283072546230441,
+      "grad_norm": 2.4375,
+      "learning_rate": 4.899028787528158e-05,
+      "loss": 0.6219,
+      "step": 960
+    },
+    {
+      "epoch": 0.7358937885253675,
+      "grad_norm": 2.4375,
+      "learning_rate": 4.894322664910242e-05,
+      "loss": 0.6285,
+      "step": 970
+    },
+    {
+      "epoch": 0.7434803224276908,
+      "grad_norm": 2.203125,
+      "learning_rate": 4.8895117186087635e-05,
+      "loss": 0.6245,
+      "step": 980
+    },
+    {
+      "epoch": 0.7510668563300142,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.884596159247428e-05,
+      "loss": 0.619,
+      "step": 990
+    },
+    {
+      "epoch": 0.7586533902323376,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.87957620202991e-05,
+      "loss": 0.6051,
+      "step": 1000
+    },
+    {
+      "epoch": 0.7586533902323376,
+      "eval_loss": 0.571657657623291,
+      "eval_runtime": 9.0866,
+      "eval_samples_per_second": 122.157,
+      "eval_steps_per_second": 15.297,
+      "step": 1000
+    },
+    {
+      "epoch": 0.766239924134661,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.874452066730436e-05,
+      "loss": 0.6233,
+      "step": 1010
+    },
+    {
+      "epoch": 0.7738264580369844,
+      "grad_norm": 2.515625,
+      "learning_rate": 4.869223977684156e-05,
+      "loss": 0.6276,
+      "step": 1020
+    },
+    {
+      "epoch": 0.7814129919393077,
+      "grad_norm": 2.375,
+      "learning_rate": 4.863892163777327e-05,
+      "loss": 0.595,
+      "step": 1030
+    },
+    {
+      "epoch": 0.7889995258416311,
+      "grad_norm": 3.203125,
+      "learning_rate": 4.8584568584372904e-05,
+      "loss": 0.6067,
+      "step": 1040
+    },
+    {
+      "epoch": 0.7965860597439545,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.852918299622252e-05,
+      "loss": 0.5986,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8041725936462779,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.847276729810867e-05,
+      "loss": 0.5928,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8117591275486012,
+      "grad_norm": 2.375,
+      "learning_rate": 4.8415323959916195e-05,
+      "loss": 0.5857,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8193456614509246,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.835685549652015e-05,
+      "loss": 0.6214,
+      "step": 1080
+    },
+    {
+      "epoch": 0.826932195353248,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.829736446767563e-05,
+      "loss": 0.6083,
+      "step": 1090
+    },
+    {
+      "epoch": 0.8345187292555714,
+      "grad_norm": 2.390625,
+      "learning_rate": 4.8236853477905805e-05,
+      "loss": 0.5877,
+      "step": 1100
+    },
+    {
+      "epoch": 0.8421052631578947,
+      "grad_norm": 2.53125,
+      "learning_rate": 4.817532517638778e-05,
+      "loss": 0.6098,
+      "step": 1110
+    },
+    {
+      "epoch": 0.8496917970602181,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.811278225683671e-05,
+      "loss": 0.5788,
+      "step": 1120
+    },
+    {
+      "epoch": 0.8572783309625415,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.80492274573878e-05,
+      "loss": 0.6283,
+      "step": 1130
+    },
+    {
+      "epoch": 0.8648648648648649,
+      "grad_norm": 2.515625,
+      "learning_rate": 4.798466356047647e-05,
+      "loss": 0.6017,
+      "step": 1140
+    },
+    {
+      "epoch": 0.8724513987671882,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.791909339271653e-05,
+      "loss": 0.5914,
+      "step": 1150
+    },
+    {
+      "epoch": 0.8800379326695116,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.785251982477642e-05,
+      "loss": 0.6226,
+      "step": 1160
+    },
+    {
+      "epoch": 0.887624466571835,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.778494577125355e-05,
+      "loss": 0.5898,
+      "step": 1170
+    },
+    {
+      "epoch": 0.8952110004741584,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.7716374190546667e-05,
+      "loss": 0.6042,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9027975343764817,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.764680808472638e-05,
+      "loss": 0.5892,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9103840682788051,
+      "grad_norm": 2.375,
+      "learning_rate": 4.75762504994037e-05,
+      "loss": 0.594,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9179706021811285,
+      "grad_norm": 2.59375,
+      "learning_rate": 4.7504704523596686e-05,
+      "loss": 0.5787,
+      "step": 1210
+    },
+    {
+      "epoch": 0.9255571360834519,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.7432173289595275e-05,
+      "loss": 0.5833,
+      "step": 1220
+    },
+    {
+      "epoch": 0.9331436699857752,
+      "grad_norm": 2.375,
+      "learning_rate": 4.735865997282405e-05,
+      "loss": 0.596,
+      "step": 1230
+    },
+    {
+      "epoch": 0.9407302038880986,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.728416779170334e-05,
+      "loss": 0.5865,
+      "step": 1240
+    },
+    {
+      "epoch": 0.948316737790422,
+      "grad_norm": 2.5,
+      "learning_rate": 4.720870000750818e-05,
+      "loss": 0.6107,
+      "step": 1250
+    },
+    {
+      "epoch": 0.9559032716927454,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.713225992422567e-05,
+      "loss": 0.5709,
+      "step": 1260
+    },
+    {
+      "epoch": 0.9634898055950688,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.705485088841023e-05,
+      "loss": 0.5834,
+      "step": 1270
+    },
+    {
+      "epoch": 0.9710763394973921,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.6976476289037106e-05,
+      "loss": 0.5948,
+      "step": 1280
+    },
+    {
+      "epoch": 0.9786628733997155,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.689713955735403e-05,
+      "loss": 0.5863,
+      "step": 1290
+    },
+    {
+      "epoch": 0.9862494073020389,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.681684416673098e-05,
+      "loss": 0.5805,
+      "step": 1300
+    },
+    {
+      "epoch": 0.9938359412043622,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.673559363250811e-05,
+      "loss": 0.5796,
+      "step": 1310
+    },
+    {
+      "epoch": 1.0007586533902324,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.6653391511841876e-05,
+      "loss": 0.5691,
+      "step": 1320
+    },
+    {
+      "epoch": 1.0083451872925557,
+      "grad_norm": 2.375,
+      "learning_rate": 4.657024140354925e-05,
+      "loss": 0.5521,
+      "step": 1330
+    },
+    {
+      "epoch": 1.015931721194879,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.648614694795023e-05,
+      "loss": 0.5593,
+      "step": 1340
+    },
+    {
+      "epoch": 1.0235182550972024,
+      "grad_norm": 2.375,
+      "learning_rate": 4.64011118267084e-05,
+      "loss": 0.5742,
+      "step": 1350
+    },
+    {
+      "epoch": 1.0311047889995257,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.63151397626698e-05,
+      "loss": 0.5513,
+      "step": 1360
+    },
+    {
+      "epoch": 1.0386913229018493,
+      "grad_norm": 2.703125,
+      "learning_rate": 4.622823451969992e-05,
+      "loss": 0.5454,
+      "step": 1370
+    },
+    {
+      "epoch": 1.0462778568041726,
+      "grad_norm": 2.375,
+      "learning_rate": 4.614039990251892e-05,
+      "loss": 0.5678,
+      "step": 1380
+    },
+    {
+      "epoch": 1.053864390706496,
+      "grad_norm": 2.40625,
+      "learning_rate": 4.605163975653502e-05,
+      "loss": 0.5541,
+      "step": 1390
+    },
+    {
+      "epoch": 1.0614509246088193,
+      "grad_norm": 2.390625,
+      "learning_rate": 4.5961957967676216e-05,
+      "loss": 0.5626,
+      "step": 1400
+    },
+    {
+      "epoch": 1.0690374585111426,
+      "grad_norm": 2.4375,
+      "learning_rate": 4.587135846222013e-05,
+      "loss": 0.576,
+      "step": 1410
+    },
+    {
+      "epoch": 1.0766239924134662,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.577984520662207e-05,
+      "loss": 0.5547,
+      "step": 1420
+    },
+    {
+      "epoch": 1.0842105263157895,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.5687422207341434e-05,
+      "loss": 0.5738,
+      "step": 1430
+    },
+    {
+      "epoch": 1.0917970602181128,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.5594093510666317e-05,
+      "loss": 0.5329,
+      "step": 1440
+    },
+    {
+      "epoch": 1.0993835941204362,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.549986320253628e-05,
+      "loss": 0.5444,
+      "step": 1450
+    },
+    {
+      "epoch": 1.1069701280227595,
+      "grad_norm": 2.484375,
+      "learning_rate": 4.540473540836356e-05,
+      "loss": 0.544,
+      "step": 1460
+    },
+    {
+      "epoch": 1.114556661925083,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.530871429285242e-05,
+      "loss": 0.5646,
+      "step": 1470
+    },
+    {
+      "epoch": 1.1221431958274064,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.521180405981681e-05,
+      "loss": 0.5635,
+      "step": 1480
+    },
+    {
+      "epoch": 1.1297297297297297,
+      "grad_norm": 2.640625,
+      "learning_rate": 4.5114008951996325e-05,
+      "loss": 0.5342,
+      "step": 1490
+    },
+    {
+      "epoch": 1.137316263632053,
+      "grad_norm": 2.40625,
+      "learning_rate": 4.50153332508705e-05,
+      "loss": 0.547,
+      "step": 1500
+    },
+    {
+      "epoch": 1.137316263632053,
+      "eval_loss": 0.5373980402946472,
+      "eval_runtime": 9.1273,
+      "eval_samples_per_second": 121.614,
+      "eval_steps_per_second": 15.229,
+      "step": 1500
+    },
+    {
+      "epoch": 1.1449027975343764,
+      "grad_norm": 2.171875,
+      "learning_rate": 4.491578127647129e-05,
+      "loss": 0.5425,
+      "step": 1510
+    },
+    {
+      "epoch": 1.1524893314367,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.481535738719402e-05,
+      "loss": 0.5678,
+      "step": 1520
+    },
+    {
+      "epoch": 1.1600758653390233,
+      "grad_norm": 2.5,
+      "learning_rate": 4.47140659796065e-05,
+      "loss": 0.5589,
+      "step": 1530
+    },
+    {
+      "epoch": 1.1676623992413466,
+      "grad_norm": 2.25,
+      "learning_rate": 4.46119114882566e-05,
+      "loss": 0.5392,
+      "step": 1540
+    },
+    {
+      "epoch": 1.17524893314367,
+      "grad_norm": 2.546875,
+      "learning_rate": 4.450889838547806e-05,
+      "loss": 0.5318,
+      "step": 1550
+    },
+    {
+      "epoch": 1.1828354670459933,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.4405031181194744e-05,
+      "loss": 0.5512,
+      "step": 1560
+    },
+    {
+      "epoch": 1.1904220009483168,
+      "grad_norm": 2.234375,
+      "learning_rate": 4.4300314422723154e-05,
+      "loss": 0.5483,
+      "step": 1570
+    },
+    {
+      "epoch": 1.1980085348506402,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.419475269457335e-05,
+      "loss": 0.5712,
+      "step": 1580
+    },
+    {
+      "epoch": 1.2055950687529635,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.408835061824824e-05,
+      "loss": 0.528,
+      "step": 1590
+    },
+    {
+      "epoch": 1.2131816026552869,
+      "grad_norm": 2.453125,
+      "learning_rate": 4.398111285204128e-05,
+      "loss": 0.5626,
+      "step": 1600
+    },
+    {
+      "epoch": 1.2207681365576102,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.387304409083251e-05,
+      "loss": 0.5391,
+      "step": 1610
+    },
+    {
+      "epoch": 1.2283546704599337,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.376414906588299e-05,
+      "loss": 0.5384,
+      "step": 1620
+    },
+    {
+      "epoch": 1.235941204362257,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.365443254462772e-05,
+      "loss": 0.534,
+      "step": 1630
+    },
+    {
+      "epoch": 1.2435277382645804,
+      "grad_norm": 2.53125,
+      "learning_rate": 4.354389933046687e-05,
+      "loss": 0.5645,
+      "step": 1640
+    },
+    {
+      "epoch": 1.2511142721669037,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.343255426255549e-05,
+      "loss": 0.551,
+      "step": 1650
+    },
+    {
+      "epoch": 1.258700806069227,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.332040221559172e-05,
+      "loss": 0.537,
+      "step": 1660
+    },
+    {
+      "epoch": 1.2662873399715506,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.3207448099603275e-05,
+      "loss": 0.5427,
+      "step": 1670
+    },
+    {
+      "epoch": 1.2738738738738737,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.3093696859732546e-05,
+      "loss": 0.5327,
+      "step": 1680
+    },
+    {
+      "epoch": 1.2814604077761973,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.29791534760201e-05,
+      "loss": 0.5518,
+      "step": 1690
+    },
+    {
+      "epoch": 1.2890469416785206,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.2863822963186604e-05,
+      "loss": 0.5531,
+      "step": 1700
+    },
+    {
+      "epoch": 1.296633475580844,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.274771037041336e-05,
+      "loss": 0.5352,
+      "step": 1710
+    },
+    {
+      "epoch": 1.3042200094831673,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.263082078112119e-05,
+      "loss": 0.5569,
+      "step": 1720
+    },
+    {
+      "epoch": 1.3118065433854906,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.251315931274789e-05,
+      "loss": 0.5288,
+      "step": 1730
+    },
+    {
+      "epoch": 1.3193930772878142,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.23947311165242e-05,
+      "loss": 0.5396,
+      "step": 1740
+    },
+    {
+      "epoch": 1.3269796111901375,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.227554137724829e-05,
+      "loss": 0.5304,
+      "step": 1750
+    },
+    {
+      "epoch": 1.3345661450924609,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.2155595313058776e-05,
+      "loss": 0.5438,
+      "step": 1760
+    },
+    {
+      "epoch": 1.3421526789947842,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.203489817520622e-05,
+      "loss": 0.5312,
+      "step": 1770
+    },
+    {
+      "epoch": 1.3497392128971075,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.191345524782327e-05,
+      "loss": 0.5271,
+      "step": 1780
+    },
+    {
+      "epoch": 1.357325746799431,
+      "grad_norm": 2.265625,
+      "learning_rate": 4.1791271847693336e-05,
+      "loss": 0.5361,
+      "step": 1790
+    },
+    {
+      "epoch": 1.3649122807017544,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.166835332401779e-05,
+      "loss": 0.5469,
+      "step": 1800
+    },
+    {
+      "epoch": 1.3724988146040777,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.154470505818177e-05,
+      "loss": 0.5259,
+      "step": 1810
+    },
+    {
+      "epoch": 1.380085348506401,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.142033246351861e-05,
+      "loss": 0.5232,
+      "step": 1820
+    },
+    {
+      "epoch": 1.3876718824087244,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.129524098507283e-05,
+      "loss": 0.526,
+      "step": 1830
+    },
+    {
+      "epoch": 1.395258416311048,
+      "grad_norm": 2.46875,
+      "learning_rate": 4.116943609936176e-05,
+      "loss": 0.5437,
+      "step": 1840
+    },
+    {
+      "epoch": 1.4028449502133713,
+      "grad_norm": 2.25,
+      "learning_rate": 4.1042923314135764e-05,
+      "loss": 0.5103,
+      "step": 1850
+    },
+    {
+      "epoch": 1.4104314841156946,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.0915708168137104e-05,
+      "loss": 0.5345,
+      "step": 1860
+    },
+    {
+      "epoch": 1.418018018018018,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.07877962308575e-05,
+      "loss": 0.5547,
+      "step": 1870
+    },
+    {
+      "epoch": 1.4256045519203413,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.0659193102294224e-05,
+      "loss": 0.5374,
+      "step": 1880
+    },
+    {
+      "epoch": 1.4331910858226649,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.0529904412704997e-05,
+      "loss": 0.5271,
+      "step": 1890
+    },
+    {
+      "epoch": 1.4407776197249882,
+      "grad_norm": 2.375,
+      "learning_rate": 4.039993582236148e-05,
+      "loss": 0.5453,
+      "step": 1900
+    },
+    {
+      "epoch": 1.4483641536273115,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.0269293021301406e-05,
+      "loss": 0.5063,
+      "step": 1910
+    },
+    {
+      "epoch": 1.4559506875296349,
+      "grad_norm": 2.390625,
+      "learning_rate": 4.013798172907958e-05,
+      "loss": 0.5453,
+      "step": 1920
+    },
+    {
+      "epoch": 1.4635372214319582,
+      "grad_norm": 2.203125,
+      "learning_rate": 4.0006007694517376e-05,
+      "loss": 0.5403,
+      "step": 1930
+    },
+    {
+      "epoch": 1.4711237553342817,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.9873376695451114e-05,
+      "loss": 0.5216,
+      "step": 1940
+    },
+    {
+      "epoch": 1.478710289236605,
+      "grad_norm": 2.25,
+      "learning_rate": 3.9740094538479064e-05,
+      "loss": 0.5573,
+      "step": 1950
+    },
+    {
+      "epoch": 1.4862968231389284,
+      "grad_norm": 2.375,
+      "learning_rate": 3.960616705870729e-05,
+      "loss": 0.5403,
+      "step": 1960
+    },
+    {
+      "epoch": 1.4938833570412517,
+      "grad_norm": 2.625,
+      "learning_rate": 3.947160011949411e-05,
+      "loss": 0.5385,
+      "step": 1970
+    },
+    {
+      "epoch": 1.501469890943575,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.9336399612193465e-05,
+      "loss": 0.5354,
+      "step": 1980
+    },
+    {
+      "epoch": 1.5090564248458986,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.920057145589697e-05,
+      "loss": 0.523,
+      "step": 1990
+    },
+    {
+      "epoch": 1.5166429587482217,
+      "grad_norm": 2.46875,
+      "learning_rate": 3.9064121597174764e-05,
+      "loss": 0.5451,
+      "step": 2000
+    },
+    {
+      "epoch": 1.5166429587482217,
+      "eval_loss": 0.5127466917037964,
+      "eval_runtime": 9.1468,
+      "eval_samples_per_second": 121.354,
+      "eval_steps_per_second": 15.197,
+      "step": 2000
+    },
+    {
+      "epoch": 1.5242294926505453,
+      "grad_norm": 2.375,
+      "learning_rate": 3.8927056009815176e-05,
+      "loss": 0.5269,
+      "step": 2010
+    },
+    {
+      "epoch": 1.5318160265528686,
+      "grad_norm": 2.390625,
+      "learning_rate": 3.878938069456322e-05,
+      "loss": 0.5322,
+      "step": 2020
+    },
+    {
+      "epoch": 1.539402560455192,
+      "grad_norm": 2.328125,
+      "learning_rate": 3.865110167885784e-05,
+      "loss": 0.5322,
+      "step": 2030
+    },
+    {
+      "epoch": 1.5469890943575155,
+      "grad_norm": 2.28125,
+      "learning_rate": 3.8512225016568034e-05,
+      "loss": 0.5373,
+      "step": 2040
+    },
+    {
+      "epoch": 1.5545756282598386,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.837275678772786e-05,
+      "loss": 0.5194,
+      "step": 2050
+    },
+    {
+      "epoch": 1.5621621621621622,
+      "grad_norm": 2.15625,
+      "learning_rate": 3.8232703098270186e-05,
+      "loss": 0.5304,
+      "step": 2060
+    },
+    {
+      "epoch": 1.5697486960644855,
+      "grad_norm": 2.421875,
+      "learning_rate": 3.8092070079759426e-05,
+      "loss": 0.5245,
+      "step": 2070
+    },
+    {
+      "epoch": 1.5773352299668089,
+      "grad_norm": 2.359375,
+      "learning_rate": 3.795086388912305e-05,
+      "loss": 0.5351,
+      "step": 2080
+    },
+    {
+      "epoch": 1.5849217638691324,
+      "grad_norm": 2.125,
+      "learning_rate": 3.78090907083821e-05,
+      "loss": 0.5214,
+      "step": 2090
+    },
+    {
+      "epoch": 1.5925082977714555,
+      "grad_norm": 2.09375,
+      "learning_rate": 3.766675674438047e-05,
+      "loss": 0.5308,
+      "step": 2100
+    },
+    {
+      "epoch": 1.600094831673779,
+      "grad_norm": 2.25,
+      "learning_rate": 3.7523868228513207e-05,
+      "loss": 0.5231,
+      "step": 2110
+    },
+    {
+      "epoch": 1.6076813655761024,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.7380431416453706e-05,
+      "loss": 0.5366,
+      "step": 2120
+    },
+    {
+      "epoch": 1.6152678994784258,
+      "grad_norm": 2.359375,
+      "learning_rate": 3.72364525878798e-05,
+      "loss": 0.5273,
+      "step": 2130
+    },
+    {
+      "epoch": 1.6228544333807493,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.709193804619889e-05,
+      "loss": 0.5111,
+      "step": 2140
+    },
+    {
+      "epoch": 1.6304409672830724,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.694689411827192e-05,
+      "loss": 0.5231,
+      "step": 2150
+    },
+    {
+      "epoch": 1.638027501185396,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.680132715413644e-05,
+      "loss": 0.5219,
+      "step": 2160
+    },
+    {
+      "epoch": 1.6456140350877193,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.6655243526728555e-05,
+      "loss": 0.5249,
+      "step": 2170
+    },
+    {
+      "epoch": 1.6532005689900426,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.6508649631603967e-05,
+      "loss": 0.5177,
+      "step": 2180
+    },
+    {
+      "epoch": 1.6607871028923662,
+      "grad_norm": 2.21875,
+      "learning_rate": 3.6361551886657915e-05,
+      "loss": 0.5548,
+      "step": 2190
+    },
+    {
+      "epoch": 1.6683736367946893,
+      "grad_norm": 2.25,
+      "learning_rate": 3.621395673184426e-05,
+      "loss": 0.5304,
+      "step": 2200
+    },
+    {
+      "epoch": 1.6759601706970129,
+      "grad_norm": 2.015625,
+      "learning_rate": 3.6065870628893504e-05,
+      "loss": 0.5168,
+      "step": 2210
+    },
+    {
+      "epoch": 1.6835467045993362,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.591730006102992e-05,
+      "loss": 0.5051,
+      "step": 2220
+    },
+    {
+      "epoch": 1.6911332385016595,
+      "grad_norm": 2.0625,
+      "learning_rate": 3.576825153268768e-05,
+      "loss": 0.5134,
+      "step": 2230
+    },
+    {
+      "epoch": 1.698719772403983,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.561873156922614e-05,
+      "loss": 0.526,
+      "step": 2240
+    },
+    {
+      "epoch": 1.7063063063063062,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.546874671664411e-05,
+      "loss": 0.5313,
+      "step": 2250
+    },
+    {
+      "epoch": 1.7138928402086298,
+      "grad_norm": 2.375,
+      "learning_rate": 3.531830354129329e-05,
+      "loss": 0.5292,
+      "step": 2260
+    },
+    {
+      "epoch": 1.721479374110953,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.5167408629590835e-05,
+      "loss": 0.5319,
+      "step": 2270
+    },
+    {
+      "epoch": 1.7290659080132764,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.501606858773092e-05,
+      "loss": 0.5208,
+      "step": 2280
+    },
+    {
+      "epoch": 1.7366524419156,
+      "grad_norm": 2.53125,
+      "learning_rate": 3.486429004139556e-05,
+      "loss": 0.514,
+      "step": 2290
+    },
+    {
+      "epoch": 1.744238975817923,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.4712079635464587e-05,
+      "loss": 0.5272,
+      "step": 2300
+    },
+    {
+      "epoch": 1.7518255097202466,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.455944403372463e-05,
+      "loss": 0.5414,
+      "step": 2310
+    },
+    {
+      "epoch": 1.75941204362257,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.440638991857747e-05,
+      "loss": 0.5363,
+      "step": 2320
+    },
+    {
+      "epoch": 1.7669985775248933,
+      "grad_norm": 2.25,
+      "learning_rate": 3.425292399074744e-05,
+      "loss": 0.5209,
+      "step": 2330
+    },
+    {
+      "epoch": 1.7745851114272166,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.4099052968988076e-05,
+      "loss": 0.5093,
+      "step": 2340
+    },
+    {
+      "epoch": 1.78217164532954,
+      "grad_norm": 2.28125,
+      "learning_rate": 3.394478358978795e-05,
+      "loss": 0.5118,
+      "step": 2350
+    },
+    {
+      "epoch": 1.7897581792318635,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.379012260707577e-05,
+      "loss": 0.5321,
+      "step": 2360
+    },
+    {
+      "epoch": 1.7973447131341869,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.363507679192469e-05,
+      "loss": 0.5126,
+      "step": 2370
+    },
+    {
+      "epoch": 1.8049312470365102,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.347965293225587e-05,
+      "loss": 0.5208,
+      "step": 2380
+    },
+    {
+      "epoch": 1.8125177809388335,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.3323857832541295e-05,
+      "loss": 0.5232,
+      "step": 2390
+    },
+    {
+      "epoch": 1.8201043148411569,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.316769831350585e-05,
+      "loss": 0.5189,
+      "step": 2400
+    },
+    {
+      "epoch": 1.8276908487434804,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.3011181211828766e-05,
+      "loss": 0.5144,
+      "step": 2410
+    },
+    {
+      "epoch": 1.8352773826458035,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.285431337984427e-05,
+      "loss": 0.513,
+      "step": 2420
+    },
+    {
+      "epoch": 1.842863916548127,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.269710168524159e-05,
+      "loss": 0.5085,
+      "step": 2430
+    },
+    {
+      "epoch": 1.8504504504504504,
+      "grad_norm": 2.21875,
+      "learning_rate": 3.253955301076429e-05,
+      "loss": 0.4978,
+      "step": 2440
+    },
+    {
+      "epoch": 1.8580369843527738,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.2381674253908965e-05,
+      "loss": 0.5428,
+      "step": 2450
+    },
+    {
+      "epoch": 1.8656235182550973,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.222347232662323e-05,
+      "loss": 0.5292,
+      "step": 2460
+    },
+    {
+      "epoch": 1.8732100521574204,
+      "grad_norm": 2.15625,
+      "learning_rate": 3.206495415500314e-05,
+      "loss": 0.5366,
+      "step": 2470
+    },
+    {
+      "epoch": 1.880796586059744,
+      "grad_norm": 2.328125,
+      "learning_rate": 3.190612667898996e-05,
+      "loss": 0.5114,
+      "step": 2480
+    },
+    {
+      "epoch": 1.8883831199620673,
+      "grad_norm": 2.078125,
+      "learning_rate": 3.1746996852066324e-05,
+      "loss": 0.5093,
+      "step": 2490
+    },
+    {
+      "epoch": 1.8959696538643906,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.158757164095183e-05,
+      "loss": 0.5014,
+      "step": 2500
+    },
+    {
+      "epoch": 1.8959696538643906,
+      "eval_loss": 0.49517810344696045,
+      "eval_runtime": 9.1582,
+      "eval_samples_per_second": 121.203,
+      "eval_steps_per_second": 15.178,
+      "step": 2500
+    },
+    {
+      "epoch": 1.9035561877667142,
+      "grad_norm": 2.234375,
+      "learning_rate": 3.142785802529804e-05,
+      "loss": 0.5117,
+      "step": 2510
+    },
+    {
+      "epoch": 1.9111427216690373,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.126786299738285e-05,
+      "loss": 0.5346,
+      "step": 2520
+    },
+    {
+      "epoch": 1.9187292555713609,
+      "grad_norm": 2.21875,
+      "learning_rate": 3.1107593561804436e-05,
+      "loss": 0.5203,
+      "step": 2530
+    },
+    {
+      "epoch": 1.9263157894736842,
+      "grad_norm": 2.421875,
+      "learning_rate": 3.0947056735174596e-05,
+      "loss": 0.5311,
+      "step": 2540
+    },
+    {
+      "epoch": 1.9339023233760075,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.078625954581149e-05,
+      "loss": 0.5109,
+      "step": 2550
+    },
+    {
+      "epoch": 1.941488857278331,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.0625209033432015e-05,
+      "loss": 0.5256,
+      "step": 2560
+    },
+    {
+      "epoch": 1.9490753911806542,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.0463912248843563e-05,
+      "loss": 0.5216,
+      "step": 2570
+    },
+    {
+      "epoch": 1.9566619250829778,
+      "grad_norm": 2.34375,
+      "learning_rate": 3.0302376253635347e-05,
+      "loss": 0.5134,
+      "step": 2580
+    },
+    {
+      "epoch": 1.964248458985301,
+      "grad_norm": 2.234375,
+      "learning_rate": 3.014060811986924e-05,
+      "loss": 0.5234,
+      "step": 2590
+    },
+    {
+      "epoch": 1.9718349928876244,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.9978614929770176e-05,
+      "loss": 0.5114,
+      "step": 2600
+    },
+    {
+      "epoch": 1.979421526789948,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.981640377541608e-05,
+      "loss": 0.5085,
+      "step": 2610
+    },
+    {
+      "epoch": 1.987008060692271,
+      "grad_norm": 2.25,
+      "learning_rate": 2.965398175842735e-05,
+      "loss": 0.5002,
+      "step": 2620
+    },
+    {
+      "epoch": 1.9945945945945946,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.9491355989656e-05,
+      "loss": 0.5161,
+      "step": 2630
+    },
+    {
+      "epoch": 2.001517306780465,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.9328533588874295e-05,
+      "loss": 0.52,
+      "step": 2640
+    },
+    {
+      "epoch": 2.009103840682788,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.9165521684463087e-05,
+      "loss": 0.487,
+      "step": 2650
+    },
+    {
+      "epoch": 2.0166903745851115,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.9002327413099705e-05,
+      "loss": 0.4851,
+      "step": 2660
+    },
+    {
+      "epoch": 2.024276908487435,
+      "grad_norm": 2.421875,
+      "learning_rate": 2.8838957919445515e-05,
+      "loss": 0.498,
+      "step": 2670
+    },
+    {
+      "epoch": 2.031863442389758,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.867542035583315e-05,
+      "loss": 0.4904,
+      "step": 2680
+    },
+    {
+      "epoch": 2.0394499762920817,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.8511721881953335e-05,
+      "loss": 0.4733,
+      "step": 2690
+    },
+    {
+      "epoch": 2.047036510194405,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.8347869664541503e-05,
+      "loss": 0.4997,
+      "step": 2700
+    },
+    {
+      "epoch": 2.0546230440967284,
+      "grad_norm": 2.203125,
+      "learning_rate": 2.8183870877063963e-05,
+      "loss": 0.4996,
+      "step": 2710
+    },
+    {
+      "epoch": 2.0622095779990515,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.801973269940388e-05,
+      "loss": 0.5077,
+      "step": 2720
+    },
+    {
+      "epoch": 2.069796111901375,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.7855462317546944e-05,
+      "loss": 0.4922,
+      "step": 2730
+    },
+    {
+      "epoch": 2.0773826458036986,
+      "grad_norm": 2.25,
+      "learning_rate": 2.7691066923266746e-05,
+      "loss": 0.4942,
+      "step": 2740
+    },
+    {
+      "epoch": 2.0849691797060217,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.7526553713809938e-05,
+      "loss": 0.4947,
+      "step": 2750
+    },
+    {
+      "epoch": 2.0925557136083452,
+      "grad_norm": 2.203125,
+      "learning_rate": 2.736192989158115e-05,
+      "loss": 0.4903,
+      "step": 2760
+    },
+    {
+      "epoch": 2.1001422475106684,
+      "grad_norm": 2.125,
+      "learning_rate": 2.7197202663827625e-05,
+      "loss": 0.4803,
+      "step": 2770
+    },
+    {
+      "epoch": 2.107728781412992,
+      "grad_norm": 2.015625,
+      "learning_rate": 2.7032379242323724e-05,
+      "loss": 0.491,
+      "step": 2780
+    },
+    {
+      "epoch": 2.1153153153153155,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.6867466843055166e-05,
+      "loss": 0.4782,
+      "step": 2790
+    },
+    {
+      "epoch": 2.1229018492176386,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.6702472685903134e-05,
+      "loss": 0.4956,
+      "step": 2800
+    },
+    {
+      "epoch": 2.130488383119962,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.6537403994328198e-05,
+      "loss": 0.4951,
+      "step": 2810
+    },
+    {
+      "epoch": 2.1380749170222852,
+      "grad_norm": 2.015625,
+      "learning_rate": 2.6372267995054023e-05,
+      "loss": 0.4723,
+      "step": 2820
+    },
+    {
+      "epoch": 2.145661450924609,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.620707191775102e-05,
+      "loss": 0.4914,
+      "step": 2830
+    },
+    {
+      "epoch": 2.1532479848269324,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.604182299471983e-05,
+      "loss": 0.4747,
+      "step": 2840
+    },
+    {
+      "epoch": 2.1608345187292555,
+      "grad_norm": 2.34375,
+      "learning_rate": 2.5876528460574672e-05,
+      "loss": 0.4858,
+      "step": 2850
+    },
+    {
+      "epoch": 2.168421052631579,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.5711195551926646e-05,
+      "loss": 0.4798,
+      "step": 2860
+    },
+    {
+      "epoch": 2.176007586533902,
+      "grad_norm": 2.4375,
+      "learning_rate": 2.5545831507066875e-05,
+      "loss": 0.4857,
+      "step": 2870
+    },
+    {
+      "epoch": 2.1835941204362257,
+      "grad_norm": 2.0625,
+      "learning_rate": 2.538044356564963e-05,
+      "loss": 0.4845,
+      "step": 2880
+    },
+    {
+      "epoch": 2.1911806543385492,
+      "grad_norm": 2.25,
+      "learning_rate": 2.5215038968375376e-05,
+      "loss": 0.4827,
+      "step": 2890
+    },
+    {
+      "epoch": 2.1987671882408724,
+      "grad_norm": 1.96875,
+      "learning_rate": 2.5049624956673773e-05,
+      "loss": 0.4728,
+      "step": 2900
+    },
+    {
+      "epoch": 2.206353722143196,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.4884208772386644e-05,
+      "loss": 0.4836,
+      "step": 2910
+    },
+    {
+      "epoch": 2.213940256045519,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.4718797657450944e-05,
+      "loss": 0.4866,
+      "step": 2920
+    },
+    {
+      "epoch": 2.2215267899478426,
+      "grad_norm": 2.375,
+      "learning_rate": 2.455339885358166e-05,
+      "loss": 0.4871,
+      "step": 2930
+    },
+    {
+      "epoch": 2.229113323850166,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.4388019601954843e-05,
+      "loss": 0.4844,
+      "step": 2940
+    },
+    {
+      "epoch": 2.2366998577524893,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.4222667142890488e-05,
+      "loss": 0.4649,
+      "step": 2950
+    },
+    {
+      "epoch": 2.244286391654813,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.4057348715535662e-05,
+      "loss": 0.4851,
+      "step": 2960
+    },
+    {
+      "epoch": 2.251872925557136,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.389207155754749e-05,
+      "loss": 0.4977,
+      "step": 2970
+    },
+    {
+      "epoch": 2.2594594594594595,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.3726842904776338e-05,
+      "loss": 0.4798,
+      "step": 2980
+    },
+    {
+      "epoch": 2.267045993361783,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.356166999094899e-05,
+      "loss": 0.4967,
+      "step": 2990
+    },
+    {
+      "epoch": 2.274632527264106,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.3396560047351984e-05,
+      "loss": 0.4817,
+      "step": 3000
+    },
+    {
+      "epoch": 2.274632527264106,
+      "eval_loss": 0.4872964918613434,
+      "eval_runtime": 9.1611,
+      "eval_samples_per_second": 121.164,
+      "eval_steps_per_second": 15.173,
+      "step": 3000
+    },
+    {
+      "epoch": 2.2822190611664297,
+      "grad_norm": 2.34375,
+      "learning_rate": 2.323152030251502e-05,
+      "loss": 0.4819,
+      "step": 3010
+    },
+    {
+      "epoch": 2.289805595068753,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.3066557981894476e-05,
+      "loss": 0.488,
+      "step": 3020
+    },
+    {
+      "epoch": 2.2973921289710764,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.2901680307557104e-05,
+      "loss": 0.481,
+      "step": 3030
+    },
+    {
+      "epoch": 2.3049786628734,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.2736894497863828e-05,
+      "loss": 0.4896,
+      "step": 3040
+    },
+    {
+      "epoch": 2.312565196775723,
+      "grad_norm": 2.0,
+      "learning_rate": 2.2572207767153718e-05,
+      "loss": 0.4781,
+      "step": 3050
+    },
+    {
+      "epoch": 2.3201517306780466,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.2407627325428177e-05,
+      "loss": 0.4711,
+      "step": 3060
+    },
+    {
+      "epoch": 2.3277382645803697,
+      "grad_norm": 2.25,
+      "learning_rate": 2.2243160378035244e-05,
+      "loss": 0.489,
+      "step": 3070
+    },
+    {
+      "epoch": 2.3353247984826933,
+      "grad_norm": 2.46875,
+      "learning_rate": 2.2078814125354192e-05,
+      "loss": 0.4967,
+      "step": 3080
+    },
+    {
+      "epoch": 2.3429113323850164,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.1914595762480233e-05,
+      "loss": 0.4815,
+      "step": 3090
+    },
+    {
+      "epoch": 2.35049786628734,
+      "grad_norm": 1.9765625,
+      "learning_rate": 2.1750512478909586e-05,
+      "loss": 0.4705,
+      "step": 3100
+    },
+    {
+      "epoch": 2.3580844001896635,
+      "grad_norm": 2.0625,
+      "learning_rate": 2.158657145822465e-05,
+      "loss": 0.4862,
+      "step": 3110
+    },
+    {
+      "epoch": 2.3656709340919866,
+      "grad_norm": 2.03125,
+      "learning_rate": 2.142277987777957e-05,
+      "loss": 0.4628,
+      "step": 3120
+    },
+    {
+      "epoch": 2.37325746799431,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.125914490838596e-05,
+      "loss": 0.4713,
+      "step": 3130
+    },
+    {
+      "epoch": 2.3808440018966337,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.1095673713999018e-05,
+      "loss": 0.4837,
+      "step": 3140
+    },
+    {
+      "epoch": 2.388430535798957,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.093237345140381e-05,
+      "loss": 0.4907,
+      "step": 3150
+    },
+    {
+      "epoch": 2.3960170697012804,
+      "grad_norm": 2.25,
+      "learning_rate": 2.076925126990204e-05,
+      "loss": 0.4757,
+      "step": 3160
+    },
+    {
+      "epoch": 2.4036036036036035,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.060631431099896e-05,
+      "loss": 0.4785,
+      "step": 3170
+    },
+    {
+      "epoch": 2.411190137505927,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.044356970809079e-05,
+      "loss": 0.4726,
+      "step": 3180
+    },
+    {
+      "epoch": 2.41877667140825,
+      "grad_norm": 2.0,
+      "learning_rate": 2.028102458615237e-05,
+      "loss": 0.4744,
+      "step": 3190
+    },
+    {
+      "epoch": 2.4263632053105737,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.0118686061425257e-05,
+      "loss": 0.4813,
+      "step": 3200
+    },
+    {
+      "epoch": 2.4339497392128973,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.995656124110614e-05,
+      "loss": 0.4929,
+      "step": 3210
+    },
+    {
+      "epoch": 2.4415362731152204,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.979465722303573e-05,
+      "loss": 0.4874,
+      "step": 3220
+    },
+    {
+      "epoch": 2.449122807017544,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.9632981095387973e-05,
+      "loss": 0.4786,
+      "step": 3230
+    },
+    {
+      "epoch": 2.4567093409198675,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.947153993635978e-05,
+      "loss": 0.4864,
+      "step": 3240
+    },
+    {
+      "epoch": 2.4642958748221906,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.9310340813861067e-05,
+      "loss": 0.4957,
+      "step": 3250
+    },
+    {
+      "epoch": 2.471882408724514,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9149390785205416e-05,
+      "loss": 0.501,
+      "step": 3260
+    },
+    {
+      "epoch": 2.4794689426268373,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.8988696896801018e-05,
+      "loss": 0.4826,
+      "step": 3270
+    },
+    {
+      "epoch": 2.487055476529161,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.882826618384224e-05,
+      "loss": 0.4844,
+      "step": 3280
+    },
+    {
+      "epoch": 2.494642010431484,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.866810567000159e-05,
+      "loss": 0.4929,
+      "step": 3290
+    },
+    {
+      "epoch": 2.5022285443338075,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.8508222367122254e-05,
+      "loss": 0.469,
+      "step": 3300
+    },
+    {
+      "epoch": 2.509815078236131,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.8348623274911057e-05,
+      "loss": 0.4721,
+      "step": 3310
+    },
+    {
+      "epoch": 2.517401612138454,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.8189315380632073e-05,
+      "loss": 0.476,
+      "step": 3320
+    },
+    {
+      "epoch": 2.5249881460407777,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.8030305658800718e-05,
+      "loss": 0.4621,
+      "step": 3330
+    },
+    {
+      "epoch": 2.5325746799431013,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.787160107087835e-05,
+      "loss": 0.4824,
+      "step": 3340
+    },
+    {
+      "epoch": 2.5401612138454244,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.771320856496759e-05,
+      "loss": 0.4868,
+      "step": 3350
+    },
+    {
+      "epoch": 2.5477477477477475,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.7555135075508013e-05,
+      "loss": 0.4811,
+      "step": 3360
+    },
+    {
+      "epoch": 2.555334281650071,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.739738752297267e-05,
+      "loss": 0.4813,
+      "step": 3370
+    },
+    {
+      "epoch": 2.5629208155523946,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.7239972813565037e-05,
+      "loss": 0.4694,
+      "step": 3380
+    },
+    {
+      "epoch": 2.5705073494547177,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.708289783891671e-05,
+      "loss": 0.4821,
+      "step": 3390
+    },
+    {
+      "epoch": 2.5780938833570413,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.692616947578562e-05,
+      "loss": 0.4894,
+      "step": 3400
+    },
+    {
+      "epoch": 2.585680417259365,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6769794585755057e-05,
+      "loss": 0.4787,
+      "step": 3410
+    },
+    {
+      "epoch": 2.593266951161688,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.6613780014933193e-05,
+      "loss": 0.4758,
+      "step": 3420
+    },
+    {
+      "epoch": 2.6008534850640115,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6458132593653407e-05,
+      "loss": 0.4642,
+      "step": 3430
+    },
+    {
+      "epoch": 2.6084400189663346,
+      "grad_norm": 2.125,
+      "learning_rate": 1.6302859136175228e-05,
+      "loss": 0.4835,
+      "step": 3440
+    },
+    {
+      "epoch": 2.616026552868658,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.614796644038602e-05,
+      "loss": 0.4801,
+      "step": 3450
+    },
+    {
+      "epoch": 2.6236130867709813,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.5993461287503337e-05,
+      "loss": 0.4723,
+      "step": 3460
+    },
+    {
+      "epoch": 2.631199620673305,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.5839350441778104e-05,
+      "loss": 0.4958,
+      "step": 3470
+    },
+    {
+      "epoch": 2.6387861545756284,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.56856406501984e-05,
+      "loss": 0.4782,
+      "step": 3480
+    },
+    {
+      "epoch": 2.6463726884779515,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.5532338642194154e-05,
+      "loss": 0.4745,
+      "step": 3490
+    },
+    {
+      "epoch": 2.653959222380275,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.5379451129342456e-05,
+      "loss": 0.4852,
+      "step": 3500
+    },
+    {
+      "epoch": 2.653959222380275,
+      "eval_loss": 0.48158586025238037,
+      "eval_runtime": 9.1707,
+      "eval_samples_per_second": 121.038,
+      "eval_steps_per_second": 15.157,
+      "step": 3500
+    },
+    {
+      "epoch": 2.6615457562825986,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.5226984805073786e-05,
+      "loss": 0.4557,
+      "step": 3510
+    },
+    {
+      "epoch": 2.6691322901849217,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.507494634437891e-05,
+      "loss": 0.4689,
+      "step": 3520
+    },
+    {
+      "epoch": 2.6767188240872453,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.4923342403516721e-05,
+      "loss": 0.4804,
+      "step": 3530
+    },
+    {
+      "epoch": 2.6843053579895684,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.477217961972277e-05,
+      "loss": 0.4719,
+      "step": 3540
+    },
+    {
+      "epoch": 2.691891891891892,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.4621464610918717e-05,
+      "loss": 0.4818,
+      "step": 3550
+    },
+    {
+      "epoch": 2.699478425794215,
+      "grad_norm": 2.375,
+      "learning_rate": 1.4471203975422576e-05,
+      "loss": 0.4962,
+      "step": 3560
+    },
+    {
+      "epoch": 2.7070649596965386,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.432140429165987e-05,
+      "loss": 0.4808,
+      "step": 3570
+    },
+    {
+      "epoch": 2.714651493598862,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.417207211787559e-05,
+      "loss": 0.4773,
+      "step": 3580
+    },
+    {
+      "epoch": 2.7222380275011853,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.4023213991847118e-05,
+      "loss": 0.4613,
+      "step": 3590
+    },
+    {
+      "epoch": 2.729824561403509,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.3874836430597943e-05,
+      "loss": 0.4942,
+      "step": 3600
+    },
+    {
+      "epoch": 2.7374110953058324,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.3726945930112418e-05,
+      "loss": 0.4747,
+      "step": 3610
+    },
+    {
+      "epoch": 2.7449976292081555,
+      "grad_norm": 2.25,
+      "learning_rate": 1.3579548965051305e-05,
+      "loss": 0.4835,
+      "step": 3620
+    },
+    {
+      "epoch": 2.752584163110479,
+      "grad_norm": 2.125,
+      "learning_rate": 1.343265198846836e-05,
+      "loss": 0.4849,
+      "step": 3630
+    },
+    {
+      "epoch": 2.760170697012802,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.3286261431527747e-05,
+      "loss": 0.4773,
+      "step": 3640
+    },
+    {
+      "epoch": 2.7677572309151257,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.3140383703222583e-05,
+      "loss": 0.4891,
+      "step": 3650
+    },
+    {
+      "epoch": 2.775343764817449,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.2995025190094251e-05,
+      "loss": 0.4867,
+      "step": 3660
+    },
+    {
+      "epoch": 2.7829302987197724,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.2850192255952875e-05,
+      "loss": 0.4754,
+      "step": 3670
+    },
+    {
+      "epoch": 2.790516832622096,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.2705891241598655e-05,
+      "loss": 0.4634,
+      "step": 3680
+    },
+    {
+      "epoch": 2.798103366524419,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.2562128464544315e-05,
+      "loss": 0.4776,
+      "step": 3690
+    },
+    {
+      "epoch": 2.8056899004267426,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.2418910218738467e-05,
+      "loss": 0.4802,
+      "step": 3700
+    },
+    {
+      "epoch": 2.813276434329066,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.227624277429012e-05,
+      "loss": 0.4902,
+      "step": 3710
+    },
+    {
+      "epoch": 2.8208629682313893,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.2134132377194112e-05,
+      "loss": 0.4655,
+      "step": 3720
+    },
+    {
+      "epoch": 2.828449502133713,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.1992585249057727e-05,
+      "loss": 0.4918,
+      "step": 3730
+    },
+    {
+      "epoch": 2.836036036036036,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.1851607586828262e-05,
+      "loss": 0.4683,
+      "step": 3740
+    },
+    {
+      "epoch": 2.8436225699383595,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.1711205562521745e-05,
+      "loss": 0.4792,
+      "step": 3750
+    },
+    {
+      "epoch": 2.8512091038406826,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.1571385322952702e-05,
+      "loss": 0.4848,
+      "step": 3760
+    },
+    {
+      "epoch": 2.858795637743006,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.1432152989465094e-05,
+      "loss": 0.4882,
+      "step": 3770
+    },
+    {
+      "epoch": 2.8663821716453297,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.1293514657664264e-05,
+      "loss": 0.4813,
+      "step": 3780
+    },
+    {
+      "epoch": 2.873968705547653,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.1155476397150133e-05,
+      "loss": 0.4697,
+      "step": 3790
+    },
+    {
+      "epoch": 2.8815552394499764,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.1018044251251415e-05,
+      "loss": 0.4685,
+      "step": 3800
+    },
+    {
+      "epoch": 2.8891417733523,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.0881224236761087e-05,
+      "loss": 0.4607,
+      "step": 3810
+    },
+    {
+      "epoch": 2.896728307254623,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.074502234367293e-05,
+      "loss": 0.4671,
+      "step": 3820
+    },
+    {
+      "epoch": 2.904314841156946,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.0609444534919334e-05,
+      "loss": 0.478,
+      "step": 3830
+    },
+    {
+      "epoch": 2.9119013750592697,
+      "grad_norm": 2.375,
+      "learning_rate": 1.0474496746110197e-05,
+      "loss": 0.4618,
+      "step": 3840
+    },
+    {
+      "epoch": 2.9194879089615933,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.03401848852731e-05,
+      "loss": 0.48,
+      "step": 3850
+    },
+    {
+      "epoch": 2.9270744428639164,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.0206514832594603e-05,
+      "loss": 0.4669,
+      "step": 3860
+    },
+    {
+      "epoch": 2.93466097676624,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.0073492440162876e-05,
+      "loss": 0.4728,
+      "step": 3870
+    },
+    {
+      "epoch": 2.9422475106685635,
+      "grad_norm": 2.171875,
+      "learning_rate": 9.941123531711428e-06,
+      "loss": 0.4852,
+      "step": 3880
+    },
+    {
+      "epoch": 2.9498340445708866,
+      "grad_norm": 2.28125,
+      "learning_rate": 9.809413902364206e-06,
+      "loss": 0.4902,
+      "step": 3890
+    },
+    {
+      "epoch": 2.95742057847321,
+      "grad_norm": 2.328125,
+      "learning_rate": 9.678369318381816e-06,
+      "loss": 0.4816,
+      "step": 3900
+    },
+    {
+      "epoch": 2.9650071123755333,
+      "grad_norm": 2.109375,
+      "learning_rate": 9.547995516909156e-06,
+      "loss": 0.4821,
+      "step": 3910
+    },
+    {
+      "epoch": 2.972593646277857,
+      "grad_norm": 2.015625,
+      "learning_rate": 9.418298205724152e-06,
+      "loss": 0.4784,
+      "step": 3920
+    },
+    {
+      "epoch": 2.98018018018018,
+      "grad_norm": 2.265625,
+      "learning_rate": 9.289283062987953e-06,
+      "loss": 0.4788,
+      "step": 3930
+    },
+    {
+      "epoch": 2.9877667140825035,
+      "grad_norm": 2.015625,
+      "learning_rate": 9.160955736996301e-06,
+      "loss": 0.4642,
+      "step": 3940
+    },
+    {
+      "epoch": 2.995353247984827,
+      "grad_norm": 2.125,
+      "learning_rate": 9.033321845932233e-06,
+      "loss": 0.4777,
+      "step": 3950
+    },
+    {
+      "epoch": 3.002275960170697,
+      "grad_norm": 2.0,
+      "learning_rate": 8.90638697762015e-06,
+      "loss": 0.475,
+      "step": 3960
+    },
+    {
+      "epoch": 3.0098624940730203,
+      "grad_norm": 2.03125,
+      "learning_rate": 8.780156689281185e-06,
+      "loss": 0.4568,
+      "step": 3970
+    },
+    {
+      "epoch": 3.017449027975344,
+      "grad_norm": 2.1875,
+      "learning_rate": 8.654636507289848e-06,
+      "loss": 0.4599,
+      "step": 3980
+    },
+    {
+      "epoch": 3.025035561877667,
+      "grad_norm": 2.21875,
+      "learning_rate": 8.529831926932163e-06,
+      "loss": 0.4807,
+      "step": 3990
+    },
+    {
+      "epoch": 3.0326220957799905,
+      "grad_norm": 2.171875,
+      "learning_rate": 8.405748412165015e-06,
+      "loss": 0.4625,
+      "step": 4000
+    },
+    {
+      "epoch": 3.0326220957799905,
+      "eval_loss": 0.47956109046936035,
+      "eval_runtime": 9.1701,
+      "eval_samples_per_second": 121.046,
+      "eval_steps_per_second": 15.158,
+      "step": 4000
+    },
+    {
+      "epoch": 3.040208629682314,
+      "grad_norm": 2.1875,
+      "learning_rate": 8.282391395376993e-06,
+      "loss": 0.4836,
+      "step": 4010
+    },
+    {
+      "epoch": 3.047795163584637,
+      "grad_norm": 2.203125,
+      "learning_rate": 8.1597662771505e-06,
+      "loss": 0.4608,
+      "step": 4020
+    },
+    {
+      "epoch": 3.0553816974869608,
+      "grad_norm": 1.9453125,
+      "learning_rate": 8.03787842602538e-06,
+      "loss": 0.4588,
+      "step": 4030
+    },
+    {
+      "epoch": 3.062968231389284,
+      "grad_norm": 2.265625,
+      "learning_rate": 7.916733178263822e-06,
+      "loss": 0.4852,
+      "step": 4040
+    },
+    {
+      "epoch": 3.0705547652916074,
+      "grad_norm": 2.125,
+      "learning_rate": 7.796335837616783e-06,
+      "loss": 0.4564,
+      "step": 4050
+    },
+    {
+      "epoch": 3.078141299193931,
+      "grad_norm": 2.375,
+      "learning_rate": 7.676691675091774e-06,
+      "loss": 0.4792,
+      "step": 4060
+    },
+    {
+      "epoch": 3.085727833096254,
+      "grad_norm": 2.109375,
+      "learning_rate": 7.557805928722092e-06,
+      "loss": 0.4707,
+      "step": 4070
+    },
+    {
+      "epoch": 3.0933143669985776,
+      "grad_norm": 2.078125,
+      "learning_rate": 7.439683803337482e-06,
+      "loss": 0.4518,
+      "step": 4080
+    },
+    {
+      "epoch": 3.1009009009009008,
+      "grad_norm": 2.078125,
+      "learning_rate": 7.3223304703363135e-06,
+      "loss": 0.4711,
+      "step": 4090
+    },
+    {
+      "epoch": 3.1084874348032243,
+      "grad_norm": 2.140625,
+      "learning_rate": 7.205751067459129e-06,
+      "loss": 0.4719,
+      "step": 4100
+    },
+    {
+      "epoch": 3.116073968705548,
+      "grad_norm": 2.296875,
+      "learning_rate": 7.0899506985637356e-06,
+      "loss": 0.477,
+      "step": 4110
+    },
+    {
+      "epoch": 3.123660502607871,
+      "grad_norm": 2.1875,
+      "learning_rate": 6.9749344334017695e-06,
+      "loss": 0.4608,
+      "step": 4120
+    },
+    {
+      "epoch": 3.1312470365101945,
+      "grad_norm": 2.09375,
+      "learning_rate": 6.860707307396708e-06,
+      "loss": 0.4653,
+      "step": 4130
+    },
+    {
+      "epoch": 3.1388335704125176,
+      "grad_norm": 2.265625,
+      "learning_rate": 6.747274321423452e-06,
+      "loss": 0.4609,
+      "step": 4140
+    },
+    {
+      "epoch": 3.146420104314841,
+      "grad_norm": 2.0625,
+      "learning_rate": 6.634640441589371e-06,
+      "loss": 0.4459,
+      "step": 4150
+    },
+    {
+      "epoch": 3.1540066382171643,
+      "grad_norm": 2.15625,
+      "learning_rate": 6.522810599016896e-06,
+      "loss": 0.4437,
+      "step": 4160
+    },
+    {
+      "epoch": 3.161593172119488,
+      "grad_norm": 2.296875,
+      "learning_rate": 6.4117896896276035e-06,
+      "loss": 0.4579,
+      "step": 4170
+    },
+    {
+      "epoch": 3.1691797060218114,
+      "grad_norm": 2.046875,
+      "learning_rate": 6.301582573927922e-06,
+      "loss": 0.475,
+      "step": 4180
+    },
+    {
+      "epoch": 3.1767662399241345,
+      "grad_norm": 2.21875,
+      "learning_rate": 6.1921940767962845e-06,
+      "loss": 0.4634,
+      "step": 4190
+    },
+    {
+      "epoch": 3.184352773826458,
+      "grad_norm": 2.0625,
+      "learning_rate": 6.08362898727195e-06,
+      "loss": 0.4633,
+      "step": 4200
+    },
+    {
+      "epoch": 3.191939307728781,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.975892058345287e-06,
+      "loss": 0.4737,
+      "step": 4210
+    },
+    {
+      "epoch": 3.1995258416311048,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.868988006749732e-06,
+      "loss": 0.4664,
+      "step": 4220
+    },
+    {
+      "epoch": 3.2071123755334283,
+      "grad_norm": 2.109375,
+      "learning_rate": 5.7629215127552455e-06,
+      "loss": 0.4622,
+      "step": 4230
+    },
+    {
+      "epoch": 3.2146989094357514,
+      "grad_norm": 2.046875,
+      "learning_rate": 5.657697219963454e-06,
+      "loss": 0.4819,
+      "step": 4240
+    },
+    {
+      "epoch": 3.222285443338075,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.5533197351043315e-06,
+      "loss": 0.4655,
+      "step": 4250
+    },
+    {
+      "epoch": 3.229871977240398,
+      "grad_norm": 2.015625,
+      "learning_rate": 5.449793627834496e-06,
+      "loss": 0.4568,
+      "step": 4260
+    },
+    {
+      "epoch": 3.2374585111427217,
+      "grad_norm": 2.140625,
+      "learning_rate": 5.3471234305371905e-06,
+      "loss": 0.467,
+      "step": 4270
+    },
+    {
+      "epoch": 3.245045045045045,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.245313638123833e-06,
+      "loss": 0.4608,
+      "step": 4280
+    },
+    {
+      "epoch": 3.2526315789473683,
+      "grad_norm": 2.15625,
+      "learning_rate": 5.144368707837216e-06,
+      "loss": 0.4596,
+      "step": 4290
+    },
+    {
+      "epoch": 3.260218112849692,
+      "grad_norm": 2.234375,
+      "learning_rate": 5.044293059056401e-06,
+      "loss": 0.483,
+      "step": 4300
+    },
+    {
+      "epoch": 3.267804646752015,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.945091073103197e-06,
+      "loss": 0.4564,
+      "step": 4310
+    },
+    {
+      "epoch": 3.2753911806543385,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.8467670930503975e-06,
+      "loss": 0.466,
+      "step": 4320
+    },
+    {
+      "epoch": 3.282977714556662,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.749325423531575e-06,
+      "loss": 0.4776,
+      "step": 4330
+    },
+    {
+      "epoch": 3.290564248458985,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.652770330552697e-06,
+      "loss": 0.47,
+      "step": 4340
+    },
+    {
+      "epoch": 3.2981507823613088,
+      "grad_norm": 2.171875,
+      "learning_rate": 4.557106041305287e-06,
+      "loss": 0.4547,
+      "step": 4350
+    },
+    {
+      "epoch": 3.305737316263632,
+      "grad_norm": 2.328125,
+      "learning_rate": 4.462336743981416e-06,
+      "loss": 0.4742,
+      "step": 4360
+    },
+    {
+      "epoch": 3.3133238501659554,
+      "grad_norm": 2.25,
+      "learning_rate": 4.368466587590314e-06,
+      "loss": 0.4789,
+      "step": 4370
+    },
+    {
+      "epoch": 3.320910384068279,
+      "grad_norm": 2.03125,
+      "learning_rate": 4.275499681776734e-06,
+      "loss": 0.4747,
+      "step": 4380
+    },
+    {
+      "epoch": 3.328496917970602,
+      "grad_norm": 2.296875,
+      "learning_rate": 4.1834400966410195e-06,
+      "loss": 0.481,
+      "step": 4390
+    },
+    {
+      "epoch": 3.3360834518729257,
+      "grad_norm": 2.09375,
+      "learning_rate": 4.09229186256094e-06,
+      "loss": 0.4673,
+      "step": 4400
+    },
+    {
+      "epoch": 3.3436699857752488,
+      "grad_norm": 2.625,
+      "learning_rate": 4.0020589700152045e-06,
+      "loss": 0.4583,
+      "step": 4410
+    },
+    {
+      "epoch": 3.3512565196775723,
+      "grad_norm": 2.125,
+      "learning_rate": 3.912745369408805e-06,
+      "loss": 0.4554,
+      "step": 4420
+    },
+    {
+      "epoch": 3.358843053579896,
+      "grad_norm": 1.984375,
+      "learning_rate": 3.824354970900015e-06,
+      "loss": 0.4734,
+      "step": 4430
+    },
+    {
+      "epoch": 3.366429587482219,
+      "grad_norm": 2.078125,
+      "learning_rate": 3.736891644229254e-06,
+      "loss": 0.4521,
+      "step": 4440
+    },
+    {
+      "epoch": 3.3740161213845425,
+      "grad_norm": 2.125,
+      "learning_rate": 3.6503592185496176e-06,
+      "loss": 0.4604,
+      "step": 4450
+    },
+    {
+      "epoch": 3.3816026552868657,
+      "grad_norm": 2.09375,
+      "learning_rate": 3.5647614822592878e-06,
+      "loss": 0.4626,
+      "step": 4460
+    },
+    {
+      "epoch": 3.389189189189189,
+      "grad_norm": 2.078125,
+      "learning_rate": 3.4801021828356374e-06,
+      "loss": 0.4652,
+      "step": 4470
+    },
+    {
+      "epoch": 3.3967757230915128,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.396385026671195e-06,
+      "loss": 0.4709,
+      "step": 4480
+    },
+    {
+      "epoch": 3.404362256993836,
+      "grad_norm": 2.03125,
+      "learning_rate": 3.3136136789113454e-06,
+      "loss": 0.4796,
+      "step": 4490
+    },
+    {
+      "epoch": 3.4119487908961594,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.231791763293901e-06,
+      "loss": 0.4644,
+      "step": 4500
+    },
+    {
+      "epoch": 3.4119487908961594,
+      "eval_loss": 0.4789316952228546,
+      "eval_runtime": 9.1801,
+      "eval_samples_per_second": 120.914,
+      "eval_steps_per_second": 15.141,
+      "step": 4500
+    },
+    {
+      "epoch": 3.4195353247984825,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.1509228619904297e-06,
+      "loss": 0.4787,
+      "step": 4510
+    },
+    {
+      "epoch": 3.427121858700806,
+      "grad_norm": 2.03125,
+      "learning_rate": 3.0710105154494573e-06,
+      "loss": 0.4501,
+      "step": 4520
+    },
+    {
+      "epoch": 3.434708392603129,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.992058222241423e-06,
+      "loss": 0.4632,
+      "step": 4530
+    },
+    {
+      "epoch": 3.4422949265054528,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.9140694389055562e-06,
+      "loss": 0.46,
+      "step": 4540
+    },
+    {
+      "epoch": 3.4498814604077763,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.8370475797985158e-06,
+      "loss": 0.4616,
+      "step": 4550
+    },
+    {
+      "epoch": 3.4574679943100994,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.7609960169449308e-06,
+      "loss": 0.4772,
+      "step": 4560
+    },
+    {
+      "epoch": 3.465054528212423,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.6859180798897543e-06,
+      "loss": 0.4608,
+      "step": 4570
+    },
+    {
+      "epoch": 3.4726410621147465,
+      "grad_norm": 2.046875,
+      "learning_rate": 2.6118170555525177e-06,
+      "loss": 0.4634,
+      "step": 4580
+    },
+    {
+      "epoch": 3.4802275960170697,
+      "grad_norm": 2.125,
+      "learning_rate": 2.5386961880834057e-06,
+      "loss": 0.4667,
+      "step": 4590
+    },
+    {
+      "epoch": 3.487814129919393,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.466558678721248e-06,
+      "loss": 0.4686,
+      "step": 4600
+    },
+    {
+      "epoch": 3.4954006638217163,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.39540768565335e-06,
+      "loss": 0.4621,
+      "step": 4610
+    },
+    {
+      "epoch": 3.50298719772404,
+      "grad_norm": 2.125,
+      "learning_rate": 2.325246323877242e-06,
+      "loss": 0.4803,
+      "step": 4620
+    },
+    {
+      "epoch": 3.510573731626363,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.2560776650642907e-06,
+      "loss": 0.4703,
+      "step": 4630
+    },
+    {
+      "epoch": 3.5181602655286865,
+      "grad_norm": 2.140625,
+      "learning_rate": 2.187904737425245e-06,
+      "loss": 0.468,
+      "step": 4640
+    },
+    {
+      "epoch": 3.52574679943101,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.120730525577619e-06,
+      "loss": 0.4779,
+      "step": 4650
+    },
+    {
+      "epoch": 3.533333333333333,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.0545579704150775e-06,
+      "loss": 0.4869,
+      "step": 4660
+    },
+    {
+      "epoch": 3.5409198672356568,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.9893899689786335e-06,
+      "loss": 0.467,
+      "step": 4670
+    },
+    {
+      "epoch": 3.5485064011379803,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.925229374329854e-06,
+      "loss": 0.4692,
+      "step": 4680
+    },
+    {
+      "epoch": 3.5560929350403034,
+      "grad_norm": 2.125,
+      "learning_rate": 1.8620789954259283e-06,
+      "loss": 0.4634,
+      "step": 4690
+    },
+    {
+      "epoch": 3.563679468942627,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.7999415969967126e-06,
+      "loss": 0.4698,
+      "step": 4700
+    },
+    {
+      "epoch": 3.57126600284495,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.7388198994236576e-06,
+      "loss": 0.4591,
+      "step": 4710
+    },
+    {
+      "epoch": 3.5788525367472737,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.6787165786207455e-06,
+      "loss": 0.4726,
+      "step": 4720
+    },
+    {
+      "epoch": 3.5864390706495968,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6196342659173164e-06,
+      "loss": 0.4791,
+      "step": 4730
+    },
+    {
+      "epoch": 3.5940256045519203,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.5615755479428785e-06,
+      "loss": 0.4741,
+      "step": 4740
+    },
+    {
+      "epoch": 3.601612138454244,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.5045429665138527e-06,
+      "loss": 0.4566,
+      "step": 4750
+    },
+    {
+      "epoch": 3.609198672356567,
+      "grad_norm": 2.125,
+      "learning_rate": 1.448539018522313e-06,
+      "loss": 0.4703,
+      "step": 4760
+    },
+    {
+      "epoch": 3.6167852062588906,
+      "grad_norm": 2.375,
+      "learning_rate": 1.3935661558266476e-06,
+      "loss": 0.4731,
+      "step": 4770
+    },
+    {
+      "epoch": 3.624371740161214,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.339626785144235e-06,
+      "loss": 0.4647,
+      "step": 4780
+    },
+    {
+      "epoch": 3.631958274063537,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.2867232679460718e-06,
+      "loss": 0.4615,
+      "step": 4790
+    },
+    {
+      "epoch": 3.6395448079658603,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.2348579203533828e-06,
+      "loss": 0.4793,
+      "step": 4800
+    },
+    {
+      "epoch": 3.647131341868184,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.184033013036226e-06,
+      "loss": 0.462,
+      "step": 4810
+    },
+    {
+      "epoch": 3.6547178757705074,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.1342507711140815e-06,
+      "loss": 0.4645,
+      "step": 4820
+    },
+    {
+      "epoch": 3.6623044096728306,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.0855133740584305e-06,
+      "loss": 0.4829,
+      "step": 4830
+    },
+    {
+      "epoch": 3.669890943575154,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.0378229555973507e-06,
+      "loss": 0.4745,
+      "step": 4840
+    },
+    {
+      "epoch": 3.6774774774774777,
+      "grad_norm": 2.109375,
+      "learning_rate": 9.91181603622085e-07,
+      "loss": 0.4582,
+      "step": 4850
+    },
+    {
+      "epoch": 3.6850640113798008,
+      "grad_norm": 2.125,
+      "learning_rate": 9.455913600956517e-07,
+      "loss": 0.4748,
+      "step": 4860
+    },
+    {
+      "epoch": 3.6926505452821243,
+      "grad_norm": 2.296875,
+      "learning_rate": 9.01054220963421e-07,
+      "loss": 0.47,
+      "step": 4870
+    },
+    {
+      "epoch": 3.7002370791844474,
+      "grad_norm": 2.28125,
+      "learning_rate": 8.57572136065768e-07,
+      "loss": 0.4639,
+      "step": 4880
+    },
+    {
+      "epoch": 3.707823613086771,
+      "grad_norm": 2.21875,
+      "learning_rate": 8.151470090526691e-07,
+      "loss": 0.4628,
+      "step": 4890
+    },
+    {
+      "epoch": 3.715410146989094,
+      "grad_norm": 1.9921875,
+      "learning_rate": 7.737806973003969e-07,
+      "loss": 0.4655,
+      "step": 4900
+    },
+    {
+      "epoch": 3.7229966808914177,
+      "grad_norm": 2.171875,
+      "learning_rate": 7.334750118301814e-07,
+      "loss": 0.4495,
+      "step": 4910
+    },
+    {
+      "epoch": 3.730583214793741,
+      "grad_norm": 2.234375,
+      "learning_rate": 6.942317172289186e-07,
+      "loss": 0.4735,
+      "step": 4920
+    },
+    {
+      "epoch": 3.7381697486960643,
+      "grad_norm": 2.21875,
+      "learning_rate": 6.560525315719435e-07,
+      "loss": 0.4895,
+      "step": 4930
+    },
+    {
+      "epoch": 3.745756282598388,
+      "grad_norm": 2.203125,
+      "learning_rate": 6.189391263477812e-07,
+      "loss": 0.4722,
+      "step": 4940
+    },
+    {
+      "epoch": 3.7533428165007114,
+      "grad_norm": 2.171875,
+      "learning_rate": 5.82893126384998e-07,
+      "loss": 0.4525,
+      "step": 4950
+    },
+    {
+      "epoch": 3.7609293504030346,
+      "grad_norm": 2.296875,
+      "learning_rate": 5.479161097810354e-07,
+      "loss": 0.4751,
+      "step": 4960
+    },
+    {
+      "epoch": 3.768515884305358,
+      "grad_norm": 2.140625,
+      "learning_rate": 5.140096078331546e-07,
+      "loss": 0.4792,
+      "step": 4970
+    },
+    {
+      "epoch": 3.776102418207681,
+      "grad_norm": 2.15625,
+      "learning_rate": 4.811751049713681e-07,
+      "loss": 0.4706,
+      "step": 4980
+    },
+    {
+      "epoch": 3.7836889521100048,
+      "grad_norm": 2.015625,
+      "learning_rate": 4.494140386934631e-07,
+      "loss": 0.4626,
+      "step": 4990
+    },
+    {
+      "epoch": 3.791275486012328,
+      "grad_norm": 2.0625,
+      "learning_rate": 4.1872779950207497e-07,
+      "loss": 0.4536,
+      "step": 5000
+    },
+    {
+      "epoch": 3.791275486012328,
+      "eval_loss": 0.47889795899391174,
+      "eval_runtime": 9.1858,
+      "eval_samples_per_second": 120.838,
+      "eval_steps_per_second": 15.132,
+      "step": 5000
+    },
+    {
+      "epoch": 3.7988620199146514,
+      "grad_norm": 2.03125,
+      "learning_rate": 3.891177308437938e-07,
+      "loss": 0.4611,
+      "step": 5010
+    },
+    {
+      "epoch": 3.806448553816975,
+      "grad_norm": 2.09375,
+      "learning_rate": 3.605851290503559e-07,
+      "loss": 0.446,
+      "step": 5020
+    },
+    {
+      "epoch": 3.814035087719298,
+      "grad_norm": 2.03125,
+      "learning_rate": 3.331312432819034e-07,
+      "loss": 0.4699,
+      "step": 5030
+    },
+    {
+      "epoch": 3.8216216216216217,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.067572754722692e-07,
+      "loss": 0.4714,
+      "step": 5040
+    },
+    {
+      "epoch": 3.8292081555239452,
+      "grad_norm": 2.140625,
+      "learning_rate": 2.8146438027638054e-07,
+      "loss": 0.4608,
+      "step": 5050
+    },
+    {
+      "epoch": 3.8367946894262683,
+      "grad_norm": 1.8828125,
+      "learning_rate": 2.572536650196966e-07,
+      "loss": 0.448,
+      "step": 5060
+    },
+    {
+      "epoch": 3.844381223328592,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.3412618964973043e-07,
+      "loss": 0.4739,
+      "step": 5070
+    },
+    {
+      "epoch": 3.851967757230915,
+      "grad_norm": 2.015625,
+      "learning_rate": 2.1208296668964455e-07,
+      "loss": 0.4616,
+      "step": 5080
+    },
+    {
+      "epoch": 3.8595542911332386,
+      "grad_norm": 2.375,
+      "learning_rate": 1.911249611939253e-07,
+      "loss": 0.4641,
+      "step": 5090
+    },
+    {
+      "epoch": 3.8671408250355617,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.7125309070613049e-07,
+      "loss": 0.4796,
+      "step": 5100
+    },
+    {
+      "epoch": 3.8747273589378852,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.5246822521872152e-07,
+      "loss": 0.4713,
+      "step": 5110
+    },
+    {
+      "epoch": 3.882313892840209,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.3477118713496882e-07,
+      "loss": 0.4762,
+      "step": 5120
+    },
+    {
+      "epoch": 3.889900426742532,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.1816275123295306e-07,
+      "loss": 0.458,
+      "step": 5130
+    },
+    {
+      "epoch": 3.8974869606448554,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.0264364463164477e-07,
+      "loss": 0.4577,
+      "step": 5140
+    },
+    {
+      "epoch": 3.905073494547179,
+      "grad_norm": 2.0625,
+      "learning_rate": 8.82145467590717e-08,
+      "loss": 0.4602,
+      "step": 5150
+    },
+    {
+      "epoch": 3.912660028449502,
+      "grad_norm": 2.171875,
+      "learning_rate": 7.487608932256474e-08,
+      "loss": 0.4561,
+      "step": 5160
+    },
+    {
+      "epoch": 3.9202465623518257,
+      "grad_norm": 2.25,
+      "learning_rate": 6.262885628111337e-08,
+      "loss": 0.4599,
+      "step": 5170
+    },
+    {
+      "epoch": 3.927833096254149,
+      "grad_norm": 2.15625,
+      "learning_rate": 5.14733838197945e-08,
+      "loss": 0.4765,
+      "step": 5180
+    },
+    {
+      "epoch": 3.9354196301564723,
+      "grad_norm": 2.03125,
+      "learning_rate": 4.141016032629952e-08,
+      "loss": 0.4678,
+      "step": 5190
+    },
+    {
+      "epoch": 3.9430061640587954,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.243962636954867e-08,
+      "loss": 0.4729,
+      "step": 5200
+    },
+    {
+      "epoch": 3.950592697961119,
+      "grad_norm": 2.125,
+      "learning_rate": 2.4562174680406447e-08,
+      "loss": 0.4641,
+      "step": 5210
+    },
+    {
+      "epoch": 3.9581792318634426,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.77781501344898e-08,
+      "loss": 0.4816,
+      "step": 5220
+    },
+    {
+      "epoch": 3.9657657657657657,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.2087849737066338e-08,
+      "loss": 0.4602,
+      "step": 5230
+    },
+    {
+      "epoch": 3.9733522996680892,
+      "grad_norm": 2.140625,
+      "learning_rate": 7.491522610050817e-09,
+      "loss": 0.4546,
+      "step": 5240
+    },
+    {
+      "epoch": 3.980938833570413,
+      "grad_norm": 2.125,
+      "learning_rate": 3.9893699811e-09,
+      "loss": 0.4764,
+      "step": 5250
+    },
+    {
+      "epoch": 3.988525367472736,
+      "grad_norm": 2.125,
+      "learning_rate": 1.581545174803023e-09,
+      "loss": 0.4762,
+      "step": 5260
+    },
+    {
+      "epoch": 3.996111901375059,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.6815360597010065e-10,
+      "loss": 0.4728,
+      "step": 5270
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5276,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.0369051143168e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-5276/training_args.bin b/checkpoint-5276/training_args.bin
new file mode 100644
index 0000000..17f56c1
--- /dev/null
+++ b/checkpoint-5276/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a647ef5efc6ef28f13ecc4d3c99da28526fcc6c25fb933bc470139f20fd302
+size 5969
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..d64495c
--- /dev/null
+++ b/config.json
@@ -0,0 +1,37 @@
+{
+  "activation_function": "gelu",
+  "apply_query_key_layer_scaling": true,
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 3,
+  "initializer_range": 0.023,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 4096,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": 3072,
+  "n_layer": 12,
+  "n_positions": 4096,
+  "normalize_attention_scores": true,
+  "pad_token_id": 0,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "tokenizer_class": "GPTSw3Tokenizer",
+  "transformers_version": "4.57.1",
+  "use_cache": false,
+  "vocab_size": 64016,
+  "max_position_embeddings": 4096
+}
\ No newline at end of file
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000..aca1c6f
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "transformers_version": "4.57.1",
+  "use_cache": false
+}
diff --git a/gpt-sw3-126m-fp16.gguf b/gpt-sw3-126m-fp16.gguf
new file mode 100644
index 0000000..5fb8d11
--- /dev/null
+++ b/gpt-sw3-126m-fp16.gguf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f69d625cecbc7a5587705c932d951a1c178a9c74fb617ba4864281122d3f5fa
+size 282794848
diff --git a/gpt-sw3-126m-q4_k_m.gguf b/gpt-sw3-126m-q4_k_m.gguf
new file mode 100644
index 0000000..96ac155
--- /dev/null
+++ b/gpt-sw3-126m-q4_k_m.gguf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46db00c68b2d56a42625226aacd8fda630c310b43afa11dbf2145ee88e91f1e2
+size 109089472
diff --git a/gpt-sw3-126m-q5_k_m.gguf b/gpt-sw3-126m-q5_k_m.gguf
new file mode 100644
index 0000000..7e495f6
--- /dev/null
+++ b/gpt-sw3-126m-q5_k_m.gguf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98b61651505304e43a7220dba102f92ba20bf76ae01b6c3d67f7a2284df2b4ec
+size 116609728
diff --git a/gpt-sw3-126m-q6_k.gguf b/gpt-sw3-126m-q6_k.gguf
new file mode 100644
index 0000000..345180d
--- /dev/null
+++ b/gpt-sw3-126m-q6_k.gguf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0b11c05901294aa28d124686953e879cd7f9a543351a37c1e0590e956b53b57
+size 124600000
diff --git a/gpt-sw3-126m-q8_0.gguf b/gpt-sw3-126m-q8_0.gguf
new file mode 100644
index 0000000..cce9b72
--- /dev/null
+++ b/gpt-sw3-126m-q8_0.gguf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:427d3e0405b3dd06397acfdafb7e57194f9fff9f6d5975e4b7133a78558e651b
+size 157077088
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000..2826b8d
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:559430d7e10555635f26dfc35386b8342f4909c661317ebe70468670cd5553f6
+size 274747040
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000..fac732c
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,127 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "--",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "free_writing",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "matching_pairs",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "reorder_tokens",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "fill_in_the_blanks",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "general_vocab",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "morphology_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "synonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "antonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "definition_only",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[LESSON]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[PRACTICE_TEXT]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[EXERCISE_VARIATIONS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[A1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[A2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[B1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[B2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<s>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}
diff --git a/spiece.model b/spiece.model
new file mode 100644
index 0000000..491f2fd
--- /dev/null
+++ b/spiece.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4bda2cd84ff0ac659cda40e746c55f47ee3e57cf18471670ad26998c28be52d
+size 1071955
diff --git a/tokenizer.model b/tokenizer.model
new file mode 100644
index 0000000..491f2fd
--- /dev/null
+++ b/tokenizer.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4bda2cd84ff0ac659cda40e746c55f47ee3e57cf18471670ad26998c28be52d
+size 1071955
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..286c2b2
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,203 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "603": {
+      "content": "--",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "free_writing",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "matching_pairs",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64002": {
+      "content": "reorder_tokens",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64003": {
+      "content": "fill_in_the_blanks",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64004": {
+      "content": "general_vocab",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64005": {
+      "content": "morphology_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64006": {
+      "content": "synonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64007": {
+      "content": "antonym_focus",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64008": {
+      "content": "definition_only",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64009": {
+      "content": "[LESSON]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64010": {
+      "content": "[PRACTICE_TEXT]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64011": {
+      "content": "[EXERCISE_VARIATIONS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64012": {
+      "content": "[A1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64013": {
+      "content": "[A2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64014": {
+      "content": "[B1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64015": {
+      "content": "[B2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "--",
+    "free_writing",
+    "matching_pairs",
+    "reorder_tokens",
+    "fill_in_the_blanks",
+    "general_vocab",
+    "morphology_focus",
+    "synonym_focus",
+    "antonym_focus",
+    "definition_only",
+    "[LESSON]",
+    "[PRACTICE_TEXT]",
+    "[EXERCISE_VARIATIONS]",
+    "[A1]",
+    "[A2]",
+    "[B1]",
+    "[B2]"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "keep_accents": false,
+  "model_max_length": 4096,
+  "pad_token": "<pad>",
+  "remove_space": false,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "GPTSw3Tokenizer",
+  "unk_token": "<unk>"
+}
\ No newline at end of file
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000..17f56c1
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a647ef5efc6ef28f13ecc4d3c99da28526fcc6c25fb933bc470139f20fd302
+size 5969