From bc62cf516cefbf723b901cae39d8c4d51c4a9e66 Mon Sep 17 00:00:00 2001
From: ModelHub XC <noreply@modelhub.org.cn>
Date: Thu, 28 May 2026 21:44:21 +0800
Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?=
 =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?=
 =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Model: Janushi/FoodExtract-gemma-3-270m-fine-tune-v1
Source: Original Platform
---
 .gitattributes                        |   40 +
 README.md                             |   58 +
 chat_template.jinja                   |   47 +
 checkpoint-142/chat_template.jinja    |   47 +
 checkpoint-142/config.json            |   62 +
 checkpoint-142/generation_config.json |   13 +
 checkpoint-142/model.safetensors      |    3 +
 checkpoint-142/optimizer.pt           |    3 +
 checkpoint-142/rng_state.pth          |    3 +
 checkpoint-142/scheduler.pt           |    3 +
 checkpoint-142/tokenizer.json         |    3 +
 checkpoint-142/tokenizer_config.json  |   24 +
 checkpoint-142/trainer_state.json     | 1476 +++++++++++++
 checkpoint-142/training_args.bin      |    3 +
 checkpoint-213/chat_template.jinja    |   47 +
 checkpoint-213/config.json            |   62 +
 checkpoint-213/generation_config.json |   13 +
 checkpoint-213/model.safetensors      |    3 +
 checkpoint-213/optimizer.pt           |    3 +
 checkpoint-213/rng_state.pth          |    3 +
 checkpoint-213/scheduler.pt           |    3 +
 checkpoint-213/tokenizer.json         |    3 +
 checkpoint-213/tokenizer_config.json  |   24 +
 checkpoint-213/trainer_state.json     | 2197 +++++++++++++++++++
 checkpoint-213/training_args.bin      |    3 +
 checkpoint-284/chat_template.jinja    |   47 +
 checkpoint-284/config.json            |   62 +
 checkpoint-284/generation_config.json |   13 +
 checkpoint-284/model.safetensors      |    3 +
 checkpoint-284/optimizer.pt           |    3 +
 checkpoint-284/rng_state.pth          |    3 +
 checkpoint-284/scheduler.pt           |    3 +
 checkpoint-284/tokenizer.json         |    3 +
 checkpoint-284/tokenizer_config.json  |   24 +
 checkpoint-284/trainer_state.json     | 2918 +++++++++++++++++++++++++
 checkpoint-284/training_args.bin      |    3 +
 checkpoint-71/chat_template.jinja     |   47 +
 checkpoint-71/config.json             |   62 +
 checkpoint-71/generation_config.json  |   13 +
 checkpoint-71/model.safetensors       |    3 +
 checkpoint-71/optimizer.pt            |    3 +
 checkpoint-71/rng_state.pth           |    3 +
 checkpoint-71/scheduler.pt            |    3 +
 checkpoint-71/tokenizer.json          |    3 +
 checkpoint-71/tokenizer_config.json   |   24 +
 checkpoint-71/trainer_state.json      |  755 +++++++
 checkpoint-71/training_args.bin       |    3 +
 config.json                           |   62 +
 generation_config.json                |   13 +
 model.safetensors                     |    3 +
 tokenizer.json                        |    3 +
 tokenizer_config.json                 |   24 +
 training_args.bin                     |    3 +
 53 files changed, 8255 insertions(+)
 create mode 100644 .gitattributes
 create mode 100644 README.md
 create mode 100644 chat_template.jinja
 create mode 100644 checkpoint-142/chat_template.jinja
 create mode 100644 checkpoint-142/config.json
 create mode 100644 checkpoint-142/generation_config.json
 create mode 100644 checkpoint-142/model.safetensors
 create mode 100644 checkpoint-142/optimizer.pt
 create mode 100644 checkpoint-142/rng_state.pth
 create mode 100644 checkpoint-142/scheduler.pt
 create mode 100644 checkpoint-142/tokenizer.json
 create mode 100644 checkpoint-142/tokenizer_config.json
 create mode 100644 checkpoint-142/trainer_state.json
 create mode 100644 checkpoint-142/training_args.bin
 create mode 100644 checkpoint-213/chat_template.jinja
 create mode 100644 checkpoint-213/config.json
 create mode 100644 checkpoint-213/generation_config.json
 create mode 100644 checkpoint-213/model.safetensors
 create mode 100644 checkpoint-213/optimizer.pt
 create mode 100644 checkpoint-213/rng_state.pth
 create mode 100644 checkpoint-213/scheduler.pt
 create mode 100644 checkpoint-213/tokenizer.json
 create mode 100644 checkpoint-213/tokenizer_config.json
 create mode 100644 checkpoint-213/trainer_state.json
 create mode 100644 checkpoint-213/training_args.bin
 create mode 100644 checkpoint-284/chat_template.jinja
 create mode 100644 checkpoint-284/config.json
 create mode 100644 checkpoint-284/generation_config.json
 create mode 100644 checkpoint-284/model.safetensors
 create mode 100644 checkpoint-284/optimizer.pt
 create mode 100644 checkpoint-284/rng_state.pth
 create mode 100644 checkpoint-284/scheduler.pt
 create mode 100644 checkpoint-284/tokenizer.json
 create mode 100644 checkpoint-284/tokenizer_config.json
 create mode 100644 checkpoint-284/trainer_state.json
 create mode 100644 checkpoint-284/training_args.bin
 create mode 100644 checkpoint-71/chat_template.jinja
 create mode 100644 checkpoint-71/config.json
 create mode 100644 checkpoint-71/generation_config.json
 create mode 100644 checkpoint-71/model.safetensors
 create mode 100644 checkpoint-71/optimizer.pt
 create mode 100644 checkpoint-71/rng_state.pth
 create mode 100644 checkpoint-71/scheduler.pt
 create mode 100644 checkpoint-71/tokenizer.json
 create mode 100644 checkpoint-71/tokenizer_config.json
 create mode 100644 checkpoint-71/trainer_state.json
 create mode 100644 checkpoint-71/training_args.bin
 create mode 100644 config.json
 create mode 100644 generation_config.json
 create mode 100644 model.safetensors
 create mode 100644 tokenizer.json
 create mode 100644 tokenizer_config.json
 create mode 100644 training_args.bin

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..1dfd596
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,40 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-142/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-213/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-284/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-71/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..fa3da60
--- /dev/null
+++ b/README.md
@@ -0,0 +1,58 @@
+---
+base_model: google/gemma-3-270m-it
+library_name: transformers
+model_name: checkpoint_models
+tags:
+- generated_from_trainer
+- sft
+- trl
+licence: license
+---
+
+# Model Card for checkpoint_models
+
+This model is a fine-tuned version of [google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 0.29.1
+- Transformers: 5.0.0
+- Pytorch: 2.10.0+cu128
+- Datasets: 4.0.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```
\ No newline at end of file
diff --git a/chat_template.jinja b/chat_template.jinja
new file mode 100644
index 0000000..1117055
--- /dev/null
+++ b/chat_template.jinja
@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}
diff --git a/checkpoint-142/chat_template.jinja b/checkpoint-142/chat_template.jinja
new file mode 100644
index 0000000..1117055
--- /dev/null
+++ b/checkpoint-142/chat_template.jinja
@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}
diff --git a/checkpoint-142/config.json b/checkpoint-142/config.json
new file mode 100644
index 0000000..436e93d
--- /dev/null
+++ b/checkpoint-142/config.json
@@ -0,0 +1,62 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 512,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_bidirectional_attention": false,
+  "use_cache": false,
+  "vocab_size": 262144
+}
diff --git a/checkpoint-142/generation_config.json b/checkpoint-142/generation_config.json
new file mode 100644
index 0000000..c652aad
--- /dev/null
+++ b/checkpoint-142/generation_config.json
@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.0.0"
+}
diff --git a/checkpoint-142/model.safetensors b/checkpoint-142/model.safetensors
new file mode 100644
index 0000000..3a96843
--- /dev/null
+++ b/checkpoint-142/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3c02138bcd0c3740b3bc32e9796591fca892c30cd2670d290ac0a17b6106bbb
+size 536223056
diff --git a/checkpoint-142/optimizer.pt b/checkpoint-142/optimizer.pt
new file mode 100644
index 0000000..0857209
--- /dev/null
+++ b/checkpoint-142/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4170dff0a8aba276a89e01b9c401196d3d7a191402b469cc8fc360fc7f5eadb1
+size 1072591115
diff --git a/checkpoint-142/rng_state.pth b/checkpoint-142/rng_state.pth
new file mode 100644
index 0000000..435e005
--- /dev/null
+++ b/checkpoint-142/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e2011629d8bed3ef560fa11175cac55684c4e12a72634bb24abf767b6c7399
+size 14645
diff --git a/checkpoint-142/scheduler.pt b/checkpoint-142/scheduler.pt
new file mode 100644
index 0000000..9d530a8
--- /dev/null
+++ b/checkpoint-142/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2732da9c9500b70ace14309058979c65a5e31cf20be217688967b97f7bd768e2
+size 1465
diff --git a/checkpoint-142/tokenizer.json b/checkpoint-142/tokenizer.json
new file mode 100644
index 0000000..899af07
--- /dev/null
+++ b/checkpoint-142/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
+size 33384443
diff --git a/checkpoint-142/tokenizer_config.json b/checkpoint-142/tokenizer_config.json
new file mode 100644
index 0000000..7202584
--- /dev/null
+++ b/checkpoint-142/tokenizer_config.json
@@ -0,0 +1,24 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}
diff --git a/checkpoint-142/trainer_state.json b/checkpoint-142/trainer_state.json
new file mode 100644
index 0000000..6cb269a
--- /dev/null
+++ b/checkpoint-142/trainer_state.json
@@ -0,0 +1,1476 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 142,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4938980638980865,
+      "epoch": 0.014084507042253521,
+      "grad_norm": 89.5,
+      "learning_rate": 5e-05,
+      "loss": 4.519379615783691,
+      "mean_token_accuracy": 0.3811451345682144,
+      "num_tokens": 2953.0,
+      "step": 1
+    },
+    {
+      "entropy": 3.6362282037734985,
+      "epoch": 0.028169014084507043,
+      "grad_norm": 29.125,
+      "learning_rate": 5e-05,
+      "loss": 3.7604172229766846,
+      "mean_token_accuracy": 0.39103876799345016,
+      "num_tokens": 5196.0,
+      "step": 2
+    },
+    {
+      "entropy": 3.796808362007141,
+      "epoch": 0.04225352112676056,
+      "grad_norm": 18.75,
+      "learning_rate": 5e-05,
+      "loss": 3.3663859367370605,
+      "mean_token_accuracy": 0.42154909670352936,
+      "num_tokens": 7537.0,
+      "step": 3
+    },
+    {
+      "entropy": 3.071028172969818,
+      "epoch": 0.056338028169014086,
+      "grad_norm": 14.5625,
+      "learning_rate": 5e-05,
+      "loss": 3.1916425228118896,
+      "mean_token_accuracy": 0.4548730403184891,
+      "num_tokens": 9584.0,
+      "step": 4
+    },
+    {
+      "entropy": 2.7734431624412537,
+      "epoch": 0.07042253521126761,
+      "grad_norm": 15.0,
+      "learning_rate": 5e-05,
+      "loss": 2.661928653717041,
+      "mean_token_accuracy": 0.5473615825176239,
+      "num_tokens": 11344.0,
+      "step": 5
+    },
+    {
+      "entropy": 2.4827571511268616,
+      "epoch": 0.08450704225352113,
+      "grad_norm": 13.6875,
+      "learning_rate": 5e-05,
+      "loss": 2.691732168197632,
+      "mean_token_accuracy": 0.5095682442188263,
+      "num_tokens": 13289.0,
+      "step": 6
+    },
+    {
+      "entropy": 2.6064285039901733,
+      "epoch": 0.09859154929577464,
+      "grad_norm": 12.625,
+      "learning_rate": 5e-05,
+      "loss": 2.7455577850341797,
+      "mean_token_accuracy": 0.5099757686257362,
+      "num_tokens": 15664.0,
+      "step": 7
+    },
+    {
+      "entropy": 2.4781495332717896,
+      "epoch": 0.11267605633802817,
+      "grad_norm": 12.75,
+      "learning_rate": 5e-05,
+      "loss": 2.5589702129364014,
+      "mean_token_accuracy": 0.5618840456008911,
+      "num_tokens": 17331.0,
+      "step": 8
+    },
+    {
+      "entropy": 2.512305438518524,
+      "epoch": 0.1267605633802817,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.5893146991729736,
+      "mean_token_accuracy": 0.5188388600945473,
+      "num_tokens": 19724.0,
+      "step": 9
+    },
+    {
+      "entropy": 2.187403589487076,
+      "epoch": 0.14084507042253522,
+      "grad_norm": 11.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4727907180786133,
+      "mean_token_accuracy": 0.6085146963596344,
+      "num_tokens": 21334.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.8956857919692993,
+      "epoch": 0.15492957746478872,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.8553805351257324,
+      "mean_token_accuracy": 0.4754949361085892,
+      "num_tokens": 23770.0,
+      "step": 11
+    },
+    {
+      "entropy": 2.4213987588882446,
+      "epoch": 0.16901408450704225,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4139676094055176,
+      "mean_token_accuracy": 0.5475910231471062,
+      "num_tokens": 26233.0,
+      "step": 12
+    },
+    {
+      "entropy": 2.668797016143799,
+      "epoch": 0.18309859154929578,
+      "grad_norm": 12.25,
+      "learning_rate": 5e-05,
+      "loss": 2.769174337387085,
+      "mean_token_accuracy": 0.5351328551769257,
+      "num_tokens": 27692.0,
+      "step": 13
+    },
+    {
+      "entropy": 2.544685661792755,
+      "epoch": 0.19718309859154928,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.4870781898498535,
+      "mean_token_accuracy": 0.514883853495121,
+      "num_tokens": 30217.0,
+      "step": 14
+    },
+    {
+      "entropy": 2.5978153944015503,
+      "epoch": 0.2112676056338028,
+      "grad_norm": 10.875,
+      "learning_rate": 5e-05,
+      "loss": 2.639450788497925,
+      "mean_token_accuracy": 0.5468774288892746,
+      "num_tokens": 31969.0,
+      "step": 15
+    },
+    {
+      "entropy": 2.452792704105377,
+      "epoch": 0.22535211267605634,
+      "grad_norm": 11.5,
+      "learning_rate": 5e-05,
+      "loss": 2.5285415649414062,
+      "mean_token_accuracy": 0.5533104464411736,
+      "num_tokens": 34160.0,
+      "step": 16
+    },
+    {
+      "entropy": 2.234907388687134,
+      "epoch": 0.23943661971830985,
+      "grad_norm": 12.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.2553577423095703,
+      "mean_token_accuracy": 0.5958019495010376,
+      "num_tokens": 35899.0,
+      "step": 17
+    },
+    {
+      "entropy": 2.2781192660331726,
+      "epoch": 0.2535211267605634,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4401230812072754,
+      "mean_token_accuracy": 0.5632342845201492,
+      "num_tokens": 37693.0,
+      "step": 18
+    },
+    {
+      "entropy": 2.292062520980835,
+      "epoch": 0.2676056338028169,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 2.53348445892334,
+      "mean_token_accuracy": 0.5441952645778656,
+      "num_tokens": 39793.0,
+      "step": 19
+    },
+    {
+      "entropy": 2.5250023007392883,
+      "epoch": 0.28169014084507044,
+      "grad_norm": 13.5,
+      "learning_rate": 5e-05,
+      "loss": 2.7173304557800293,
+      "mean_token_accuracy": 0.4843936040997505,
+      "num_tokens": 42035.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.3936269879341125,
+      "epoch": 0.29577464788732394,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.397752046585083,
+      "mean_token_accuracy": 0.5632070526480675,
+      "num_tokens": 43606.0,
+      "step": 21
+    },
+    {
+      "entropy": 2.7215799689292908,
+      "epoch": 0.30985915492957744,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.7266643047332764,
+      "mean_token_accuracy": 0.4814189076423645,
+      "num_tokens": 46160.0,
+      "step": 22
+    },
+    {
+      "entropy": 2.4695653319358826,
+      "epoch": 0.323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3790149688720703,
+      "mean_token_accuracy": 0.5413414835929871,
+      "num_tokens": 48749.0,
+      "step": 23
+    },
+    {
+      "entropy": 2.3166028261184692,
+      "epoch": 0.3380281690140845,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.274803638458252,
+      "mean_token_accuracy": 0.5993590205907822,
+      "num_tokens": 50770.0,
+      "step": 24
+    },
+    {
+      "entropy": 2.285377472639084,
+      "epoch": 0.352112676056338,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.2308878898620605,
+      "mean_token_accuracy": 0.5828141123056412,
+      "num_tokens": 53004.0,
+      "step": 25
+    },
+    {
+      "entropy": 2.1157060265541077,
+      "epoch": 0.36619718309859156,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1033847332000732,
+      "mean_token_accuracy": 0.5867631435394287,
+      "num_tokens": 55108.0,
+      "step": 26
+    },
+    {
+      "entropy": 2.5209829807281494,
+      "epoch": 0.38028169014084506,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.3607499599456787,
+      "mean_token_accuracy": 0.5426046550273895,
+      "num_tokens": 57080.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.974183738231659,
+      "epoch": 0.39436619718309857,
+      "grad_norm": 7.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.9418365955352783,
+      "mean_token_accuracy": 0.6255736351013184,
+      "num_tokens": 59237.0,
+      "step": 28
+    },
+    {
+      "entropy": 2.252374231815338,
+      "epoch": 0.4084507042253521,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 2.228358507156372,
+      "mean_token_accuracy": 0.5917280316352844,
+      "num_tokens": 60850.0,
+      "step": 29
+    },
+    {
+      "entropy": 2.327530801296234,
+      "epoch": 0.4225352112676056,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.3329389095306396,
+      "mean_token_accuracy": 0.5718153715133667,
+      "num_tokens": 62734.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.2620502710342407,
+      "epoch": 0.43661971830985913,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1953892707824707,
+      "mean_token_accuracy": 0.5675001591444016,
+      "num_tokens": 64637.0,
+      "step": 31
+    },
+    {
+      "entropy": 2.445267140865326,
+      "epoch": 0.4507042253521127,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.501152276992798,
+      "mean_token_accuracy": 0.5355894565582275,
+      "num_tokens": 66880.0,
+      "step": 32
+    },
+    {
+      "entropy": 2.3873718976974487,
+      "epoch": 0.4647887323943662,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3468880653381348,
+      "mean_token_accuracy": 0.5372899621725082,
+      "num_tokens": 68623.0,
+      "step": 33
+    },
+    {
+      "entropy": 2.0545941591262817,
+      "epoch": 0.4788732394366197,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.1947593688964844,
+      "mean_token_accuracy": 0.6112199202179909,
+      "num_tokens": 70769.0,
+      "step": 34
+    },
+    {
+      "entropy": 2.350032150745392,
+      "epoch": 0.49295774647887325,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.418266773223877,
+      "mean_token_accuracy": 0.5104246437549591,
+      "num_tokens": 73653.0,
+      "step": 35
+    },
+    {
+      "entropy": 2.237062931060791,
+      "epoch": 0.5070422535211268,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4755356311798096,
+      "mean_token_accuracy": 0.5747656747698784,
+      "num_tokens": 75524.0,
+      "step": 36
+    },
+    {
+      "entropy": 2.0994813442230225,
+      "epoch": 0.5211267605633803,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.044315814971924,
+      "mean_token_accuracy": 0.6056438237428665,
+      "num_tokens": 77246.0,
+      "step": 37
+    },
+    {
+      "entropy": 2.5556156635284424,
+      "epoch": 0.5352112676056338,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.446904182434082,
+      "mean_token_accuracy": 0.5177429020404816,
+      "num_tokens": 79970.0,
+      "step": 38
+    },
+    {
+      "entropy": 2.337540030479431,
+      "epoch": 0.5492957746478874,
+      "grad_norm": 8.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.216331720352173,
+      "mean_token_accuracy": 0.58082115650177,
+      "num_tokens": 81876.0,
+      "step": 39
+    },
+    {
+      "entropy": 2.043498247861862,
+      "epoch": 0.5633802816901409,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9643354415893555,
+      "mean_token_accuracy": 0.607293888926506,
+      "num_tokens": 84298.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.4852016270160675,
+      "epoch": 0.5774647887323944,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.571437358856201,
+      "mean_token_accuracy": 0.5233406946063042,
+      "num_tokens": 86291.0,
+      "step": 41
+    },
+    {
+      "entropy": 2.5456899404525757,
+      "epoch": 0.5915492957746479,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 2.4434070587158203,
+      "mean_token_accuracy": 0.5472028106451035,
+      "num_tokens": 88194.0,
+      "step": 42
+    },
+    {
+      "entropy": 2.2191686630249023,
+      "epoch": 0.6056338028169014,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.225029468536377,
+      "mean_token_accuracy": 0.6006355881690979,
+      "num_tokens": 89922.0,
+      "step": 43
+    },
+    {
+      "entropy": 2.2295450568199158,
+      "epoch": 0.6197183098591549,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4000232219696045,
+      "mean_token_accuracy": 0.580116793513298,
+      "num_tokens": 92171.0,
+      "step": 44
+    },
+    {
+      "entropy": 2.2142042219638824,
+      "epoch": 0.6338028169014085,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2619423866271973,
+      "mean_token_accuracy": 0.5818447023630142,
+      "num_tokens": 94362.0,
+      "step": 45
+    },
+    {
+      "entropy": 2.3695861101150513,
+      "epoch": 0.647887323943662,
+      "grad_norm": 7.21875,
+      "learning_rate": 5e-05,
+      "loss": 2.4540324211120605,
+      "mean_token_accuracy": 0.5298221409320831,
+      "num_tokens": 96559.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.9680090248584747,
+      "epoch": 0.6619718309859155,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.9608612060546875,
+      "mean_token_accuracy": 0.6151116490364075,
+      "num_tokens": 99018.0,
+      "step": 47
+    },
+    {
+      "entropy": 2.449136793613434,
+      "epoch": 0.676056338028169,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.364119529724121,
+      "mean_token_accuracy": 0.5657317340373993,
+      "num_tokens": 100885.0,
+      "step": 48
+    },
+    {
+      "entropy": 2.109535813331604,
+      "epoch": 0.6901408450704225,
+      "grad_norm": 6.875,
+      "learning_rate": 5e-05,
+      "loss": 2.1192779541015625,
+      "mean_token_accuracy": 0.5970411896705627,
+      "num_tokens": 103357.0,
+      "step": 49
+    },
+    {
+      "entropy": 2.3080650568008423,
+      "epoch": 0.704225352112676,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.351897716522217,
+      "mean_token_accuracy": 0.5307685062289238,
+      "num_tokens": 105870.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.1463415920734406,
+      "epoch": 0.7183098591549296,
+      "grad_norm": 7.5,
+      "learning_rate": 5e-05,
+      "loss": 2.2693376541137695,
+      "mean_token_accuracy": 0.5934874191880226,
+      "num_tokens": 107995.0,
+      "step": 51
+    },
+    {
+      "entropy": 2.1181687712669373,
+      "epoch": 0.7323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.1629841327667236,
+      "mean_token_accuracy": 0.6063478291034698,
+      "num_tokens": 110358.0,
+      "step": 52
+    },
+    {
+      "entropy": 2.0169546008110046,
+      "epoch": 0.7464788732394366,
+      "grad_norm": 7.78125,
+      "learning_rate": 5e-05,
+      "loss": 1.817868709564209,
+      "mean_token_accuracy": 0.6273332387208939,
+      "num_tokens": 112267.0,
+      "step": 53
+    },
+    {
+      "entropy": 2.189443975687027,
+      "epoch": 0.7605633802816901,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.3292086124420166,
+      "mean_token_accuracy": 0.6041209101676941,
+      "num_tokens": 113899.0,
+      "step": 54
+    },
+    {
+      "entropy": 2.1946000456809998,
+      "epoch": 0.7746478873239436,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2774949073791504,
+      "mean_token_accuracy": 0.5620942413806915,
+      "num_tokens": 116015.0,
+      "step": 55
+    },
+    {
+      "entropy": 2.1172711849212646,
+      "epoch": 0.7887323943661971,
+      "grad_norm": 7.90625,
+      "learning_rate": 5e-05,
+      "loss": 2.1236391067504883,
+      "mean_token_accuracy": 0.5911316424608231,
+      "num_tokens": 118044.0,
+      "step": 56
+    },
+    {
+      "entropy": 2.144031971693039,
+      "epoch": 0.8028169014084507,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.187140703201294,
+      "mean_token_accuracy": 0.5624879449605942,
+      "num_tokens": 120180.0,
+      "step": 57
+    },
+    {
+      "entropy": 2.181178390979767,
+      "epoch": 0.8169014084507042,
+      "grad_norm": 10.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.113279104232788,
+      "mean_token_accuracy": 0.6004882454872131,
+      "num_tokens": 121676.0,
+      "step": 58
+    },
+    {
+      "entropy": 2.314671218395233,
+      "epoch": 0.8309859154929577,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.331024169921875,
+      "mean_token_accuracy": 0.5824068263173103,
+      "num_tokens": 123161.0,
+      "step": 59
+    },
+    {
+      "entropy": 2.0048855245113373,
+      "epoch": 0.8450704225352113,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.070265054702759,
+      "mean_token_accuracy": 0.6141902804374695,
+      "num_tokens": 125016.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.966452568769455,
+      "epoch": 0.8591549295774648,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.9488521814346313,
+      "mean_token_accuracy": 0.5854089111089706,
+      "num_tokens": 127349.0,
+      "step": 61
+    },
+    {
+      "entropy": 2.1519170999526978,
+      "epoch": 0.8732394366197183,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.1743807792663574,
+      "mean_token_accuracy": 0.5636171847581863,
+      "num_tokens": 129742.0,
+      "step": 62
+    },
+    {
+      "entropy": 2.7546687722206116,
+      "epoch": 0.8873239436619719,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.7343056201934814,
+      "mean_token_accuracy": 0.496803916990757,
+      "num_tokens": 131743.0,
+      "step": 63
+    },
+    {
+      "entropy": 2.421773910522461,
+      "epoch": 0.9014084507042254,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.6149866580963135,
+      "mean_token_accuracy": 0.5249136164784431,
+      "num_tokens": 134002.0,
+      "step": 64
+    },
+    {
+      "entropy": 2.0106727480888367,
+      "epoch": 0.9154929577464789,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.940914273262024,
+      "mean_token_accuracy": 0.5855439230799675,
+      "num_tokens": 136120.0,
+      "step": 65
+    },
+    {
+      "entropy": 2.1143648624420166,
+      "epoch": 0.9295774647887324,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.2469482421875,
+      "mean_token_accuracy": 0.6145937740802765,
+      "num_tokens": 137997.0,
+      "step": 66
+    },
+    {
+      "entropy": 2.1894670128822327,
+      "epoch": 0.9436619718309859,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.20756459236145,
+      "mean_token_accuracy": 0.5872255340218544,
+      "num_tokens": 139904.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.75501748919487,
+      "epoch": 0.9577464788732394,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.9068934917449951,
+      "mean_token_accuracy": 0.6642919778823853,
+      "num_tokens": 141166.0,
+      "step": 68
+    },
+    {
+      "entropy": 2.204477220773697,
+      "epoch": 0.971830985915493,
+      "grad_norm": 7.875,
+      "learning_rate": 5e-05,
+      "loss": 2.286216974258423,
+      "mean_token_accuracy": 0.5354761853814125,
+      "num_tokens": 143738.0,
+      "step": 69
+    },
+    {
+      "entropy": 2.1219914853572845,
+      "epoch": 0.9859154929577465,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 2.274585723876953,
+      "mean_token_accuracy": 0.609759122133255,
+      "num_tokens": 145439.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.9260511100292206,
+      "epoch": 1.0,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.8763841390609741,
+      "mean_token_accuracy": 0.5979989916086197,
+      "num_tokens": 147650.0,
+      "step": 71
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 2.1684498711371085,
+      "eval_loss": 2.284580707550049,
+      "eval_mean_token_accuracy": 0.5757196218195096,
+      "eval_num_tokens": 147650.0,
+      "eval_runtime": 18.8201,
+      "eval_samples_per_second": 15.09,
+      "eval_steps_per_second": 3.773,
+      "step": 71
+    },
+    {
+      "entropy": 2.24437940120697,
+      "epoch": 1.0140845070422535,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 2.148402214050293,
+      "mean_token_accuracy": 0.5866604596376419,
+      "num_tokens": 149372.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.630838692188263,
+      "epoch": 1.028169014084507,
+      "grad_norm": 7.0,
+      "learning_rate": 5e-05,
+      "loss": 1.526665449142456,
+      "mean_token_accuracy": 0.6816482096910477,
+      "num_tokens": 151274.0,
+      "step": 73
+    },
+    {
+      "entropy": 2.193234443664551,
+      "epoch": 1.0422535211267605,
+      "grad_norm": 7.90625,
+      "learning_rate": 5e-05,
+      "loss": 2.176100969314575,
+      "mean_token_accuracy": 0.6048732623457909,
+      "num_tokens": 153103.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.7758266925811768,
+      "epoch": 1.056338028169014,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.7472716569900513,
+      "mean_token_accuracy": 0.6403823792934418,
+      "num_tokens": 155291.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.8887377083301544,
+      "epoch": 1.0704225352112675,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.872934341430664,
+      "mean_token_accuracy": 0.6085800677537918,
+      "num_tokens": 157847.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.6055135428905487,
+      "epoch": 1.084507042253521,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.6859486103057861,
+      "mean_token_accuracy": 0.6478956639766693,
+      "num_tokens": 160538.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.9197329580783844,
+      "epoch": 1.0985915492957747,
+      "grad_norm": 7.375,
+      "learning_rate": 5e-05,
+      "loss": 1.929700493812561,
+      "mean_token_accuracy": 0.6054274141788483,
+      "num_tokens": 162709.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.6682755053043365,
+      "epoch": 1.1126760563380282,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.899672269821167,
+      "mean_token_accuracy": 0.6613650918006897,
+      "num_tokens": 164478.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.8098062872886658,
+      "epoch": 1.1267605633802817,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7989305257797241,
+      "mean_token_accuracy": 0.6530113369226456,
+      "num_tokens": 165767.0,
+      "step": 80
+    },
+    {
+      "entropy": 2.3263643383979797,
+      "epoch": 1.1408450704225352,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.32468843460083,
+      "mean_token_accuracy": 0.5657684504985809,
+      "num_tokens": 167374.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.6652490496635437,
+      "epoch": 1.1549295774647887,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.821378469467163,
+      "mean_token_accuracy": 0.6479186862707138,
+      "num_tokens": 169216.0,
+      "step": 82
+    },
+    {
+      "entropy": 2.157005727291107,
+      "epoch": 1.1690140845070423,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.1137959957122803,
+      "mean_token_accuracy": 0.5775181204080582,
+      "num_tokens": 171194.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.6896095722913742,
+      "epoch": 1.1830985915492958,
+      "grad_norm": 8.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.785524606704712,
+      "mean_token_accuracy": 0.6791908890008926,
+      "num_tokens": 173018.0,
+      "step": 84
+    },
+    {
+      "entropy": 2.241276800632477,
+      "epoch": 1.1971830985915493,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.1982414722442627,
+      "mean_token_accuracy": 0.5830243527889252,
+      "num_tokens": 174822.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.8407301902770996,
+      "epoch": 1.2112676056338028,
+      "grad_norm": 7.65625,
+      "learning_rate": 5e-05,
+      "loss": 1.752529501914978,
+      "mean_token_accuracy": 0.6172220706939697,
+      "num_tokens": 176932.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.707479178905487,
+      "epoch": 1.2253521126760563,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.6565136909484863,
+      "mean_token_accuracy": 0.6619749516248703,
+      "num_tokens": 178804.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.6418649852275848,
+      "epoch": 1.2394366197183098,
+      "grad_norm": 7.15625,
+      "learning_rate": 5e-05,
+      "loss": 1.5298699140548706,
+      "mean_token_accuracy": 0.6645439267158508,
+      "num_tokens": 181353.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.6070191860198975,
+      "epoch": 1.2535211267605635,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.7814466953277588,
+      "mean_token_accuracy": 0.6640422344207764,
+      "num_tokens": 183278.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.8768837749958038,
+      "epoch": 1.267605633802817,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9979424476623535,
+      "mean_token_accuracy": 0.6454368084669113,
+      "num_tokens": 185041.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.809422641992569,
+      "epoch": 1.2816901408450705,
+      "grad_norm": 7.78125,
+      "learning_rate": 5e-05,
+      "loss": 1.8236162662506104,
+      "mean_token_accuracy": 0.6187984794378281,
+      "num_tokens": 187646.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.8032868206501007,
+      "epoch": 1.295774647887324,
+      "grad_norm": 7.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.8008391857147217,
+      "mean_token_accuracy": 0.6168265342712402,
+      "num_tokens": 189952.0,
+      "step": 92
+    },
+    {
+      "entropy": 2.107921302318573,
+      "epoch": 1.3098591549295775,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 2.077434539794922,
+      "mean_token_accuracy": 0.5662726014852524,
+      "num_tokens": 192592.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.6842139065265656,
+      "epoch": 1.323943661971831,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.6454025506973267,
+      "mean_token_accuracy": 0.6700995117425919,
+      "num_tokens": 194549.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.859749436378479,
+      "epoch": 1.3380281690140845,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9091846942901611,
+      "mean_token_accuracy": 0.6098746657371521,
+      "num_tokens": 196448.0,
+      "step": 95
+    },
+    {
+      "entropy": 2.018050581216812,
+      "epoch": 1.352112676056338,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.1190457344055176,
+      "mean_token_accuracy": 0.5959457382559776,
+      "num_tokens": 198615.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.9890158474445343,
+      "epoch": 1.3661971830985915,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.960221529006958,
+      "mean_token_accuracy": 0.5875463634729385,
+      "num_tokens": 201013.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.8716930747032166,
+      "epoch": 1.380281690140845,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 2.0004959106445312,
+      "mean_token_accuracy": 0.6175930351018906,
+      "num_tokens": 202969.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.7610785365104675,
+      "epoch": 1.3943661971830985,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.6850508451461792,
+      "mean_token_accuracy": 0.6504905521869659,
+      "num_tokens": 204906.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.9090750515460968,
+      "epoch": 1.408450704225352,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.938533067703247,
+      "mean_token_accuracy": 0.621729701757431,
+      "num_tokens": 206869.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.8917019963264465,
+      "epoch": 1.4225352112676055,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.0368175506591797,
+      "mean_token_accuracy": 0.6267035603523254,
+      "num_tokens": 208432.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.7962620854377747,
+      "epoch": 1.436619718309859,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.8455748558044434,
+      "mean_token_accuracy": 0.608839362859726,
+      "num_tokens": 210949.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.724296659231186,
+      "epoch": 1.4507042253521127,
+      "grad_norm": 10.25,
+      "learning_rate": 5e-05,
+      "loss": 1.7760952711105347,
+      "mean_token_accuracy": 0.6573535352945328,
+      "num_tokens": 212595.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.8014512360095978,
+      "epoch": 1.4647887323943662,
+      "grad_norm": 7.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.8614972829818726,
+      "mean_token_accuracy": 0.5946371257305145,
+      "num_tokens": 215384.0,
+      "step": 104
+    },
+    {
+      "entropy": 1.8427681922912598,
+      "epoch": 1.4788732394366197,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 1.880817174911499,
+      "mean_token_accuracy": 0.6408448368310928,
+      "num_tokens": 217561.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.6576282978057861,
+      "epoch": 1.4929577464788732,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.6814919710159302,
+      "mean_token_accuracy": 0.6402300000190735,
+      "num_tokens": 219087.0,
+      "step": 106
+    },
+    {
+      "entropy": 1.5478721261024475,
+      "epoch": 1.5070422535211268,
+      "grad_norm": 7.59375,
+      "learning_rate": 5e-05,
+      "loss": 1.5617148876190186,
+      "mean_token_accuracy": 0.6625793874263763,
+      "num_tokens": 221887.0,
+      "step": 107
+    },
+    {
+      "entropy": 1.923494279384613,
+      "epoch": 1.5211267605633803,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.0690817832946777,
+      "mean_token_accuracy": 0.5978109538555145,
+      "num_tokens": 224182.0,
+      "step": 108
+    },
+    {
+      "entropy": 1.8152073919773102,
+      "epoch": 1.5352112676056338,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 1.7398602962493896,
+      "mean_token_accuracy": 0.6207661181688309,
+      "num_tokens": 226591.0,
+      "step": 109
+    },
+    {
+      "entropy": 2.1172706484794617,
+      "epoch": 1.5492957746478875,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.237187385559082,
+      "mean_token_accuracy": 0.5684898793697357,
+      "num_tokens": 228529.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.686636283993721,
+      "epoch": 1.563380281690141,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.8167091608047485,
+      "mean_token_accuracy": 0.6692909449338913,
+      "num_tokens": 229973.0,
+      "step": 111
+    },
+    {
+      "entropy": 2.285969853401184,
+      "epoch": 1.5774647887323945,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 2.308091878890991,
+      "mean_token_accuracy": 0.5341236144304276,
+      "num_tokens": 232232.0,
+      "step": 112
+    },
+    {
+      "entropy": 1.7772781550884247,
+      "epoch": 1.591549295774648,
+      "grad_norm": 7.09375,
+      "learning_rate": 5e-05,
+      "loss": 1.7048449516296387,
+      "mean_token_accuracy": 0.6309333443641663,
+      "num_tokens": 234998.0,
+      "step": 113
+    },
+    {
+      "entropy": 1.8632723987102509,
+      "epoch": 1.6056338028169015,
+      "grad_norm": 6.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.8152518272399902,
+      "mean_token_accuracy": 0.5958341658115387,
+      "num_tokens": 237842.0,
+      "step": 114
+    },
+    {
+      "entropy": 1.7296560108661652,
+      "epoch": 1.619718309859155,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.7719354629516602,
+      "mean_token_accuracy": 0.6910708993673325,
+      "num_tokens": 239068.0,
+      "step": 115
+    },
+    {
+      "entropy": 2.026154547929764,
+      "epoch": 1.6338028169014085,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 2.067565441131592,
+      "mean_token_accuracy": 0.5664031803607941,
+      "num_tokens": 241497.0,
+      "step": 116
+    },
+    {
+      "entropy": 1.6592170596122742,
+      "epoch": 1.647887323943662,
+      "grad_norm": 7.65625,
+      "learning_rate": 5e-05,
+      "loss": 1.7524383068084717,
+      "mean_token_accuracy": 0.6437603831291199,
+      "num_tokens": 243721.0,
+      "step": 117
+    },
+    {
+      "entropy": 1.8260776698589325,
+      "epoch": 1.6619718309859155,
+      "grad_norm": 8.0,
+      "learning_rate": 5e-05,
+      "loss": 1.7933809757232666,
+      "mean_token_accuracy": 0.6192324906587601,
+      "num_tokens": 246193.0,
+      "step": 118
+    },
+    {
+      "entropy": 1.6278434097766876,
+      "epoch": 1.676056338028169,
+      "grad_norm": 7.875,
+      "learning_rate": 5e-05,
+      "loss": 1.6023352146148682,
+      "mean_token_accuracy": 0.6632742285728455,
+      "num_tokens": 248065.0,
+      "step": 119
+    },
+    {
+      "entropy": 1.5735027194023132,
+      "epoch": 1.6901408450704225,
+      "grad_norm": 7.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.5997726917266846,
+      "mean_token_accuracy": 0.6820999830961227,
+      "num_tokens": 250126.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.7936385571956635,
+      "epoch": 1.704225352112676,
+      "grad_norm": 7.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.662062644958496,
+      "mean_token_accuracy": 0.6343215107917786,
+      "num_tokens": 252242.0,
+      "step": 121
+    },
+    {
+      "entropy": 2.0255846083164215,
+      "epoch": 1.7183098591549295,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.9866116046905518,
+      "mean_token_accuracy": 0.5996693223714828,
+      "num_tokens": 254704.0,
+      "step": 122
+    },
+    {
+      "entropy": 1.8428587317466736,
+      "epoch": 1.732394366197183,
+      "grad_norm": 7.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.786312460899353,
+      "mean_token_accuracy": 0.6079171597957611,
+      "num_tokens": 257154.0,
+      "step": 123
+    },
+    {
+      "entropy": 1.8701068460941315,
+      "epoch": 1.7464788732394365,
+      "grad_norm": 6.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.814254641532898,
+      "mean_token_accuracy": 0.6146602630615234,
+      "num_tokens": 260050.0,
+      "step": 124
+    },
+    {
+      "entropy": 1.7365414798259735,
+      "epoch": 1.76056338028169,
+      "grad_norm": 6.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.6392512321472168,
+      "mean_token_accuracy": 0.631788820028305,
+      "num_tokens": 262731.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.5750393569469452,
+      "epoch": 1.7746478873239435,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.652256727218628,
+      "mean_token_accuracy": 0.6727721691131592,
+      "num_tokens": 264264.0,
+      "step": 126
+    },
+    {
+      "entropy": 1.6853447258472443,
+      "epoch": 1.788732394366197,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.8110965490341187,
+      "mean_token_accuracy": 0.6612876951694489,
+      "num_tokens": 265839.0,
+      "step": 127
+    },
+    {
+      "entropy": 1.9790962934494019,
+      "epoch": 1.8028169014084507,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.9432560205459595,
+      "mean_token_accuracy": 0.5927930474281311,
+      "num_tokens": 268531.0,
+      "step": 128
+    },
+    {
+      "entropy": 1.6438206434249878,
+      "epoch": 1.8169014084507042,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 2.005178689956665,
+      "mean_token_accuracy": 0.6584273278713226,
+      "num_tokens": 270295.0,
+      "step": 129
+    },
+    {
+      "entropy": 1.8002346009016037,
+      "epoch": 1.8309859154929577,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.9875435829162598,
+      "mean_token_accuracy": 0.6358410120010376,
+      "num_tokens": 272470.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.811629831790924,
+      "epoch": 1.8450704225352113,
+      "grad_norm": 9.25,
+      "learning_rate": 5e-05,
+      "loss": 1.896381139755249,
+      "mean_token_accuracy": 0.6606249660253525,
+      "num_tokens": 274166.0,
+      "step": 131
+    },
+    {
+      "entropy": 1.7871823608875275,
+      "epoch": 1.8591549295774648,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7262006998062134,
+      "mean_token_accuracy": 0.6428729891777039,
+      "num_tokens": 276354.0,
+      "step": 132
+    },
+    {
+      "entropy": 1.5866018235683441,
+      "epoch": 1.8732394366197183,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.54076087474823,
+      "mean_token_accuracy": 0.6837680488824844,
+      "num_tokens": 278044.0,
+      "step": 133
+    },
+    {
+      "entropy": 1.9443781971931458,
+      "epoch": 1.887323943661972,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.935890555381775,
+      "mean_token_accuracy": 0.5984784960746765,
+      "num_tokens": 279727.0,
+      "step": 134
+    },
+    {
+      "entropy": 1.6744503676891327,
+      "epoch": 1.9014084507042255,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.667353868484497,
+      "mean_token_accuracy": 0.6531110107898712,
+      "num_tokens": 281727.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.5089002698659897,
+      "epoch": 1.915492957746479,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.7763428688049316,
+      "mean_token_accuracy": 0.6703489422798157,
+      "num_tokens": 283499.0,
+      "step": 136
+    },
+    {
+      "entropy": 1.7822388112545013,
+      "epoch": 1.9295774647887325,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.7652621269226074,
+      "mean_token_accuracy": 0.6092153340578079,
+      "num_tokens": 286098.0,
+      "step": 137
+    },
+    {
+      "entropy": 1.633983552455902,
+      "epoch": 1.943661971830986,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7642450332641602,
+      "mean_token_accuracy": 0.6580131351947784,
+      "num_tokens": 288020.0,
+      "step": 138
+    },
+    {
+      "entropy": 1.9797956943511963,
+      "epoch": 1.9577464788732395,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.8309262990951538,
+      "mean_token_accuracy": 0.6048336029052734,
+      "num_tokens": 290401.0,
+      "step": 139
+    },
+    {
+      "entropy": 1.5087556838989258,
+      "epoch": 1.971830985915493,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.6536986827850342,
+      "mean_token_accuracy": 0.7231832593679428,
+      "num_tokens": 291584.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.9238645434379578,
+      "epoch": 1.9859154929577465,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.7226345539093018,
+      "mean_token_accuracy": 0.6254735887050629,
+      "num_tokens": 293607.0,
+      "step": 141
+    },
+    {
+      "entropy": 1.587816596031189,
+      "epoch": 2.0,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.5887134075164795,
+      "mean_token_accuracy": 0.6791131645441055,
+      "num_tokens": 295300.0,
+      "step": 142
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 1.972671693479511,
+      "eval_loss": 2.2974655628204346,
+      "eval_mean_token_accuracy": 0.5787080614499642,
+      "eval_num_tokens": 295300.0,
+      "eval_runtime": 18.8723,
+      "eval_samples_per_second": 15.049,
+      "eval_steps_per_second": 3.762,
+      "step": 142
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 284,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 303200469906432.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-142/training_args.bin b/checkpoint-142/training_args.bin
new file mode 100644
index 0000000..0b099d6
--- /dev/null
+++ b/checkpoint-142/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
+size 5585
diff --git a/checkpoint-213/chat_template.jinja b/checkpoint-213/chat_template.jinja
new file mode 100644
index 0000000..1117055
--- /dev/null
+++ b/checkpoint-213/chat_template.jinja
@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}
diff --git a/checkpoint-213/config.json b/checkpoint-213/config.json
new file mode 100644
index 0000000..436e93d
--- /dev/null
+++ b/checkpoint-213/config.json
@@ -0,0 +1,62 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 512,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_bidirectional_attention": false,
+  "use_cache": false,
+  "vocab_size": 262144
+}
diff --git a/checkpoint-213/generation_config.json b/checkpoint-213/generation_config.json
new file mode 100644
index 0000000..c652aad
--- /dev/null
+++ b/checkpoint-213/generation_config.json
@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.0.0"
+}
diff --git a/checkpoint-213/model.safetensors b/checkpoint-213/model.safetensors
new file mode 100644
index 0000000..c53b071
--- /dev/null
+++ b/checkpoint-213/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4eaf9a42a809acc2e2e2e1a02042ad0dbd267ec8a17813c5a7a4e79f6d5f988
+size 536223056
diff --git a/checkpoint-213/optimizer.pt b/checkpoint-213/optimizer.pt
new file mode 100644
index 0000000..6a04aa5
--- /dev/null
+++ b/checkpoint-213/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b2068fdfa062a74347042b74ff1e647b9bb9b3a0d868506e385e0eba26e216e
+size 1072591115
diff --git a/checkpoint-213/rng_state.pth b/checkpoint-213/rng_state.pth
new file mode 100644
index 0000000..3ab6836
--- /dev/null
+++ b/checkpoint-213/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01f9a0f7843a37be87edd23f4e88aa93b38b95cc2c07503eeb1cf2e4632453a2
+size 14645
diff --git a/checkpoint-213/scheduler.pt b/checkpoint-213/scheduler.pt
new file mode 100644
index 0000000..5cca0b0
--- /dev/null
+++ b/checkpoint-213/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7aa4a23cfb25aa3311924461ac68a6e00e95172dc91f470348c1ae92a8f6ccc
+size 1465
diff --git a/checkpoint-213/tokenizer.json b/checkpoint-213/tokenizer.json
new file mode 100644
index 0000000..899af07
--- /dev/null
+++ b/checkpoint-213/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
+size 33384443
diff --git a/checkpoint-213/tokenizer_config.json b/checkpoint-213/tokenizer_config.json
new file mode 100644
index 0000000..7202584
--- /dev/null
+++ b/checkpoint-213/tokenizer_config.json
@@ -0,0 +1,24 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}
diff --git a/checkpoint-213/trainer_state.json b/checkpoint-213/trainer_state.json
new file mode 100644
index 0000000..47ea40c
--- /dev/null
+++ b/checkpoint-213/trainer_state.json
@@ -0,0 +1,2197 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 213,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4938980638980865,
+      "epoch": 0.014084507042253521,
+      "grad_norm": 89.5,
+      "learning_rate": 5e-05,
+      "loss": 4.519379615783691,
+      "mean_token_accuracy": 0.3811451345682144,
+      "num_tokens": 2953.0,
+      "step": 1
+    },
+    {
+      "entropy": 3.6362282037734985,
+      "epoch": 0.028169014084507043,
+      "grad_norm": 29.125,
+      "learning_rate": 5e-05,
+      "loss": 3.7604172229766846,
+      "mean_token_accuracy": 0.39103876799345016,
+      "num_tokens": 5196.0,
+      "step": 2
+    },
+    {
+      "entropy": 3.796808362007141,
+      "epoch": 0.04225352112676056,
+      "grad_norm": 18.75,
+      "learning_rate": 5e-05,
+      "loss": 3.3663859367370605,
+      "mean_token_accuracy": 0.42154909670352936,
+      "num_tokens": 7537.0,
+      "step": 3
+    },
+    {
+      "entropy": 3.071028172969818,
+      "epoch": 0.056338028169014086,
+      "grad_norm": 14.5625,
+      "learning_rate": 5e-05,
+      "loss": 3.1916425228118896,
+      "mean_token_accuracy": 0.4548730403184891,
+      "num_tokens": 9584.0,
+      "step": 4
+    },
+    {
+      "entropy": 2.7734431624412537,
+      "epoch": 0.07042253521126761,
+      "grad_norm": 15.0,
+      "learning_rate": 5e-05,
+      "loss": 2.661928653717041,
+      "mean_token_accuracy": 0.5473615825176239,
+      "num_tokens": 11344.0,
+      "step": 5
+    },
+    {
+      "entropy": 2.4827571511268616,
+      "epoch": 0.08450704225352113,
+      "grad_norm": 13.6875,
+      "learning_rate": 5e-05,
+      "loss": 2.691732168197632,
+      "mean_token_accuracy": 0.5095682442188263,
+      "num_tokens": 13289.0,
+      "step": 6
+    },
+    {
+      "entropy": 2.6064285039901733,
+      "epoch": 0.09859154929577464,
+      "grad_norm": 12.625,
+      "learning_rate": 5e-05,
+      "loss": 2.7455577850341797,
+      "mean_token_accuracy": 0.5099757686257362,
+      "num_tokens": 15664.0,
+      "step": 7
+    },
+    {
+      "entropy": 2.4781495332717896,
+      "epoch": 0.11267605633802817,
+      "grad_norm": 12.75,
+      "learning_rate": 5e-05,
+      "loss": 2.5589702129364014,
+      "mean_token_accuracy": 0.5618840456008911,
+      "num_tokens": 17331.0,
+      "step": 8
+    },
+    {
+      "entropy": 2.512305438518524,
+      "epoch": 0.1267605633802817,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.5893146991729736,
+      "mean_token_accuracy": 0.5188388600945473,
+      "num_tokens": 19724.0,
+      "step": 9
+    },
+    {
+      "entropy": 2.187403589487076,
+      "epoch": 0.14084507042253522,
+      "grad_norm": 11.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4727907180786133,
+      "mean_token_accuracy": 0.6085146963596344,
+      "num_tokens": 21334.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.8956857919692993,
+      "epoch": 0.15492957746478872,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.8553805351257324,
+      "mean_token_accuracy": 0.4754949361085892,
+      "num_tokens": 23770.0,
+      "step": 11
+    },
+    {
+      "entropy": 2.4213987588882446,
+      "epoch": 0.16901408450704225,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4139676094055176,
+      "mean_token_accuracy": 0.5475910231471062,
+      "num_tokens": 26233.0,
+      "step": 12
+    },
+    {
+      "entropy": 2.668797016143799,
+      "epoch": 0.18309859154929578,
+      "grad_norm": 12.25,
+      "learning_rate": 5e-05,
+      "loss": 2.769174337387085,
+      "mean_token_accuracy": 0.5351328551769257,
+      "num_tokens": 27692.0,
+      "step": 13
+    },
+    {
+      "entropy": 2.544685661792755,
+      "epoch": 0.19718309859154928,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.4870781898498535,
+      "mean_token_accuracy": 0.514883853495121,
+      "num_tokens": 30217.0,
+      "step": 14
+    },
+    {
+      "entropy": 2.5978153944015503,
+      "epoch": 0.2112676056338028,
+      "grad_norm": 10.875,
+      "learning_rate": 5e-05,
+      "loss": 2.639450788497925,
+      "mean_token_accuracy": 0.5468774288892746,
+      "num_tokens": 31969.0,
+      "step": 15
+    },
+    {
+      "entropy": 2.452792704105377,
+      "epoch": 0.22535211267605634,
+      "grad_norm": 11.5,
+      "learning_rate": 5e-05,
+      "loss": 2.5285415649414062,
+      "mean_token_accuracy": 0.5533104464411736,
+      "num_tokens": 34160.0,
+      "step": 16
+    },
+    {
+      "entropy": 2.234907388687134,
+      "epoch": 0.23943661971830985,
+      "grad_norm": 12.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.2553577423095703,
+      "mean_token_accuracy": 0.5958019495010376,
+      "num_tokens": 35899.0,
+      "step": 17
+    },
+    {
+      "entropy": 2.2781192660331726,
+      "epoch": 0.2535211267605634,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4401230812072754,
+      "mean_token_accuracy": 0.5632342845201492,
+      "num_tokens": 37693.0,
+      "step": 18
+    },
+    {
+      "entropy": 2.292062520980835,
+      "epoch": 0.2676056338028169,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 2.53348445892334,
+      "mean_token_accuracy": 0.5441952645778656,
+      "num_tokens": 39793.0,
+      "step": 19
+    },
+    {
+      "entropy": 2.5250023007392883,
+      "epoch": 0.28169014084507044,
+      "grad_norm": 13.5,
+      "learning_rate": 5e-05,
+      "loss": 2.7173304557800293,
+      "mean_token_accuracy": 0.4843936040997505,
+      "num_tokens": 42035.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.3936269879341125,
+      "epoch": 0.29577464788732394,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.397752046585083,
+      "mean_token_accuracy": 0.5632070526480675,
+      "num_tokens": 43606.0,
+      "step": 21
+    },
+    {
+      "entropy": 2.7215799689292908,
+      "epoch": 0.30985915492957744,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.7266643047332764,
+      "mean_token_accuracy": 0.4814189076423645,
+      "num_tokens": 46160.0,
+      "step": 22
+    },
+    {
+      "entropy": 2.4695653319358826,
+      "epoch": 0.323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3790149688720703,
+      "mean_token_accuracy": 0.5413414835929871,
+      "num_tokens": 48749.0,
+      "step": 23
+    },
+    {
+      "entropy": 2.3166028261184692,
+      "epoch": 0.3380281690140845,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.274803638458252,
+      "mean_token_accuracy": 0.5993590205907822,
+      "num_tokens": 50770.0,
+      "step": 24
+    },
+    {
+      "entropy": 2.285377472639084,
+      "epoch": 0.352112676056338,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.2308878898620605,
+      "mean_token_accuracy": 0.5828141123056412,
+      "num_tokens": 53004.0,
+      "step": 25
+    },
+    {
+      "entropy": 2.1157060265541077,
+      "epoch": 0.36619718309859156,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1033847332000732,
+      "mean_token_accuracy": 0.5867631435394287,
+      "num_tokens": 55108.0,
+      "step": 26
+    },
+    {
+      "entropy": 2.5209829807281494,
+      "epoch": 0.38028169014084506,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.3607499599456787,
+      "mean_token_accuracy": 0.5426046550273895,
+      "num_tokens": 57080.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.974183738231659,
+      "epoch": 0.39436619718309857,
+      "grad_norm": 7.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.9418365955352783,
+      "mean_token_accuracy": 0.6255736351013184,
+      "num_tokens": 59237.0,
+      "step": 28
+    },
+    {
+      "entropy": 2.252374231815338,
+      "epoch": 0.4084507042253521,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 2.228358507156372,
+      "mean_token_accuracy": 0.5917280316352844,
+      "num_tokens": 60850.0,
+      "step": 29
+    },
+    {
+      "entropy": 2.327530801296234,
+      "epoch": 0.4225352112676056,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.3329389095306396,
+      "mean_token_accuracy": 0.5718153715133667,
+      "num_tokens": 62734.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.2620502710342407,
+      "epoch": 0.43661971830985913,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1953892707824707,
+      "mean_token_accuracy": 0.5675001591444016,
+      "num_tokens": 64637.0,
+      "step": 31
+    },
+    {
+      "entropy": 2.445267140865326,
+      "epoch": 0.4507042253521127,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.501152276992798,
+      "mean_token_accuracy": 0.5355894565582275,
+      "num_tokens": 66880.0,
+      "step": 32
+    },
+    {
+      "entropy": 2.3873718976974487,
+      "epoch": 0.4647887323943662,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3468880653381348,
+      "mean_token_accuracy": 0.5372899621725082,
+      "num_tokens": 68623.0,
+      "step": 33
+    },
+    {
+      "entropy": 2.0545941591262817,
+      "epoch": 0.4788732394366197,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.1947593688964844,
+      "mean_token_accuracy": 0.6112199202179909,
+      "num_tokens": 70769.0,
+      "step": 34
+    },
+    {
+      "entropy": 2.350032150745392,
+      "epoch": 0.49295774647887325,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.418266773223877,
+      "mean_token_accuracy": 0.5104246437549591,
+      "num_tokens": 73653.0,
+      "step": 35
+    },
+    {
+      "entropy": 2.237062931060791,
+      "epoch": 0.5070422535211268,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4755356311798096,
+      "mean_token_accuracy": 0.5747656747698784,
+      "num_tokens": 75524.0,
+      "step": 36
+    },
+    {
+      "entropy": 2.0994813442230225,
+      "epoch": 0.5211267605633803,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.044315814971924,
+      "mean_token_accuracy": 0.6056438237428665,
+      "num_tokens": 77246.0,
+      "step": 37
+    },
+    {
+      "entropy": 2.5556156635284424,
+      "epoch": 0.5352112676056338,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.446904182434082,
+      "mean_token_accuracy": 0.5177429020404816,
+      "num_tokens": 79970.0,
+      "step": 38
+    },
+    {
+      "entropy": 2.337540030479431,
+      "epoch": 0.5492957746478874,
+      "grad_norm": 8.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.216331720352173,
+      "mean_token_accuracy": 0.58082115650177,
+      "num_tokens": 81876.0,
+      "step": 39
+    },
+    {
+      "entropy": 2.043498247861862,
+      "epoch": 0.5633802816901409,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9643354415893555,
+      "mean_token_accuracy": 0.607293888926506,
+      "num_tokens": 84298.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.4852016270160675,
+      "epoch": 0.5774647887323944,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.571437358856201,
+      "mean_token_accuracy": 0.5233406946063042,
+      "num_tokens": 86291.0,
+      "step": 41
+    },
+    {
+      "entropy": 2.5456899404525757,
+      "epoch": 0.5915492957746479,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 2.4434070587158203,
+      "mean_token_accuracy": 0.5472028106451035,
+      "num_tokens": 88194.0,
+      "step": 42
+    },
+    {
+      "entropy": 2.2191686630249023,
+      "epoch": 0.6056338028169014,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.225029468536377,
+      "mean_token_accuracy": 0.6006355881690979,
+      "num_tokens": 89922.0,
+      "step": 43
+    },
+    {
+      "entropy": 2.2295450568199158,
+      "epoch": 0.6197183098591549,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4000232219696045,
+      "mean_token_accuracy": 0.580116793513298,
+      "num_tokens": 92171.0,
+      "step": 44
+    },
+    {
+      "entropy": 2.2142042219638824,
+      "epoch": 0.6338028169014085,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2619423866271973,
+      "mean_token_accuracy": 0.5818447023630142,
+      "num_tokens": 94362.0,
+      "step": 45
+    },
+    {
+      "entropy": 2.3695861101150513,
+      "epoch": 0.647887323943662,
+      "grad_norm": 7.21875,
+      "learning_rate": 5e-05,
+      "loss": 2.4540324211120605,
+      "mean_token_accuracy": 0.5298221409320831,
+      "num_tokens": 96559.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.9680090248584747,
+      "epoch": 0.6619718309859155,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.9608612060546875,
+      "mean_token_accuracy": 0.6151116490364075,
+      "num_tokens": 99018.0,
+      "step": 47
+    },
+    {
+      "entropy": 2.449136793613434,
+      "epoch": 0.676056338028169,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.364119529724121,
+      "mean_token_accuracy": 0.5657317340373993,
+      "num_tokens": 100885.0,
+      "step": 48
+    },
+    {
+      "entropy": 2.109535813331604,
+      "epoch": 0.6901408450704225,
+      "grad_norm": 6.875,
+      "learning_rate": 5e-05,
+      "loss": 2.1192779541015625,
+      "mean_token_accuracy": 0.5970411896705627,
+      "num_tokens": 103357.0,
+      "step": 49
+    },
+    {
+      "entropy": 2.3080650568008423,
+      "epoch": 0.704225352112676,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.351897716522217,
+      "mean_token_accuracy": 0.5307685062289238,
+      "num_tokens": 105870.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.1463415920734406,
+      "epoch": 0.7183098591549296,
+      "grad_norm": 7.5,
+      "learning_rate": 5e-05,
+      "loss": 2.2693376541137695,
+      "mean_token_accuracy": 0.5934874191880226,
+      "num_tokens": 107995.0,
+      "step": 51
+    },
+    {
+      "entropy": 2.1181687712669373,
+      "epoch": 0.7323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.1629841327667236,
+      "mean_token_accuracy": 0.6063478291034698,
+      "num_tokens": 110358.0,
+      "step": 52
+    },
+    {
+      "entropy": 2.0169546008110046,
+      "epoch": 0.7464788732394366,
+      "grad_norm": 7.78125,
+      "learning_rate": 5e-05,
+      "loss": 1.817868709564209,
+      "mean_token_accuracy": 0.6273332387208939,
+      "num_tokens": 112267.0,
+      "step": 53
+    },
+    {
+      "entropy": 2.189443975687027,
+      "epoch": 0.7605633802816901,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.3292086124420166,
+      "mean_token_accuracy": 0.6041209101676941,
+      "num_tokens": 113899.0,
+      "step": 54
+    },
+    {
+      "entropy": 2.1946000456809998,
+      "epoch": 0.7746478873239436,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2774949073791504,
+      "mean_token_accuracy": 0.5620942413806915,
+      "num_tokens": 116015.0,
+      "step": 55
+    },
+    {
+      "entropy": 2.1172711849212646,
+      "epoch": 0.7887323943661971,
+      "grad_norm": 7.90625,
+      "learning_rate": 5e-05,
+      "loss": 2.1236391067504883,
+      "mean_token_accuracy": 0.5911316424608231,
+      "num_tokens": 118044.0,
+      "step": 56
+    },
+    {
+      "entropy": 2.144031971693039,
+      "epoch": 0.8028169014084507,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.187140703201294,
+      "mean_token_accuracy": 0.5624879449605942,
+      "num_tokens": 120180.0,
+      "step": 57
+    },
+    {
+      "entropy": 2.181178390979767,
+      "epoch": 0.8169014084507042,
+      "grad_norm": 10.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.113279104232788,
+      "mean_token_accuracy": 0.6004882454872131,
+      "num_tokens": 121676.0,
+      "step": 58
+    },
+    {
+      "entropy": 2.314671218395233,
+      "epoch": 0.8309859154929577,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.331024169921875,
+      "mean_token_accuracy": 0.5824068263173103,
+      "num_tokens": 123161.0,
+      "step": 59
+    },
+    {
+      "entropy": 2.0048855245113373,
+      "epoch": 0.8450704225352113,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.070265054702759,
+      "mean_token_accuracy": 0.6141902804374695,
+      "num_tokens": 125016.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.966452568769455,
+      "epoch": 0.8591549295774648,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.9488521814346313,
+      "mean_token_accuracy": 0.5854089111089706,
+      "num_tokens": 127349.0,
+      "step": 61
+    },
+    {
+      "entropy": 2.1519170999526978,
+      "epoch": 0.8732394366197183,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.1743807792663574,
+      "mean_token_accuracy": 0.5636171847581863,
+      "num_tokens": 129742.0,
+      "step": 62
+    },
+    {
+      "entropy": 2.7546687722206116,
+      "epoch": 0.8873239436619719,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.7343056201934814,
+      "mean_token_accuracy": 0.496803916990757,
+      "num_tokens": 131743.0,
+      "step": 63
+    },
+    {
+      "entropy": 2.421773910522461,
+      "epoch": 0.9014084507042254,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.6149866580963135,
+      "mean_token_accuracy": 0.5249136164784431,
+      "num_tokens": 134002.0,
+      "step": 64
+    },
+    {
+      "entropy": 2.0106727480888367,
+      "epoch": 0.9154929577464789,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.940914273262024,
+      "mean_token_accuracy": 0.5855439230799675,
+      "num_tokens": 136120.0,
+      "step": 65
+    },
+    {
+      "entropy": 2.1143648624420166,
+      "epoch": 0.9295774647887324,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.2469482421875,
+      "mean_token_accuracy": 0.6145937740802765,
+      "num_tokens": 137997.0,
+      "step": 66
+    },
+    {
+      "entropy": 2.1894670128822327,
+      "epoch": 0.9436619718309859,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.20756459236145,
+      "mean_token_accuracy": 0.5872255340218544,
+      "num_tokens": 139904.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.75501748919487,
+      "epoch": 0.9577464788732394,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.9068934917449951,
+      "mean_token_accuracy": 0.6642919778823853,
+      "num_tokens": 141166.0,
+      "step": 68
+    },
+    {
+      "entropy": 2.204477220773697,
+      "epoch": 0.971830985915493,
+      "grad_norm": 7.875,
+      "learning_rate": 5e-05,
+      "loss": 2.286216974258423,
+      "mean_token_accuracy": 0.5354761853814125,
+      "num_tokens": 143738.0,
+      "step": 69
+    },
+    {
+      "entropy": 2.1219914853572845,
+      "epoch": 0.9859154929577465,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 2.274585723876953,
+      "mean_token_accuracy": 0.609759122133255,
+      "num_tokens": 145439.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.9260511100292206,
+      "epoch": 1.0,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.8763841390609741,
+      "mean_token_accuracy": 0.5979989916086197,
+      "num_tokens": 147650.0,
+      "step": 71
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 2.1684498711371085,
+      "eval_loss": 2.284580707550049,
+      "eval_mean_token_accuracy": 0.5757196218195096,
+      "eval_num_tokens": 147650.0,
+      "eval_runtime": 18.8201,
+      "eval_samples_per_second": 15.09,
+      "eval_steps_per_second": 3.773,
+      "step": 71
+    },
+    {
+      "entropy": 2.24437940120697,
+      "epoch": 1.0140845070422535,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 2.148402214050293,
+      "mean_token_accuracy": 0.5866604596376419,
+      "num_tokens": 149372.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.630838692188263,
+      "epoch": 1.028169014084507,
+      "grad_norm": 7.0,
+      "learning_rate": 5e-05,
+      "loss": 1.526665449142456,
+      "mean_token_accuracy": 0.6816482096910477,
+      "num_tokens": 151274.0,
+      "step": 73
+    },
+    {
+      "entropy": 2.193234443664551,
+      "epoch": 1.0422535211267605,
+      "grad_norm": 7.90625,
+      "learning_rate": 5e-05,
+      "loss": 2.176100969314575,
+      "mean_token_accuracy": 0.6048732623457909,
+      "num_tokens": 153103.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.7758266925811768,
+      "epoch": 1.056338028169014,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.7472716569900513,
+      "mean_token_accuracy": 0.6403823792934418,
+      "num_tokens": 155291.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.8887377083301544,
+      "epoch": 1.0704225352112675,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.872934341430664,
+      "mean_token_accuracy": 0.6085800677537918,
+      "num_tokens": 157847.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.6055135428905487,
+      "epoch": 1.084507042253521,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.6859486103057861,
+      "mean_token_accuracy": 0.6478956639766693,
+      "num_tokens": 160538.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.9197329580783844,
+      "epoch": 1.0985915492957747,
+      "grad_norm": 7.375,
+      "learning_rate": 5e-05,
+      "loss": 1.929700493812561,
+      "mean_token_accuracy": 0.6054274141788483,
+      "num_tokens": 162709.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.6682755053043365,
+      "epoch": 1.1126760563380282,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.899672269821167,
+      "mean_token_accuracy": 0.6613650918006897,
+      "num_tokens": 164478.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.8098062872886658,
+      "epoch": 1.1267605633802817,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7989305257797241,
+      "mean_token_accuracy": 0.6530113369226456,
+      "num_tokens": 165767.0,
+      "step": 80
+    },
+    {
+      "entropy": 2.3263643383979797,
+      "epoch": 1.1408450704225352,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.32468843460083,
+      "mean_token_accuracy": 0.5657684504985809,
+      "num_tokens": 167374.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.6652490496635437,
+      "epoch": 1.1549295774647887,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.821378469467163,
+      "mean_token_accuracy": 0.6479186862707138,
+      "num_tokens": 169216.0,
+      "step": 82
+    },
+    {
+      "entropy": 2.157005727291107,
+      "epoch": 1.1690140845070423,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.1137959957122803,
+      "mean_token_accuracy": 0.5775181204080582,
+      "num_tokens": 171194.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.6896095722913742,
+      "epoch": 1.1830985915492958,
+      "grad_norm": 8.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.785524606704712,
+      "mean_token_accuracy": 0.6791908890008926,
+      "num_tokens": 173018.0,
+      "step": 84
+    },
+    {
+      "entropy": 2.241276800632477,
+      "epoch": 1.1971830985915493,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.1982414722442627,
+      "mean_token_accuracy": 0.5830243527889252,
+      "num_tokens": 174822.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.8407301902770996,
+      "epoch": 1.2112676056338028,
+      "grad_norm": 7.65625,
+      "learning_rate": 5e-05,
+      "loss": 1.752529501914978,
+      "mean_token_accuracy": 0.6172220706939697,
+      "num_tokens": 176932.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.707479178905487,
+      "epoch": 1.2253521126760563,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.6565136909484863,
+      "mean_token_accuracy": 0.6619749516248703,
+      "num_tokens": 178804.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.6418649852275848,
+      "epoch": 1.2394366197183098,
+      "grad_norm": 7.15625,
+      "learning_rate": 5e-05,
+      "loss": 1.5298699140548706,
+      "mean_token_accuracy": 0.6645439267158508,
+      "num_tokens": 181353.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.6070191860198975,
+      "epoch": 1.2535211267605635,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.7814466953277588,
+      "mean_token_accuracy": 0.6640422344207764,
+      "num_tokens": 183278.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.8768837749958038,
+      "epoch": 1.267605633802817,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9979424476623535,
+      "mean_token_accuracy": 0.6454368084669113,
+      "num_tokens": 185041.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.809422641992569,
+      "epoch": 1.2816901408450705,
+      "grad_norm": 7.78125,
+      "learning_rate": 5e-05,
+      "loss": 1.8236162662506104,
+      "mean_token_accuracy": 0.6187984794378281,
+      "num_tokens": 187646.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.8032868206501007,
+      "epoch": 1.295774647887324,
+      "grad_norm": 7.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.8008391857147217,
+      "mean_token_accuracy": 0.6168265342712402,
+      "num_tokens": 189952.0,
+      "step": 92
+    },
+    {
+      "entropy": 2.107921302318573,
+      "epoch": 1.3098591549295775,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 2.077434539794922,
+      "mean_token_accuracy": 0.5662726014852524,
+      "num_tokens": 192592.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.6842139065265656,
+      "epoch": 1.323943661971831,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.6454025506973267,
+      "mean_token_accuracy": 0.6700995117425919,
+      "num_tokens": 194549.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.859749436378479,
+      "epoch": 1.3380281690140845,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9091846942901611,
+      "mean_token_accuracy": 0.6098746657371521,
+      "num_tokens": 196448.0,
+      "step": 95
+    },
+    {
+      "entropy": 2.018050581216812,
+      "epoch": 1.352112676056338,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.1190457344055176,
+      "mean_token_accuracy": 0.5959457382559776,
+      "num_tokens": 198615.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.9890158474445343,
+      "epoch": 1.3661971830985915,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.960221529006958,
+      "mean_token_accuracy": 0.5875463634729385,
+      "num_tokens": 201013.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.8716930747032166,
+      "epoch": 1.380281690140845,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 2.0004959106445312,
+      "mean_token_accuracy": 0.6175930351018906,
+      "num_tokens": 202969.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.7610785365104675,
+      "epoch": 1.3943661971830985,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.6850508451461792,
+      "mean_token_accuracy": 0.6504905521869659,
+      "num_tokens": 204906.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.9090750515460968,
+      "epoch": 1.408450704225352,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.938533067703247,
+      "mean_token_accuracy": 0.621729701757431,
+      "num_tokens": 206869.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.8917019963264465,
+      "epoch": 1.4225352112676055,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.0368175506591797,
+      "mean_token_accuracy": 0.6267035603523254,
+      "num_tokens": 208432.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.7962620854377747,
+      "epoch": 1.436619718309859,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.8455748558044434,
+      "mean_token_accuracy": 0.608839362859726,
+      "num_tokens": 210949.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.724296659231186,
+      "epoch": 1.4507042253521127,
+      "grad_norm": 10.25,
+      "learning_rate": 5e-05,
+      "loss": 1.7760952711105347,
+      "mean_token_accuracy": 0.6573535352945328,
+      "num_tokens": 212595.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.8014512360095978,
+      "epoch": 1.4647887323943662,
+      "grad_norm": 7.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.8614972829818726,
+      "mean_token_accuracy": 0.5946371257305145,
+      "num_tokens": 215384.0,
+      "step": 104
+    },
+    {
+      "entropy": 1.8427681922912598,
+      "epoch": 1.4788732394366197,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 1.880817174911499,
+      "mean_token_accuracy": 0.6408448368310928,
+      "num_tokens": 217561.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.6576282978057861,
+      "epoch": 1.4929577464788732,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.6814919710159302,
+      "mean_token_accuracy": 0.6402300000190735,
+      "num_tokens": 219087.0,
+      "step": 106
+    },
+    {
+      "entropy": 1.5478721261024475,
+      "epoch": 1.5070422535211268,
+      "grad_norm": 7.59375,
+      "learning_rate": 5e-05,
+      "loss": 1.5617148876190186,
+      "mean_token_accuracy": 0.6625793874263763,
+      "num_tokens": 221887.0,
+      "step": 107
+    },
+    {
+      "entropy": 1.923494279384613,
+      "epoch": 1.5211267605633803,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.0690817832946777,
+      "mean_token_accuracy": 0.5978109538555145,
+      "num_tokens": 224182.0,
+      "step": 108
+    },
+    {
+      "entropy": 1.8152073919773102,
+      "epoch": 1.5352112676056338,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 1.7398602962493896,
+      "mean_token_accuracy": 0.6207661181688309,
+      "num_tokens": 226591.0,
+      "step": 109
+    },
+    {
+      "entropy": 2.1172706484794617,
+      "epoch": 1.5492957746478875,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.237187385559082,
+      "mean_token_accuracy": 0.5684898793697357,
+      "num_tokens": 228529.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.686636283993721,
+      "epoch": 1.563380281690141,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.8167091608047485,
+      "mean_token_accuracy": 0.6692909449338913,
+      "num_tokens": 229973.0,
+      "step": 111
+    },
+    {
+      "entropy": 2.285969853401184,
+      "epoch": 1.5774647887323945,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 2.308091878890991,
+      "mean_token_accuracy": 0.5341236144304276,
+      "num_tokens": 232232.0,
+      "step": 112
+    },
+    {
+      "entropy": 1.7772781550884247,
+      "epoch": 1.591549295774648,
+      "grad_norm": 7.09375,
+      "learning_rate": 5e-05,
+      "loss": 1.7048449516296387,
+      "mean_token_accuracy": 0.6309333443641663,
+      "num_tokens": 234998.0,
+      "step": 113
+    },
+    {
+      "entropy": 1.8632723987102509,
+      "epoch": 1.6056338028169015,
+      "grad_norm": 6.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.8152518272399902,
+      "mean_token_accuracy": 0.5958341658115387,
+      "num_tokens": 237842.0,
+      "step": 114
+    },
+    {
+      "entropy": 1.7296560108661652,
+      "epoch": 1.619718309859155,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.7719354629516602,
+      "mean_token_accuracy": 0.6910708993673325,
+      "num_tokens": 239068.0,
+      "step": 115
+    },
+    {
+      "entropy": 2.026154547929764,
+      "epoch": 1.6338028169014085,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 2.067565441131592,
+      "mean_token_accuracy": 0.5664031803607941,
+      "num_tokens": 241497.0,
+      "step": 116
+    },
+    {
+      "entropy": 1.6592170596122742,
+      "epoch": 1.647887323943662,
+      "grad_norm": 7.65625,
+      "learning_rate": 5e-05,
+      "loss": 1.7524383068084717,
+      "mean_token_accuracy": 0.6437603831291199,
+      "num_tokens": 243721.0,
+      "step": 117
+    },
+    {
+      "entropy": 1.8260776698589325,
+      "epoch": 1.6619718309859155,
+      "grad_norm": 8.0,
+      "learning_rate": 5e-05,
+      "loss": 1.7933809757232666,
+      "mean_token_accuracy": 0.6192324906587601,
+      "num_tokens": 246193.0,
+      "step": 118
+    },
+    {
+      "entropy": 1.6278434097766876,
+      "epoch": 1.676056338028169,
+      "grad_norm": 7.875,
+      "learning_rate": 5e-05,
+      "loss": 1.6023352146148682,
+      "mean_token_accuracy": 0.6632742285728455,
+      "num_tokens": 248065.0,
+      "step": 119
+    },
+    {
+      "entropy": 1.5735027194023132,
+      "epoch": 1.6901408450704225,
+      "grad_norm": 7.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.5997726917266846,
+      "mean_token_accuracy": 0.6820999830961227,
+      "num_tokens": 250126.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.7936385571956635,
+      "epoch": 1.704225352112676,
+      "grad_norm": 7.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.662062644958496,
+      "mean_token_accuracy": 0.6343215107917786,
+      "num_tokens": 252242.0,
+      "step": 121
+    },
+    {
+      "entropy": 2.0255846083164215,
+      "epoch": 1.7183098591549295,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.9866116046905518,
+      "mean_token_accuracy": 0.5996693223714828,
+      "num_tokens": 254704.0,
+      "step": 122
+    },
+    {
+      "entropy": 1.8428587317466736,
+      "epoch": 1.732394366197183,
+      "grad_norm": 7.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.786312460899353,
+      "mean_token_accuracy": 0.6079171597957611,
+      "num_tokens": 257154.0,
+      "step": 123
+    },
+    {
+      "entropy": 1.8701068460941315,
+      "epoch": 1.7464788732394365,
+      "grad_norm": 6.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.814254641532898,
+      "mean_token_accuracy": 0.6146602630615234,
+      "num_tokens": 260050.0,
+      "step": 124
+    },
+    {
+      "entropy": 1.7365414798259735,
+      "epoch": 1.76056338028169,
+      "grad_norm": 6.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.6392512321472168,
+      "mean_token_accuracy": 0.631788820028305,
+      "num_tokens": 262731.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.5750393569469452,
+      "epoch": 1.7746478873239435,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.652256727218628,
+      "mean_token_accuracy": 0.6727721691131592,
+      "num_tokens": 264264.0,
+      "step": 126
+    },
+    {
+      "entropy": 1.6853447258472443,
+      "epoch": 1.788732394366197,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.8110965490341187,
+      "mean_token_accuracy": 0.6612876951694489,
+      "num_tokens": 265839.0,
+      "step": 127
+    },
+    {
+      "entropy": 1.9790962934494019,
+      "epoch": 1.8028169014084507,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.9432560205459595,
+      "mean_token_accuracy": 0.5927930474281311,
+      "num_tokens": 268531.0,
+      "step": 128
+    },
+    {
+      "entropy": 1.6438206434249878,
+      "epoch": 1.8169014084507042,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 2.005178689956665,
+      "mean_token_accuracy": 0.6584273278713226,
+      "num_tokens": 270295.0,
+      "step": 129
+    },
+    {
+      "entropy": 1.8002346009016037,
+      "epoch": 1.8309859154929577,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.9875435829162598,
+      "mean_token_accuracy": 0.6358410120010376,
+      "num_tokens": 272470.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.811629831790924,
+      "epoch": 1.8450704225352113,
+      "grad_norm": 9.25,
+      "learning_rate": 5e-05,
+      "loss": 1.896381139755249,
+      "mean_token_accuracy": 0.6606249660253525,
+      "num_tokens": 274166.0,
+      "step": 131
+    },
+    {
+      "entropy": 1.7871823608875275,
+      "epoch": 1.8591549295774648,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7262006998062134,
+      "mean_token_accuracy": 0.6428729891777039,
+      "num_tokens": 276354.0,
+      "step": 132
+    },
+    {
+      "entropy": 1.5866018235683441,
+      "epoch": 1.8732394366197183,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.54076087474823,
+      "mean_token_accuracy": 0.6837680488824844,
+      "num_tokens": 278044.0,
+      "step": 133
+    },
+    {
+      "entropy": 1.9443781971931458,
+      "epoch": 1.887323943661972,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.935890555381775,
+      "mean_token_accuracy": 0.5984784960746765,
+      "num_tokens": 279727.0,
+      "step": 134
+    },
+    {
+      "entropy": 1.6744503676891327,
+      "epoch": 1.9014084507042255,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.667353868484497,
+      "mean_token_accuracy": 0.6531110107898712,
+      "num_tokens": 281727.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.5089002698659897,
+      "epoch": 1.915492957746479,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.7763428688049316,
+      "mean_token_accuracy": 0.6703489422798157,
+      "num_tokens": 283499.0,
+      "step": 136
+    },
+    {
+      "entropy": 1.7822388112545013,
+      "epoch": 1.9295774647887325,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.7652621269226074,
+      "mean_token_accuracy": 0.6092153340578079,
+      "num_tokens": 286098.0,
+      "step": 137
+    },
+    {
+      "entropy": 1.633983552455902,
+      "epoch": 1.943661971830986,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7642450332641602,
+      "mean_token_accuracy": 0.6580131351947784,
+      "num_tokens": 288020.0,
+      "step": 138
+    },
+    {
+      "entropy": 1.9797956943511963,
+      "epoch": 1.9577464788732395,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.8309262990951538,
+      "mean_token_accuracy": 0.6048336029052734,
+      "num_tokens": 290401.0,
+      "step": 139
+    },
+    {
+      "entropy": 1.5087556838989258,
+      "epoch": 1.971830985915493,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.6536986827850342,
+      "mean_token_accuracy": 0.7231832593679428,
+      "num_tokens": 291584.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.9238645434379578,
+      "epoch": 1.9859154929577465,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.7226345539093018,
+      "mean_token_accuracy": 0.6254735887050629,
+      "num_tokens": 293607.0,
+      "step": 141
+    },
+    {
+      "entropy": 1.587816596031189,
+      "epoch": 2.0,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.5887134075164795,
+      "mean_token_accuracy": 0.6791131645441055,
+      "num_tokens": 295300.0,
+      "step": 142
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 1.972671693479511,
+      "eval_loss": 2.2974655628204346,
+      "eval_mean_token_accuracy": 0.5787080614499642,
+      "eval_num_tokens": 295300.0,
+      "eval_runtime": 18.8723,
+      "eval_samples_per_second": 15.049,
+      "eval_steps_per_second": 3.762,
+      "step": 142
+    },
+    {
+      "entropy": 1.9815530478954315,
+      "epoch": 2.0140845070422535,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.6473958492279053,
+      "mean_token_accuracy": 0.6364074051380157,
+      "num_tokens": 297724.0,
+      "step": 143
+    },
+    {
+      "entropy": 1.641160249710083,
+      "epoch": 2.028169014084507,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.4365612268447876,
+      "mean_token_accuracy": 0.678604006767273,
+      "num_tokens": 300066.0,
+      "step": 144
+    },
+    {
+      "entropy": 1.4447078704833984,
+      "epoch": 2.0422535211267605,
+      "grad_norm": 7.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4016954898834229,
+      "mean_token_accuracy": 0.6932056993246078,
+      "num_tokens": 302321.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.5544451773166656,
+      "epoch": 2.056338028169014,
+      "grad_norm": 9.25,
+      "learning_rate": 5e-05,
+      "loss": 1.5597270727157593,
+      "mean_token_accuracy": 0.6651838272809982,
+      "num_tokens": 304496.0,
+      "step": 146
+    },
+    {
+      "entropy": 1.85980024933815,
+      "epoch": 2.0704225352112675,
+      "grad_norm": 9.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.8932693004608154,
+      "mean_token_accuracy": 0.6070298254489899,
+      "num_tokens": 306612.0,
+      "step": 147
+    },
+    {
+      "entropy": 1.6910157799720764,
+      "epoch": 2.084507042253521,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.7285449504852295,
+      "mean_token_accuracy": 0.6408755928277969,
+      "num_tokens": 308440.0,
+      "step": 148
+    },
+    {
+      "entropy": 1.367365151643753,
+      "epoch": 2.0985915492957745,
+      "grad_norm": 9.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.4051942825317383,
+      "mean_token_accuracy": 0.6981760710477829,
+      "num_tokens": 310467.0,
+      "step": 149
+    },
+    {
+      "entropy": 1.3810956180095673,
+      "epoch": 2.112676056338028,
+      "grad_norm": 11.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4497928619384766,
+      "mean_token_accuracy": 0.6659071892499924,
+      "num_tokens": 313093.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.6426885724067688,
+      "epoch": 2.1267605633802815,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.730147123336792,
+      "mean_token_accuracy": 0.636825755238533,
+      "num_tokens": 315604.0,
+      "step": 151
+    },
+    {
+      "entropy": 1.5957364737987518,
+      "epoch": 2.140845070422535,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.6714507341384888,
+      "mean_token_accuracy": 0.6542615443468094,
+      "num_tokens": 317613.0,
+      "step": 152
+    },
+    {
+      "entropy": 2.0231593549251556,
+      "epoch": 2.1549295774647885,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.9902608394622803,
+      "mean_token_accuracy": 0.6068925857543945,
+      "num_tokens": 318980.0,
+      "step": 153
+    },
+    {
+      "entropy": 1.5751276016235352,
+      "epoch": 2.169014084507042,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.5008318424224854,
+      "mean_token_accuracy": 0.6704117655754089,
+      "num_tokens": 320981.0,
+      "step": 154
+    },
+    {
+      "entropy": 1.611257165670395,
+      "epoch": 2.183098591549296,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.5167430639266968,
+      "mean_token_accuracy": 0.6731022894382477,
+      "num_tokens": 323159.0,
+      "step": 155
+    },
+    {
+      "entropy": 1.5024433732032776,
+      "epoch": 2.1971830985915495,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.441955804824829,
+      "mean_token_accuracy": 0.6796386241912842,
+      "num_tokens": 325305.0,
+      "step": 156
+    },
+    {
+      "entropy": 1.7239812314510345,
+      "epoch": 2.211267605633803,
+      "grad_norm": 12.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.6520541906356812,
+      "mean_token_accuracy": 0.6401138156652451,
+      "num_tokens": 327629.0,
+      "step": 157
+    },
+    {
+      "entropy": 1.7822472155094147,
+      "epoch": 2.2253521126760565,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.6968837976455688,
+      "mean_token_accuracy": 0.6756918206810951,
+      "num_tokens": 328919.0,
+      "step": 158
+    },
+    {
+      "entropy": 1.3915225267410278,
+      "epoch": 2.23943661971831,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.410017490386963,
+      "mean_token_accuracy": 0.6763953566551208,
+      "num_tokens": 331333.0,
+      "step": 159
+    },
+    {
+      "entropy": 1.502710074186325,
+      "epoch": 2.2535211267605635,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.4881656169891357,
+      "mean_token_accuracy": 0.6672618389129639,
+      "num_tokens": 332856.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.9380434155464172,
+      "epoch": 2.267605633802817,
+      "grad_norm": 11.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.9208719730377197,
+      "mean_token_accuracy": 0.624039888381958,
+      "num_tokens": 334373.0,
+      "step": 161
+    },
+    {
+      "entropy": 1.6169701665639877,
+      "epoch": 2.2816901408450705,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.8250868320465088,
+      "mean_token_accuracy": 0.6674070954322815,
+      "num_tokens": 336125.0,
+      "step": 162
+    },
+    {
+      "entropy": 1.7778233587741852,
+      "epoch": 2.295774647887324,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.7127352952957153,
+      "mean_token_accuracy": 0.6442407220602036,
+      "num_tokens": 337666.0,
+      "step": 163
+    },
+    {
+      "entropy": 1.5933920741081238,
+      "epoch": 2.3098591549295775,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.615399956703186,
+      "mean_token_accuracy": 0.6445006728172302,
+      "num_tokens": 340298.0,
+      "step": 164
+    },
+    {
+      "entropy": 1.774703323841095,
+      "epoch": 2.323943661971831,
+      "grad_norm": 10.375,
+      "learning_rate": 5e-05,
+      "loss": 1.6529101133346558,
+      "mean_token_accuracy": 0.6362189799547195,
+      "num_tokens": 342187.0,
+      "step": 165
+    },
+    {
+      "entropy": 1.2685332596302032,
+      "epoch": 2.3380281690140845,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.2466249465942383,
+      "mean_token_accuracy": 0.7213118225336075,
+      "num_tokens": 344043.0,
+      "step": 166
+    },
+    {
+      "entropy": 1.5397302210330963,
+      "epoch": 2.352112676056338,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.428548812866211,
+      "mean_token_accuracy": 0.6775327771902084,
+      "num_tokens": 346429.0,
+      "step": 167
+    },
+    {
+      "entropy": 1.6372707784175873,
+      "epoch": 2.3661971830985915,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.587751865386963,
+      "mean_token_accuracy": 0.6544711589813232,
+      "num_tokens": 348838.0,
+      "step": 168
+    },
+    {
+      "entropy": 1.427915170788765,
+      "epoch": 2.380281690140845,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4900782108306885,
+      "mean_token_accuracy": 0.7009869068861008,
+      "num_tokens": 350555.0,
+      "step": 169
+    },
+    {
+      "entropy": 1.374229371547699,
+      "epoch": 2.3943661971830985,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 1.3360053300857544,
+      "mean_token_accuracy": 0.7052380293607712,
+      "num_tokens": 352317.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.3870916664600372,
+      "epoch": 2.408450704225352,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.4269676208496094,
+      "mean_token_accuracy": 0.6933766007423401,
+      "num_tokens": 354187.0,
+      "step": 171
+    },
+    {
+      "entropy": 1.4595828652381897,
+      "epoch": 2.4225352112676055,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.4954618215560913,
+      "mean_token_accuracy": 0.6528928130865097,
+      "num_tokens": 356527.0,
+      "step": 172
+    },
+    {
+      "entropy": 1.5412043631076813,
+      "epoch": 2.436619718309859,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.5568714141845703,
+      "mean_token_accuracy": 0.6727624386548996,
+      "num_tokens": 358804.0,
+      "step": 173
+    },
+    {
+      "entropy": 1.4133048951625824,
+      "epoch": 2.4507042253521125,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4476163387298584,
+      "mean_token_accuracy": 0.6771093457937241,
+      "num_tokens": 361168.0,
+      "step": 174
+    },
+    {
+      "entropy": 1.5175727903842926,
+      "epoch": 2.464788732394366,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 1.6465704441070557,
+      "mean_token_accuracy": 0.6511463969945908,
+      "num_tokens": 363307.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.7925034761428833,
+      "epoch": 2.4788732394366195,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.8684073686599731,
+      "mean_token_accuracy": 0.6113807708024979,
+      "num_tokens": 365159.0,
+      "step": 176
+    },
+    {
+      "entropy": 1.6575580835342407,
+      "epoch": 2.492957746478873,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 1.6698346138000488,
+      "mean_token_accuracy": 0.6580274552106857,
+      "num_tokens": 366815.0,
+      "step": 177
+    },
+    {
+      "entropy": 1.1517049968242645,
+      "epoch": 2.507042253521127,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.0882394313812256,
+      "mean_token_accuracy": 0.7630608826875687,
+      "num_tokens": 368716.0,
+      "step": 178
+    },
+    {
+      "entropy": 1.4394071251153946,
+      "epoch": 2.52112676056338,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.5650489330291748,
+      "mean_token_accuracy": 0.6978088915348053,
+      "num_tokens": 370762.0,
+      "step": 179
+    },
+    {
+      "entropy": 1.329190582036972,
+      "epoch": 2.535211267605634,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 1.339747667312622,
+      "mean_token_accuracy": 0.6991136074066162,
+      "num_tokens": 372772.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.4634642452001572,
+      "epoch": 2.5492957746478875,
+      "grad_norm": 7.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.3990017175674438,
+      "mean_token_accuracy": 0.6824762970209122,
+      "num_tokens": 375270.0,
+      "step": 181
+    },
+    {
+      "entropy": 1.440472275018692,
+      "epoch": 2.563380281690141,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 1.416210412979126,
+      "mean_token_accuracy": 0.7090055644512177,
+      "num_tokens": 377207.0,
+      "step": 182
+    },
+    {
+      "entropy": 1.5881553888320923,
+      "epoch": 2.5774647887323945,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.5490307807922363,
+      "mean_token_accuracy": 0.6557108908891678,
+      "num_tokens": 379285.0,
+      "step": 183
+    },
+    {
+      "entropy": 1.8855535686016083,
+      "epoch": 2.591549295774648,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.8609915971755981,
+      "mean_token_accuracy": 0.6001175194978714,
+      "num_tokens": 381931.0,
+      "step": 184
+    },
+    {
+      "entropy": 1.5735717117786407,
+      "epoch": 2.6056338028169015,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.5267468690872192,
+      "mean_token_accuracy": 0.6682861894369125,
+      "num_tokens": 383646.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.5233647227287292,
+      "epoch": 2.619718309859155,
+      "grad_norm": 10.0,
+      "learning_rate": 5e-05,
+      "loss": 1.5220509767532349,
+      "mean_token_accuracy": 0.6776503473520279,
+      "num_tokens": 385375.0,
+      "step": 186
+    },
+    {
+      "entropy": 1.8002847731113434,
+      "epoch": 2.6338028169014085,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.6597533226013184,
+      "mean_token_accuracy": 0.6361639946699142,
+      "num_tokens": 387083.0,
+      "step": 187
+    },
+    {
+      "entropy": 1.4071598052978516,
+      "epoch": 2.647887323943662,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.3929013013839722,
+      "mean_token_accuracy": 0.6784840375185013,
+      "num_tokens": 389755.0,
+      "step": 188
+    },
+    {
+      "entropy": 1.5969584584236145,
+      "epoch": 2.6619718309859155,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.5413727760314941,
+      "mean_token_accuracy": 0.6579905152320862,
+      "num_tokens": 392159.0,
+      "step": 189
+    },
+    {
+      "entropy": 1.4266707003116608,
+      "epoch": 2.676056338028169,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 1.4889869689941406,
+      "mean_token_accuracy": 0.6940827667713165,
+      "num_tokens": 394318.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.3000743389129639,
+      "epoch": 2.6901408450704225,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.2992713451385498,
+      "mean_token_accuracy": 0.7003718316555023,
+      "num_tokens": 396683.0,
+      "step": 191
+    },
+    {
+      "entropy": 1.454126387834549,
+      "epoch": 2.704225352112676,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 1.5633444786071777,
+      "mean_token_accuracy": 0.674123615026474,
+      "num_tokens": 398548.0,
+      "step": 192
+    },
+    {
+      "entropy": 1.2735390067100525,
+      "epoch": 2.7183098591549295,
+      "grad_norm": 7.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.2833411693572998,
+      "mean_token_accuracy": 0.7046811580657959,
+      "num_tokens": 401214.0,
+      "step": 193
+    },
+    {
+      "entropy": 1.6532493829727173,
+      "epoch": 2.732394366197183,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.6411373615264893,
+      "mean_token_accuracy": 0.6505441665649414,
+      "num_tokens": 403386.0,
+      "step": 194
+    },
+    {
+      "entropy": 1.3735451996326447,
+      "epoch": 2.7464788732394365,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.3553458452224731,
+      "mean_token_accuracy": 0.6837523132562637,
+      "num_tokens": 405696.0,
+      "step": 195
+    },
+    {
+      "entropy": 2.081565737724304,
+      "epoch": 2.76056338028169,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9327874183654785,
+      "mean_token_accuracy": 0.5979610085487366,
+      "num_tokens": 407659.0,
+      "step": 196
+    },
+    {
+      "entropy": 1.9338420629501343,
+      "epoch": 2.7746478873239435,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.8871653079986572,
+      "mean_token_accuracy": 0.5969646275043488,
+      "num_tokens": 410057.0,
+      "step": 197
+    },
+    {
+      "entropy": 1.5678223371505737,
+      "epoch": 2.788732394366197,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.6835041046142578,
+      "mean_token_accuracy": 0.6561465561389923,
+      "num_tokens": 412345.0,
+      "step": 198
+    },
+    {
+      "entropy": 1.349662259221077,
+      "epoch": 2.802816901408451,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.4067068099975586,
+      "mean_token_accuracy": 0.6867142766714096,
+      "num_tokens": 414435.0,
+      "step": 199
+    },
+    {
+      "entropy": 1.4967040419578552,
+      "epoch": 2.816901408450704,
+      "grad_norm": 10.125,
+      "learning_rate": 5e-05,
+      "loss": 1.535017490386963,
+      "mean_token_accuracy": 0.6887214332818985,
+      "num_tokens": 416573.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.580811321735382,
+      "epoch": 2.830985915492958,
+      "grad_norm": 7.625,
+      "learning_rate": 5e-05,
+      "loss": 1.5120457410812378,
+      "mean_token_accuracy": 0.6450829207897186,
+      "num_tokens": 419194.0,
+      "step": 201
+    },
+    {
+      "entropy": 1.5417903065681458,
+      "epoch": 2.845070422535211,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.5814214944839478,
+      "mean_token_accuracy": 0.6863117665052414,
+      "num_tokens": 421333.0,
+      "step": 202
+    },
+    {
+      "entropy": 1.6860767006874084,
+      "epoch": 2.859154929577465,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.757788896560669,
+      "mean_token_accuracy": 0.6364521831274033,
+      "num_tokens": 423578.0,
+      "step": 203
+    },
+    {
+      "entropy": 1.5526399314403534,
+      "epoch": 2.873239436619718,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.5645313262939453,
+      "mean_token_accuracy": 0.660129725933075,
+      "num_tokens": 426228.0,
+      "step": 204
+    },
+    {
+      "entropy": 1.574429988861084,
+      "epoch": 2.887323943661972,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.4116606712341309,
+      "mean_token_accuracy": 0.67585289478302,
+      "num_tokens": 428377.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.5494361817836761,
+      "epoch": 2.9014084507042255,
+      "grad_norm": 10.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.4719582796096802,
+      "mean_token_accuracy": 0.6877067238092422,
+      "num_tokens": 430248.0,
+      "step": 206
+    },
+    {
+      "entropy": 1.4029511511325836,
+      "epoch": 2.915492957746479,
+      "grad_norm": 9.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.3776860237121582,
+      "mean_token_accuracy": 0.6826284527778625,
+      "num_tokens": 432195.0,
+      "step": 207
+    },
+    {
+      "entropy": 1.3329160958528519,
+      "epoch": 2.9295774647887325,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.4112855195999146,
+      "mean_token_accuracy": 0.710347130894661,
+      "num_tokens": 433833.0,
+      "step": 208
+    },
+    {
+      "entropy": 1.3620904386043549,
+      "epoch": 2.943661971830986,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.4375736713409424,
+      "mean_token_accuracy": 0.66610486805439,
+      "num_tokens": 435846.0,
+      "step": 209
+    },
+    {
+      "entropy": 1.3039760887622833,
+      "epoch": 2.9577464788732395,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.4245851039886475,
+      "mean_token_accuracy": 0.6863886713981628,
+      "num_tokens": 437817.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.3174594566226006,
+      "epoch": 2.971830985915493,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.4897089004516602,
+      "mean_token_accuracy": 0.7268126755952835,
+      "num_tokens": 439001.0,
+      "step": 211
+    },
+    {
+      "entropy": 1.597790390253067,
+      "epoch": 2.9859154929577465,
+      "grad_norm": 10.0,
+      "learning_rate": 5e-05,
+      "loss": 1.637603998184204,
+      "mean_token_accuracy": 0.6423007696866989,
+      "num_tokens": 441067.0,
+      "step": 212
+    },
+    {
+      "entropy": 1.4755942821502686,
+      "epoch": 3.0,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.566728115081787,
+      "mean_token_accuracy": 0.6867347657680511,
+      "num_tokens": 442950.0,
+      "step": 213
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 1.742875314934153,
+      "eval_loss": 2.3794198036193848,
+      "eval_mean_token_accuracy": 0.5764347096564064,
+      "eval_num_tokens": 442950.0,
+      "eval_runtime": 18.96,
+      "eval_samples_per_second": 14.979,
+      "eval_steps_per_second": 3.745,
+      "step": 213
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 284,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 461623275251712.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-213/training_args.bin b/checkpoint-213/training_args.bin
new file mode 100644
index 0000000..0b099d6
--- /dev/null
+++ b/checkpoint-213/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
+size 5585
diff --git a/checkpoint-284/chat_template.jinja b/checkpoint-284/chat_template.jinja
new file mode 100644
index 0000000..1117055
--- /dev/null
+++ b/checkpoint-284/chat_template.jinja
@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}
diff --git a/checkpoint-284/config.json b/checkpoint-284/config.json
new file mode 100644
index 0000000..436e93d
--- /dev/null
+++ b/checkpoint-284/config.json
@@ -0,0 +1,62 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 512,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_bidirectional_attention": false,
+  "use_cache": false,
+  "vocab_size": 262144
+}
diff --git a/checkpoint-284/generation_config.json b/checkpoint-284/generation_config.json
new file mode 100644
index 0000000..c652aad
--- /dev/null
+++ b/checkpoint-284/generation_config.json
@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.0.0"
+}
diff --git a/checkpoint-284/model.safetensors b/checkpoint-284/model.safetensors
new file mode 100644
index 0000000..0aae080
--- /dev/null
+++ b/checkpoint-284/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d96c91306eb76ff6917e36d262afbcc696836520af2951f30d4fe02f9f50ad8
+size 536223056
diff --git a/checkpoint-284/optimizer.pt b/checkpoint-284/optimizer.pt
new file mode 100644
index 0000000..3fc1779
--- /dev/null
+++ b/checkpoint-284/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1599c40ff1a4dbea243940689319509368598856b6a4085fbe267e44eeb8a59
+size 1072591115
diff --git a/checkpoint-284/rng_state.pth b/checkpoint-284/rng_state.pth
new file mode 100644
index 0000000..1fb544a
--- /dev/null
+++ b/checkpoint-284/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdc37bbd2e979f041dfbbb004a5c74bab6cdda159cb18116df728588515a9ef6
+size 14645
diff --git a/checkpoint-284/scheduler.pt b/checkpoint-284/scheduler.pt
new file mode 100644
index 0000000..b68fcab
--- /dev/null
+++ b/checkpoint-284/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:173f9bcfe6d98de3b547cda3c500547190199c4bb0a20a7f6ab221a288667d8b
+size 1465
diff --git a/checkpoint-284/tokenizer.json b/checkpoint-284/tokenizer.json
new file mode 100644
index 0000000..899af07
--- /dev/null
+++ b/checkpoint-284/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
+size 33384443
diff --git a/checkpoint-284/tokenizer_config.json b/checkpoint-284/tokenizer_config.json
new file mode 100644
index 0000000..7202584
--- /dev/null
+++ b/checkpoint-284/tokenizer_config.json
@@ -0,0 +1,24 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}
diff --git a/checkpoint-284/trainer_state.json b/checkpoint-284/trainer_state.json
new file mode 100644
index 0000000..0623200
--- /dev/null
+++ b/checkpoint-284/trainer_state.json
@@ -0,0 +1,2918 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 284,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4938980638980865,
+      "epoch": 0.014084507042253521,
+      "grad_norm": 89.5,
+      "learning_rate": 5e-05,
+      "loss": 4.519379615783691,
+      "mean_token_accuracy": 0.3811451345682144,
+      "num_tokens": 2953.0,
+      "step": 1
+    },
+    {
+      "entropy": 3.6362282037734985,
+      "epoch": 0.028169014084507043,
+      "grad_norm": 29.125,
+      "learning_rate": 5e-05,
+      "loss": 3.7604172229766846,
+      "mean_token_accuracy": 0.39103876799345016,
+      "num_tokens": 5196.0,
+      "step": 2
+    },
+    {
+      "entropy": 3.796808362007141,
+      "epoch": 0.04225352112676056,
+      "grad_norm": 18.75,
+      "learning_rate": 5e-05,
+      "loss": 3.3663859367370605,
+      "mean_token_accuracy": 0.42154909670352936,
+      "num_tokens": 7537.0,
+      "step": 3
+    },
+    {
+      "entropy": 3.071028172969818,
+      "epoch": 0.056338028169014086,
+      "grad_norm": 14.5625,
+      "learning_rate": 5e-05,
+      "loss": 3.1916425228118896,
+      "mean_token_accuracy": 0.4548730403184891,
+      "num_tokens": 9584.0,
+      "step": 4
+    },
+    {
+      "entropy": 2.7734431624412537,
+      "epoch": 0.07042253521126761,
+      "grad_norm": 15.0,
+      "learning_rate": 5e-05,
+      "loss": 2.661928653717041,
+      "mean_token_accuracy": 0.5473615825176239,
+      "num_tokens": 11344.0,
+      "step": 5
+    },
+    {
+      "entropy": 2.4827571511268616,
+      "epoch": 0.08450704225352113,
+      "grad_norm": 13.6875,
+      "learning_rate": 5e-05,
+      "loss": 2.691732168197632,
+      "mean_token_accuracy": 0.5095682442188263,
+      "num_tokens": 13289.0,
+      "step": 6
+    },
+    {
+      "entropy": 2.6064285039901733,
+      "epoch": 0.09859154929577464,
+      "grad_norm": 12.625,
+      "learning_rate": 5e-05,
+      "loss": 2.7455577850341797,
+      "mean_token_accuracy": 0.5099757686257362,
+      "num_tokens": 15664.0,
+      "step": 7
+    },
+    {
+      "entropy": 2.4781495332717896,
+      "epoch": 0.11267605633802817,
+      "grad_norm": 12.75,
+      "learning_rate": 5e-05,
+      "loss": 2.5589702129364014,
+      "mean_token_accuracy": 0.5618840456008911,
+      "num_tokens": 17331.0,
+      "step": 8
+    },
+    {
+      "entropy": 2.512305438518524,
+      "epoch": 0.1267605633802817,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.5893146991729736,
+      "mean_token_accuracy": 0.5188388600945473,
+      "num_tokens": 19724.0,
+      "step": 9
+    },
+    {
+      "entropy": 2.187403589487076,
+      "epoch": 0.14084507042253522,
+      "grad_norm": 11.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4727907180786133,
+      "mean_token_accuracy": 0.6085146963596344,
+      "num_tokens": 21334.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.8956857919692993,
+      "epoch": 0.15492957746478872,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.8553805351257324,
+      "mean_token_accuracy": 0.4754949361085892,
+      "num_tokens": 23770.0,
+      "step": 11
+    },
+    {
+      "entropy": 2.4213987588882446,
+      "epoch": 0.16901408450704225,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4139676094055176,
+      "mean_token_accuracy": 0.5475910231471062,
+      "num_tokens": 26233.0,
+      "step": 12
+    },
+    {
+      "entropy": 2.668797016143799,
+      "epoch": 0.18309859154929578,
+      "grad_norm": 12.25,
+      "learning_rate": 5e-05,
+      "loss": 2.769174337387085,
+      "mean_token_accuracy": 0.5351328551769257,
+      "num_tokens": 27692.0,
+      "step": 13
+    },
+    {
+      "entropy": 2.544685661792755,
+      "epoch": 0.19718309859154928,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.4870781898498535,
+      "mean_token_accuracy": 0.514883853495121,
+      "num_tokens": 30217.0,
+      "step": 14
+    },
+    {
+      "entropy": 2.5978153944015503,
+      "epoch": 0.2112676056338028,
+      "grad_norm": 10.875,
+      "learning_rate": 5e-05,
+      "loss": 2.639450788497925,
+      "mean_token_accuracy": 0.5468774288892746,
+      "num_tokens": 31969.0,
+      "step": 15
+    },
+    {
+      "entropy": 2.452792704105377,
+      "epoch": 0.22535211267605634,
+      "grad_norm": 11.5,
+      "learning_rate": 5e-05,
+      "loss": 2.5285415649414062,
+      "mean_token_accuracy": 0.5533104464411736,
+      "num_tokens": 34160.0,
+      "step": 16
+    },
+    {
+      "entropy": 2.234907388687134,
+      "epoch": 0.23943661971830985,
+      "grad_norm": 12.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.2553577423095703,
+      "mean_token_accuracy": 0.5958019495010376,
+      "num_tokens": 35899.0,
+      "step": 17
+    },
+    {
+      "entropy": 2.2781192660331726,
+      "epoch": 0.2535211267605634,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4401230812072754,
+      "mean_token_accuracy": 0.5632342845201492,
+      "num_tokens": 37693.0,
+      "step": 18
+    },
+    {
+      "entropy": 2.292062520980835,
+      "epoch": 0.2676056338028169,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 2.53348445892334,
+      "mean_token_accuracy": 0.5441952645778656,
+      "num_tokens": 39793.0,
+      "step": 19
+    },
+    {
+      "entropy": 2.5250023007392883,
+      "epoch": 0.28169014084507044,
+      "grad_norm": 13.5,
+      "learning_rate": 5e-05,
+      "loss": 2.7173304557800293,
+      "mean_token_accuracy": 0.4843936040997505,
+      "num_tokens": 42035.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.3936269879341125,
+      "epoch": 0.29577464788732394,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.397752046585083,
+      "mean_token_accuracy": 0.5632070526480675,
+      "num_tokens": 43606.0,
+      "step": 21
+    },
+    {
+      "entropy": 2.7215799689292908,
+      "epoch": 0.30985915492957744,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.7266643047332764,
+      "mean_token_accuracy": 0.4814189076423645,
+      "num_tokens": 46160.0,
+      "step": 22
+    },
+    {
+      "entropy": 2.4695653319358826,
+      "epoch": 0.323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3790149688720703,
+      "mean_token_accuracy": 0.5413414835929871,
+      "num_tokens": 48749.0,
+      "step": 23
+    },
+    {
+      "entropy": 2.3166028261184692,
+      "epoch": 0.3380281690140845,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.274803638458252,
+      "mean_token_accuracy": 0.5993590205907822,
+      "num_tokens": 50770.0,
+      "step": 24
+    },
+    {
+      "entropy": 2.285377472639084,
+      "epoch": 0.352112676056338,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.2308878898620605,
+      "mean_token_accuracy": 0.5828141123056412,
+      "num_tokens": 53004.0,
+      "step": 25
+    },
+    {
+      "entropy": 2.1157060265541077,
+      "epoch": 0.36619718309859156,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1033847332000732,
+      "mean_token_accuracy": 0.5867631435394287,
+      "num_tokens": 55108.0,
+      "step": 26
+    },
+    {
+      "entropy": 2.5209829807281494,
+      "epoch": 0.38028169014084506,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.3607499599456787,
+      "mean_token_accuracy": 0.5426046550273895,
+      "num_tokens": 57080.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.974183738231659,
+      "epoch": 0.39436619718309857,
+      "grad_norm": 7.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.9418365955352783,
+      "mean_token_accuracy": 0.6255736351013184,
+      "num_tokens": 59237.0,
+      "step": 28
+    },
+    {
+      "entropy": 2.252374231815338,
+      "epoch": 0.4084507042253521,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 2.228358507156372,
+      "mean_token_accuracy": 0.5917280316352844,
+      "num_tokens": 60850.0,
+      "step": 29
+    },
+    {
+      "entropy": 2.327530801296234,
+      "epoch": 0.4225352112676056,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.3329389095306396,
+      "mean_token_accuracy": 0.5718153715133667,
+      "num_tokens": 62734.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.2620502710342407,
+      "epoch": 0.43661971830985913,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1953892707824707,
+      "mean_token_accuracy": 0.5675001591444016,
+      "num_tokens": 64637.0,
+      "step": 31
+    },
+    {
+      "entropy": 2.445267140865326,
+      "epoch": 0.4507042253521127,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.501152276992798,
+      "mean_token_accuracy": 0.5355894565582275,
+      "num_tokens": 66880.0,
+      "step": 32
+    },
+    {
+      "entropy": 2.3873718976974487,
+      "epoch": 0.4647887323943662,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3468880653381348,
+      "mean_token_accuracy": 0.5372899621725082,
+      "num_tokens": 68623.0,
+      "step": 33
+    },
+    {
+      "entropy": 2.0545941591262817,
+      "epoch": 0.4788732394366197,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.1947593688964844,
+      "mean_token_accuracy": 0.6112199202179909,
+      "num_tokens": 70769.0,
+      "step": 34
+    },
+    {
+      "entropy": 2.350032150745392,
+      "epoch": 0.49295774647887325,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.418266773223877,
+      "mean_token_accuracy": 0.5104246437549591,
+      "num_tokens": 73653.0,
+      "step": 35
+    },
+    {
+      "entropy": 2.237062931060791,
+      "epoch": 0.5070422535211268,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4755356311798096,
+      "mean_token_accuracy": 0.5747656747698784,
+      "num_tokens": 75524.0,
+      "step": 36
+    },
+    {
+      "entropy": 2.0994813442230225,
+      "epoch": 0.5211267605633803,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.044315814971924,
+      "mean_token_accuracy": 0.6056438237428665,
+      "num_tokens": 77246.0,
+      "step": 37
+    },
+    {
+      "entropy": 2.5556156635284424,
+      "epoch": 0.5352112676056338,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.446904182434082,
+      "mean_token_accuracy": 0.5177429020404816,
+      "num_tokens": 79970.0,
+      "step": 38
+    },
+    {
+      "entropy": 2.337540030479431,
+      "epoch": 0.5492957746478874,
+      "grad_norm": 8.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.216331720352173,
+      "mean_token_accuracy": 0.58082115650177,
+      "num_tokens": 81876.0,
+      "step": 39
+    },
+    {
+      "entropy": 2.043498247861862,
+      "epoch": 0.5633802816901409,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9643354415893555,
+      "mean_token_accuracy": 0.607293888926506,
+      "num_tokens": 84298.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.4852016270160675,
+      "epoch": 0.5774647887323944,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.571437358856201,
+      "mean_token_accuracy": 0.5233406946063042,
+      "num_tokens": 86291.0,
+      "step": 41
+    },
+    {
+      "entropy": 2.5456899404525757,
+      "epoch": 0.5915492957746479,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 2.4434070587158203,
+      "mean_token_accuracy": 0.5472028106451035,
+      "num_tokens": 88194.0,
+      "step": 42
+    },
+    {
+      "entropy": 2.2191686630249023,
+      "epoch": 0.6056338028169014,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.225029468536377,
+      "mean_token_accuracy": 0.6006355881690979,
+      "num_tokens": 89922.0,
+      "step": 43
+    },
+    {
+      "entropy": 2.2295450568199158,
+      "epoch": 0.6197183098591549,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4000232219696045,
+      "mean_token_accuracy": 0.580116793513298,
+      "num_tokens": 92171.0,
+      "step": 44
+    },
+    {
+      "entropy": 2.2142042219638824,
+      "epoch": 0.6338028169014085,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2619423866271973,
+      "mean_token_accuracy": 0.5818447023630142,
+      "num_tokens": 94362.0,
+      "step": 45
+    },
+    {
+      "entropy": 2.3695861101150513,
+      "epoch": 0.647887323943662,
+      "grad_norm": 7.21875,
+      "learning_rate": 5e-05,
+      "loss": 2.4540324211120605,
+      "mean_token_accuracy": 0.5298221409320831,
+      "num_tokens": 96559.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.9680090248584747,
+      "epoch": 0.6619718309859155,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.9608612060546875,
+      "mean_token_accuracy": 0.6151116490364075,
+      "num_tokens": 99018.0,
+      "step": 47
+    },
+    {
+      "entropy": 2.449136793613434,
+      "epoch": 0.676056338028169,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.364119529724121,
+      "mean_token_accuracy": 0.5657317340373993,
+      "num_tokens": 100885.0,
+      "step": 48
+    },
+    {
+      "entropy": 2.109535813331604,
+      "epoch": 0.6901408450704225,
+      "grad_norm": 6.875,
+      "learning_rate": 5e-05,
+      "loss": 2.1192779541015625,
+      "mean_token_accuracy": 0.5970411896705627,
+      "num_tokens": 103357.0,
+      "step": 49
+    },
+    {
+      "entropy": 2.3080650568008423,
+      "epoch": 0.704225352112676,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.351897716522217,
+      "mean_token_accuracy": 0.5307685062289238,
+      "num_tokens": 105870.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.1463415920734406,
+      "epoch": 0.7183098591549296,
+      "grad_norm": 7.5,
+      "learning_rate": 5e-05,
+      "loss": 2.2693376541137695,
+      "mean_token_accuracy": 0.5934874191880226,
+      "num_tokens": 107995.0,
+      "step": 51
+    },
+    {
+      "entropy": 2.1181687712669373,
+      "epoch": 0.7323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.1629841327667236,
+      "mean_token_accuracy": 0.6063478291034698,
+      "num_tokens": 110358.0,
+      "step": 52
+    },
+    {
+      "entropy": 2.0169546008110046,
+      "epoch": 0.7464788732394366,
+      "grad_norm": 7.78125,
+      "learning_rate": 5e-05,
+      "loss": 1.817868709564209,
+      "mean_token_accuracy": 0.6273332387208939,
+      "num_tokens": 112267.0,
+      "step": 53
+    },
+    {
+      "entropy": 2.189443975687027,
+      "epoch": 0.7605633802816901,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.3292086124420166,
+      "mean_token_accuracy": 0.6041209101676941,
+      "num_tokens": 113899.0,
+      "step": 54
+    },
+    {
+      "entropy": 2.1946000456809998,
+      "epoch": 0.7746478873239436,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2774949073791504,
+      "mean_token_accuracy": 0.5620942413806915,
+      "num_tokens": 116015.0,
+      "step": 55
+    },
+    {
+      "entropy": 2.1172711849212646,
+      "epoch": 0.7887323943661971,
+      "grad_norm": 7.90625,
+      "learning_rate": 5e-05,
+      "loss": 2.1236391067504883,
+      "mean_token_accuracy": 0.5911316424608231,
+      "num_tokens": 118044.0,
+      "step": 56
+    },
+    {
+      "entropy": 2.144031971693039,
+      "epoch": 0.8028169014084507,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.187140703201294,
+      "mean_token_accuracy": 0.5624879449605942,
+      "num_tokens": 120180.0,
+      "step": 57
+    },
+    {
+      "entropy": 2.181178390979767,
+      "epoch": 0.8169014084507042,
+      "grad_norm": 10.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.113279104232788,
+      "mean_token_accuracy": 0.6004882454872131,
+      "num_tokens": 121676.0,
+      "step": 58
+    },
+    {
+      "entropy": 2.314671218395233,
+      "epoch": 0.8309859154929577,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.331024169921875,
+      "mean_token_accuracy": 0.5824068263173103,
+      "num_tokens": 123161.0,
+      "step": 59
+    },
+    {
+      "entropy": 2.0048855245113373,
+      "epoch": 0.8450704225352113,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.070265054702759,
+      "mean_token_accuracy": 0.6141902804374695,
+      "num_tokens": 125016.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.966452568769455,
+      "epoch": 0.8591549295774648,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.9488521814346313,
+      "mean_token_accuracy": 0.5854089111089706,
+      "num_tokens": 127349.0,
+      "step": 61
+    },
+    {
+      "entropy": 2.1519170999526978,
+      "epoch": 0.8732394366197183,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.1743807792663574,
+      "mean_token_accuracy": 0.5636171847581863,
+      "num_tokens": 129742.0,
+      "step": 62
+    },
+    {
+      "entropy": 2.7546687722206116,
+      "epoch": 0.8873239436619719,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.7343056201934814,
+      "mean_token_accuracy": 0.496803916990757,
+      "num_tokens": 131743.0,
+      "step": 63
+    },
+    {
+      "entropy": 2.421773910522461,
+      "epoch": 0.9014084507042254,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.6149866580963135,
+      "mean_token_accuracy": 0.5249136164784431,
+      "num_tokens": 134002.0,
+      "step": 64
+    },
+    {
+      "entropy": 2.0106727480888367,
+      "epoch": 0.9154929577464789,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.940914273262024,
+      "mean_token_accuracy": 0.5855439230799675,
+      "num_tokens": 136120.0,
+      "step": 65
+    },
+    {
+      "entropy": 2.1143648624420166,
+      "epoch": 0.9295774647887324,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.2469482421875,
+      "mean_token_accuracy": 0.6145937740802765,
+      "num_tokens": 137997.0,
+      "step": 66
+    },
+    {
+      "entropy": 2.1894670128822327,
+      "epoch": 0.9436619718309859,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.20756459236145,
+      "mean_token_accuracy": 0.5872255340218544,
+      "num_tokens": 139904.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.75501748919487,
+      "epoch": 0.9577464788732394,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.9068934917449951,
+      "mean_token_accuracy": 0.6642919778823853,
+      "num_tokens": 141166.0,
+      "step": 68
+    },
+    {
+      "entropy": 2.204477220773697,
+      "epoch": 0.971830985915493,
+      "grad_norm": 7.875,
+      "learning_rate": 5e-05,
+      "loss": 2.286216974258423,
+      "mean_token_accuracy": 0.5354761853814125,
+      "num_tokens": 143738.0,
+      "step": 69
+    },
+    {
+      "entropy": 2.1219914853572845,
+      "epoch": 0.9859154929577465,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 2.274585723876953,
+      "mean_token_accuracy": 0.609759122133255,
+      "num_tokens": 145439.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.9260511100292206,
+      "epoch": 1.0,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.8763841390609741,
+      "mean_token_accuracy": 0.5979989916086197,
+      "num_tokens": 147650.0,
+      "step": 71
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 2.1684498711371085,
+      "eval_loss": 2.284580707550049,
+      "eval_mean_token_accuracy": 0.5757196218195096,
+      "eval_num_tokens": 147650.0,
+      "eval_runtime": 18.8201,
+      "eval_samples_per_second": 15.09,
+      "eval_steps_per_second": 3.773,
+      "step": 71
+    },
+    {
+      "entropy": 2.24437940120697,
+      "epoch": 1.0140845070422535,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 2.148402214050293,
+      "mean_token_accuracy": 0.5866604596376419,
+      "num_tokens": 149372.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.630838692188263,
+      "epoch": 1.028169014084507,
+      "grad_norm": 7.0,
+      "learning_rate": 5e-05,
+      "loss": 1.526665449142456,
+      "mean_token_accuracy": 0.6816482096910477,
+      "num_tokens": 151274.0,
+      "step": 73
+    },
+    {
+      "entropy": 2.193234443664551,
+      "epoch": 1.0422535211267605,
+      "grad_norm": 7.90625,
+      "learning_rate": 5e-05,
+      "loss": 2.176100969314575,
+      "mean_token_accuracy": 0.6048732623457909,
+      "num_tokens": 153103.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.7758266925811768,
+      "epoch": 1.056338028169014,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.7472716569900513,
+      "mean_token_accuracy": 0.6403823792934418,
+      "num_tokens": 155291.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.8887377083301544,
+      "epoch": 1.0704225352112675,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.872934341430664,
+      "mean_token_accuracy": 0.6085800677537918,
+      "num_tokens": 157847.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.6055135428905487,
+      "epoch": 1.084507042253521,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.6859486103057861,
+      "mean_token_accuracy": 0.6478956639766693,
+      "num_tokens": 160538.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.9197329580783844,
+      "epoch": 1.0985915492957747,
+      "grad_norm": 7.375,
+      "learning_rate": 5e-05,
+      "loss": 1.929700493812561,
+      "mean_token_accuracy": 0.6054274141788483,
+      "num_tokens": 162709.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.6682755053043365,
+      "epoch": 1.1126760563380282,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.899672269821167,
+      "mean_token_accuracy": 0.6613650918006897,
+      "num_tokens": 164478.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.8098062872886658,
+      "epoch": 1.1267605633802817,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7989305257797241,
+      "mean_token_accuracy": 0.6530113369226456,
+      "num_tokens": 165767.0,
+      "step": 80
+    },
+    {
+      "entropy": 2.3263643383979797,
+      "epoch": 1.1408450704225352,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.32468843460083,
+      "mean_token_accuracy": 0.5657684504985809,
+      "num_tokens": 167374.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.6652490496635437,
+      "epoch": 1.1549295774647887,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.821378469467163,
+      "mean_token_accuracy": 0.6479186862707138,
+      "num_tokens": 169216.0,
+      "step": 82
+    },
+    {
+      "entropy": 2.157005727291107,
+      "epoch": 1.1690140845070423,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.1137959957122803,
+      "mean_token_accuracy": 0.5775181204080582,
+      "num_tokens": 171194.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.6896095722913742,
+      "epoch": 1.1830985915492958,
+      "grad_norm": 8.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.785524606704712,
+      "mean_token_accuracy": 0.6791908890008926,
+      "num_tokens": 173018.0,
+      "step": 84
+    },
+    {
+      "entropy": 2.241276800632477,
+      "epoch": 1.1971830985915493,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.1982414722442627,
+      "mean_token_accuracy": 0.5830243527889252,
+      "num_tokens": 174822.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.8407301902770996,
+      "epoch": 1.2112676056338028,
+      "grad_norm": 7.65625,
+      "learning_rate": 5e-05,
+      "loss": 1.752529501914978,
+      "mean_token_accuracy": 0.6172220706939697,
+      "num_tokens": 176932.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.707479178905487,
+      "epoch": 1.2253521126760563,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.6565136909484863,
+      "mean_token_accuracy": 0.6619749516248703,
+      "num_tokens": 178804.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.6418649852275848,
+      "epoch": 1.2394366197183098,
+      "grad_norm": 7.15625,
+      "learning_rate": 5e-05,
+      "loss": 1.5298699140548706,
+      "mean_token_accuracy": 0.6645439267158508,
+      "num_tokens": 181353.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.6070191860198975,
+      "epoch": 1.2535211267605635,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.7814466953277588,
+      "mean_token_accuracy": 0.6640422344207764,
+      "num_tokens": 183278.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.8768837749958038,
+      "epoch": 1.267605633802817,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9979424476623535,
+      "mean_token_accuracy": 0.6454368084669113,
+      "num_tokens": 185041.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.809422641992569,
+      "epoch": 1.2816901408450705,
+      "grad_norm": 7.78125,
+      "learning_rate": 5e-05,
+      "loss": 1.8236162662506104,
+      "mean_token_accuracy": 0.6187984794378281,
+      "num_tokens": 187646.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.8032868206501007,
+      "epoch": 1.295774647887324,
+      "grad_norm": 7.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.8008391857147217,
+      "mean_token_accuracy": 0.6168265342712402,
+      "num_tokens": 189952.0,
+      "step": 92
+    },
+    {
+      "entropy": 2.107921302318573,
+      "epoch": 1.3098591549295775,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 2.077434539794922,
+      "mean_token_accuracy": 0.5662726014852524,
+      "num_tokens": 192592.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.6842139065265656,
+      "epoch": 1.323943661971831,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.6454025506973267,
+      "mean_token_accuracy": 0.6700995117425919,
+      "num_tokens": 194549.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.859749436378479,
+      "epoch": 1.3380281690140845,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9091846942901611,
+      "mean_token_accuracy": 0.6098746657371521,
+      "num_tokens": 196448.0,
+      "step": 95
+    },
+    {
+      "entropy": 2.018050581216812,
+      "epoch": 1.352112676056338,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.1190457344055176,
+      "mean_token_accuracy": 0.5959457382559776,
+      "num_tokens": 198615.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.9890158474445343,
+      "epoch": 1.3661971830985915,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.960221529006958,
+      "mean_token_accuracy": 0.5875463634729385,
+      "num_tokens": 201013.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.8716930747032166,
+      "epoch": 1.380281690140845,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 2.0004959106445312,
+      "mean_token_accuracy": 0.6175930351018906,
+      "num_tokens": 202969.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.7610785365104675,
+      "epoch": 1.3943661971830985,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.6850508451461792,
+      "mean_token_accuracy": 0.6504905521869659,
+      "num_tokens": 204906.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.9090750515460968,
+      "epoch": 1.408450704225352,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.938533067703247,
+      "mean_token_accuracy": 0.621729701757431,
+      "num_tokens": 206869.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.8917019963264465,
+      "epoch": 1.4225352112676055,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.0368175506591797,
+      "mean_token_accuracy": 0.6267035603523254,
+      "num_tokens": 208432.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.7962620854377747,
+      "epoch": 1.436619718309859,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.8455748558044434,
+      "mean_token_accuracy": 0.608839362859726,
+      "num_tokens": 210949.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.724296659231186,
+      "epoch": 1.4507042253521127,
+      "grad_norm": 10.25,
+      "learning_rate": 5e-05,
+      "loss": 1.7760952711105347,
+      "mean_token_accuracy": 0.6573535352945328,
+      "num_tokens": 212595.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.8014512360095978,
+      "epoch": 1.4647887323943662,
+      "grad_norm": 7.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.8614972829818726,
+      "mean_token_accuracy": 0.5946371257305145,
+      "num_tokens": 215384.0,
+      "step": 104
+    },
+    {
+      "entropy": 1.8427681922912598,
+      "epoch": 1.4788732394366197,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 1.880817174911499,
+      "mean_token_accuracy": 0.6408448368310928,
+      "num_tokens": 217561.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.6576282978057861,
+      "epoch": 1.4929577464788732,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.6814919710159302,
+      "mean_token_accuracy": 0.6402300000190735,
+      "num_tokens": 219087.0,
+      "step": 106
+    },
+    {
+      "entropy": 1.5478721261024475,
+      "epoch": 1.5070422535211268,
+      "grad_norm": 7.59375,
+      "learning_rate": 5e-05,
+      "loss": 1.5617148876190186,
+      "mean_token_accuracy": 0.6625793874263763,
+      "num_tokens": 221887.0,
+      "step": 107
+    },
+    {
+      "entropy": 1.923494279384613,
+      "epoch": 1.5211267605633803,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.0690817832946777,
+      "mean_token_accuracy": 0.5978109538555145,
+      "num_tokens": 224182.0,
+      "step": 108
+    },
+    {
+      "entropy": 1.8152073919773102,
+      "epoch": 1.5352112676056338,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 1.7398602962493896,
+      "mean_token_accuracy": 0.6207661181688309,
+      "num_tokens": 226591.0,
+      "step": 109
+    },
+    {
+      "entropy": 2.1172706484794617,
+      "epoch": 1.5492957746478875,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.237187385559082,
+      "mean_token_accuracy": 0.5684898793697357,
+      "num_tokens": 228529.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.686636283993721,
+      "epoch": 1.563380281690141,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.8167091608047485,
+      "mean_token_accuracy": 0.6692909449338913,
+      "num_tokens": 229973.0,
+      "step": 111
+    },
+    {
+      "entropy": 2.285969853401184,
+      "epoch": 1.5774647887323945,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 2.308091878890991,
+      "mean_token_accuracy": 0.5341236144304276,
+      "num_tokens": 232232.0,
+      "step": 112
+    },
+    {
+      "entropy": 1.7772781550884247,
+      "epoch": 1.591549295774648,
+      "grad_norm": 7.09375,
+      "learning_rate": 5e-05,
+      "loss": 1.7048449516296387,
+      "mean_token_accuracy": 0.6309333443641663,
+      "num_tokens": 234998.0,
+      "step": 113
+    },
+    {
+      "entropy": 1.8632723987102509,
+      "epoch": 1.6056338028169015,
+      "grad_norm": 6.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.8152518272399902,
+      "mean_token_accuracy": 0.5958341658115387,
+      "num_tokens": 237842.0,
+      "step": 114
+    },
+    {
+      "entropy": 1.7296560108661652,
+      "epoch": 1.619718309859155,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.7719354629516602,
+      "mean_token_accuracy": 0.6910708993673325,
+      "num_tokens": 239068.0,
+      "step": 115
+    },
+    {
+      "entropy": 2.026154547929764,
+      "epoch": 1.6338028169014085,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 2.067565441131592,
+      "mean_token_accuracy": 0.5664031803607941,
+      "num_tokens": 241497.0,
+      "step": 116
+    },
+    {
+      "entropy": 1.6592170596122742,
+      "epoch": 1.647887323943662,
+      "grad_norm": 7.65625,
+      "learning_rate": 5e-05,
+      "loss": 1.7524383068084717,
+      "mean_token_accuracy": 0.6437603831291199,
+      "num_tokens": 243721.0,
+      "step": 117
+    },
+    {
+      "entropy": 1.8260776698589325,
+      "epoch": 1.6619718309859155,
+      "grad_norm": 8.0,
+      "learning_rate": 5e-05,
+      "loss": 1.7933809757232666,
+      "mean_token_accuracy": 0.6192324906587601,
+      "num_tokens": 246193.0,
+      "step": 118
+    },
+    {
+      "entropy": 1.6278434097766876,
+      "epoch": 1.676056338028169,
+      "grad_norm": 7.875,
+      "learning_rate": 5e-05,
+      "loss": 1.6023352146148682,
+      "mean_token_accuracy": 0.6632742285728455,
+      "num_tokens": 248065.0,
+      "step": 119
+    },
+    {
+      "entropy": 1.5735027194023132,
+      "epoch": 1.6901408450704225,
+      "grad_norm": 7.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.5997726917266846,
+      "mean_token_accuracy": 0.6820999830961227,
+      "num_tokens": 250126.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.7936385571956635,
+      "epoch": 1.704225352112676,
+      "grad_norm": 7.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.662062644958496,
+      "mean_token_accuracy": 0.6343215107917786,
+      "num_tokens": 252242.0,
+      "step": 121
+    },
+    {
+      "entropy": 2.0255846083164215,
+      "epoch": 1.7183098591549295,
+      "grad_norm": 7.53125,
+      "learning_rate": 5e-05,
+      "loss": 1.9866116046905518,
+      "mean_token_accuracy": 0.5996693223714828,
+      "num_tokens": 254704.0,
+      "step": 122
+    },
+    {
+      "entropy": 1.8428587317466736,
+      "epoch": 1.732394366197183,
+      "grad_norm": 7.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.786312460899353,
+      "mean_token_accuracy": 0.6079171597957611,
+      "num_tokens": 257154.0,
+      "step": 123
+    },
+    {
+      "entropy": 1.8701068460941315,
+      "epoch": 1.7464788732394365,
+      "grad_norm": 6.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.814254641532898,
+      "mean_token_accuracy": 0.6146602630615234,
+      "num_tokens": 260050.0,
+      "step": 124
+    },
+    {
+      "entropy": 1.7365414798259735,
+      "epoch": 1.76056338028169,
+      "grad_norm": 6.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.6392512321472168,
+      "mean_token_accuracy": 0.631788820028305,
+      "num_tokens": 262731.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.5750393569469452,
+      "epoch": 1.7746478873239435,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.652256727218628,
+      "mean_token_accuracy": 0.6727721691131592,
+      "num_tokens": 264264.0,
+      "step": 126
+    },
+    {
+      "entropy": 1.6853447258472443,
+      "epoch": 1.788732394366197,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.8110965490341187,
+      "mean_token_accuracy": 0.6612876951694489,
+      "num_tokens": 265839.0,
+      "step": 127
+    },
+    {
+      "entropy": 1.9790962934494019,
+      "epoch": 1.8028169014084507,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.9432560205459595,
+      "mean_token_accuracy": 0.5927930474281311,
+      "num_tokens": 268531.0,
+      "step": 128
+    },
+    {
+      "entropy": 1.6438206434249878,
+      "epoch": 1.8169014084507042,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 2.005178689956665,
+      "mean_token_accuracy": 0.6584273278713226,
+      "num_tokens": 270295.0,
+      "step": 129
+    },
+    {
+      "entropy": 1.8002346009016037,
+      "epoch": 1.8309859154929577,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.9875435829162598,
+      "mean_token_accuracy": 0.6358410120010376,
+      "num_tokens": 272470.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.811629831790924,
+      "epoch": 1.8450704225352113,
+      "grad_norm": 9.25,
+      "learning_rate": 5e-05,
+      "loss": 1.896381139755249,
+      "mean_token_accuracy": 0.6606249660253525,
+      "num_tokens": 274166.0,
+      "step": 131
+    },
+    {
+      "entropy": 1.7871823608875275,
+      "epoch": 1.8591549295774648,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7262006998062134,
+      "mean_token_accuracy": 0.6428729891777039,
+      "num_tokens": 276354.0,
+      "step": 132
+    },
+    {
+      "entropy": 1.5866018235683441,
+      "epoch": 1.8732394366197183,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.54076087474823,
+      "mean_token_accuracy": 0.6837680488824844,
+      "num_tokens": 278044.0,
+      "step": 133
+    },
+    {
+      "entropy": 1.9443781971931458,
+      "epoch": 1.887323943661972,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.935890555381775,
+      "mean_token_accuracy": 0.5984784960746765,
+      "num_tokens": 279727.0,
+      "step": 134
+    },
+    {
+      "entropy": 1.6744503676891327,
+      "epoch": 1.9014084507042255,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.667353868484497,
+      "mean_token_accuracy": 0.6531110107898712,
+      "num_tokens": 281727.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.5089002698659897,
+      "epoch": 1.915492957746479,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.7763428688049316,
+      "mean_token_accuracy": 0.6703489422798157,
+      "num_tokens": 283499.0,
+      "step": 136
+    },
+    {
+      "entropy": 1.7822388112545013,
+      "epoch": 1.9295774647887325,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.7652621269226074,
+      "mean_token_accuracy": 0.6092153340578079,
+      "num_tokens": 286098.0,
+      "step": 137
+    },
+    {
+      "entropy": 1.633983552455902,
+      "epoch": 1.943661971830986,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.7642450332641602,
+      "mean_token_accuracy": 0.6580131351947784,
+      "num_tokens": 288020.0,
+      "step": 138
+    },
+    {
+      "entropy": 1.9797956943511963,
+      "epoch": 1.9577464788732395,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.8309262990951538,
+      "mean_token_accuracy": 0.6048336029052734,
+      "num_tokens": 290401.0,
+      "step": 139
+    },
+    {
+      "entropy": 1.5087556838989258,
+      "epoch": 1.971830985915493,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.6536986827850342,
+      "mean_token_accuracy": 0.7231832593679428,
+      "num_tokens": 291584.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.9238645434379578,
+      "epoch": 1.9859154929577465,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.7226345539093018,
+      "mean_token_accuracy": 0.6254735887050629,
+      "num_tokens": 293607.0,
+      "step": 141
+    },
+    {
+      "entropy": 1.587816596031189,
+      "epoch": 2.0,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.5887134075164795,
+      "mean_token_accuracy": 0.6791131645441055,
+      "num_tokens": 295300.0,
+      "step": 142
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 1.972671693479511,
+      "eval_loss": 2.2974655628204346,
+      "eval_mean_token_accuracy": 0.5787080614499642,
+      "eval_num_tokens": 295300.0,
+      "eval_runtime": 18.8723,
+      "eval_samples_per_second": 15.049,
+      "eval_steps_per_second": 3.762,
+      "step": 142
+    },
+    {
+      "entropy": 1.9815530478954315,
+      "epoch": 2.0140845070422535,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.6473958492279053,
+      "mean_token_accuracy": 0.6364074051380157,
+      "num_tokens": 297724.0,
+      "step": 143
+    },
+    {
+      "entropy": 1.641160249710083,
+      "epoch": 2.028169014084507,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.4365612268447876,
+      "mean_token_accuracy": 0.678604006767273,
+      "num_tokens": 300066.0,
+      "step": 144
+    },
+    {
+      "entropy": 1.4447078704833984,
+      "epoch": 2.0422535211267605,
+      "grad_norm": 7.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4016954898834229,
+      "mean_token_accuracy": 0.6932056993246078,
+      "num_tokens": 302321.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.5544451773166656,
+      "epoch": 2.056338028169014,
+      "grad_norm": 9.25,
+      "learning_rate": 5e-05,
+      "loss": 1.5597270727157593,
+      "mean_token_accuracy": 0.6651838272809982,
+      "num_tokens": 304496.0,
+      "step": 146
+    },
+    {
+      "entropy": 1.85980024933815,
+      "epoch": 2.0704225352112675,
+      "grad_norm": 9.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.8932693004608154,
+      "mean_token_accuracy": 0.6070298254489899,
+      "num_tokens": 306612.0,
+      "step": 147
+    },
+    {
+      "entropy": 1.6910157799720764,
+      "epoch": 2.084507042253521,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.7285449504852295,
+      "mean_token_accuracy": 0.6408755928277969,
+      "num_tokens": 308440.0,
+      "step": 148
+    },
+    {
+      "entropy": 1.367365151643753,
+      "epoch": 2.0985915492957745,
+      "grad_norm": 9.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.4051942825317383,
+      "mean_token_accuracy": 0.6981760710477829,
+      "num_tokens": 310467.0,
+      "step": 149
+    },
+    {
+      "entropy": 1.3810956180095673,
+      "epoch": 2.112676056338028,
+      "grad_norm": 11.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4497928619384766,
+      "mean_token_accuracy": 0.6659071892499924,
+      "num_tokens": 313093.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.6426885724067688,
+      "epoch": 2.1267605633802815,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.730147123336792,
+      "mean_token_accuracy": 0.636825755238533,
+      "num_tokens": 315604.0,
+      "step": 151
+    },
+    {
+      "entropy": 1.5957364737987518,
+      "epoch": 2.140845070422535,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.6714507341384888,
+      "mean_token_accuracy": 0.6542615443468094,
+      "num_tokens": 317613.0,
+      "step": 152
+    },
+    {
+      "entropy": 2.0231593549251556,
+      "epoch": 2.1549295774647885,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.9902608394622803,
+      "mean_token_accuracy": 0.6068925857543945,
+      "num_tokens": 318980.0,
+      "step": 153
+    },
+    {
+      "entropy": 1.5751276016235352,
+      "epoch": 2.169014084507042,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.5008318424224854,
+      "mean_token_accuracy": 0.6704117655754089,
+      "num_tokens": 320981.0,
+      "step": 154
+    },
+    {
+      "entropy": 1.611257165670395,
+      "epoch": 2.183098591549296,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.5167430639266968,
+      "mean_token_accuracy": 0.6731022894382477,
+      "num_tokens": 323159.0,
+      "step": 155
+    },
+    {
+      "entropy": 1.5024433732032776,
+      "epoch": 2.1971830985915495,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.441955804824829,
+      "mean_token_accuracy": 0.6796386241912842,
+      "num_tokens": 325305.0,
+      "step": 156
+    },
+    {
+      "entropy": 1.7239812314510345,
+      "epoch": 2.211267605633803,
+      "grad_norm": 12.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.6520541906356812,
+      "mean_token_accuracy": 0.6401138156652451,
+      "num_tokens": 327629.0,
+      "step": 157
+    },
+    {
+      "entropy": 1.7822472155094147,
+      "epoch": 2.2253521126760565,
+      "grad_norm": 8.125,
+      "learning_rate": 5e-05,
+      "loss": 1.6968837976455688,
+      "mean_token_accuracy": 0.6756918206810951,
+      "num_tokens": 328919.0,
+      "step": 158
+    },
+    {
+      "entropy": 1.3915225267410278,
+      "epoch": 2.23943661971831,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.410017490386963,
+      "mean_token_accuracy": 0.6763953566551208,
+      "num_tokens": 331333.0,
+      "step": 159
+    },
+    {
+      "entropy": 1.502710074186325,
+      "epoch": 2.2535211267605635,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.4881656169891357,
+      "mean_token_accuracy": 0.6672618389129639,
+      "num_tokens": 332856.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.9380434155464172,
+      "epoch": 2.267605633802817,
+      "grad_norm": 11.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.9208719730377197,
+      "mean_token_accuracy": 0.624039888381958,
+      "num_tokens": 334373.0,
+      "step": 161
+    },
+    {
+      "entropy": 1.6169701665639877,
+      "epoch": 2.2816901408450705,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.8250868320465088,
+      "mean_token_accuracy": 0.6674070954322815,
+      "num_tokens": 336125.0,
+      "step": 162
+    },
+    {
+      "entropy": 1.7778233587741852,
+      "epoch": 2.295774647887324,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.7127352952957153,
+      "mean_token_accuracy": 0.6442407220602036,
+      "num_tokens": 337666.0,
+      "step": 163
+    },
+    {
+      "entropy": 1.5933920741081238,
+      "epoch": 2.3098591549295775,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.615399956703186,
+      "mean_token_accuracy": 0.6445006728172302,
+      "num_tokens": 340298.0,
+      "step": 164
+    },
+    {
+      "entropy": 1.774703323841095,
+      "epoch": 2.323943661971831,
+      "grad_norm": 10.375,
+      "learning_rate": 5e-05,
+      "loss": 1.6529101133346558,
+      "mean_token_accuracy": 0.6362189799547195,
+      "num_tokens": 342187.0,
+      "step": 165
+    },
+    {
+      "entropy": 1.2685332596302032,
+      "epoch": 2.3380281690140845,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.2466249465942383,
+      "mean_token_accuracy": 0.7213118225336075,
+      "num_tokens": 344043.0,
+      "step": 166
+    },
+    {
+      "entropy": 1.5397302210330963,
+      "epoch": 2.352112676056338,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.428548812866211,
+      "mean_token_accuracy": 0.6775327771902084,
+      "num_tokens": 346429.0,
+      "step": 167
+    },
+    {
+      "entropy": 1.6372707784175873,
+      "epoch": 2.3661971830985915,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.587751865386963,
+      "mean_token_accuracy": 0.6544711589813232,
+      "num_tokens": 348838.0,
+      "step": 168
+    },
+    {
+      "entropy": 1.427915170788765,
+      "epoch": 2.380281690140845,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4900782108306885,
+      "mean_token_accuracy": 0.7009869068861008,
+      "num_tokens": 350555.0,
+      "step": 169
+    },
+    {
+      "entropy": 1.374229371547699,
+      "epoch": 2.3943661971830985,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 1.3360053300857544,
+      "mean_token_accuracy": 0.7052380293607712,
+      "num_tokens": 352317.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.3870916664600372,
+      "epoch": 2.408450704225352,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.4269676208496094,
+      "mean_token_accuracy": 0.6933766007423401,
+      "num_tokens": 354187.0,
+      "step": 171
+    },
+    {
+      "entropy": 1.4595828652381897,
+      "epoch": 2.4225352112676055,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.4954618215560913,
+      "mean_token_accuracy": 0.6528928130865097,
+      "num_tokens": 356527.0,
+      "step": 172
+    },
+    {
+      "entropy": 1.5412043631076813,
+      "epoch": 2.436619718309859,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.5568714141845703,
+      "mean_token_accuracy": 0.6727624386548996,
+      "num_tokens": 358804.0,
+      "step": 173
+    },
+    {
+      "entropy": 1.4133048951625824,
+      "epoch": 2.4507042253521125,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.4476163387298584,
+      "mean_token_accuracy": 0.6771093457937241,
+      "num_tokens": 361168.0,
+      "step": 174
+    },
+    {
+      "entropy": 1.5175727903842926,
+      "epoch": 2.464788732394366,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 1.6465704441070557,
+      "mean_token_accuracy": 0.6511463969945908,
+      "num_tokens": 363307.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.7925034761428833,
+      "epoch": 2.4788732394366195,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.8684073686599731,
+      "mean_token_accuracy": 0.6113807708024979,
+      "num_tokens": 365159.0,
+      "step": 176
+    },
+    {
+      "entropy": 1.6575580835342407,
+      "epoch": 2.492957746478873,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 1.6698346138000488,
+      "mean_token_accuracy": 0.6580274552106857,
+      "num_tokens": 366815.0,
+      "step": 177
+    },
+    {
+      "entropy": 1.1517049968242645,
+      "epoch": 2.507042253521127,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.0882394313812256,
+      "mean_token_accuracy": 0.7630608826875687,
+      "num_tokens": 368716.0,
+      "step": 178
+    },
+    {
+      "entropy": 1.4394071251153946,
+      "epoch": 2.52112676056338,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.5650489330291748,
+      "mean_token_accuracy": 0.6978088915348053,
+      "num_tokens": 370762.0,
+      "step": 179
+    },
+    {
+      "entropy": 1.329190582036972,
+      "epoch": 2.535211267605634,
+      "grad_norm": 7.71875,
+      "learning_rate": 5e-05,
+      "loss": 1.339747667312622,
+      "mean_token_accuracy": 0.6991136074066162,
+      "num_tokens": 372772.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.4634642452001572,
+      "epoch": 2.5492957746478875,
+      "grad_norm": 7.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.3990017175674438,
+      "mean_token_accuracy": 0.6824762970209122,
+      "num_tokens": 375270.0,
+      "step": 181
+    },
+    {
+      "entropy": 1.440472275018692,
+      "epoch": 2.563380281690141,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 1.416210412979126,
+      "mean_token_accuracy": 0.7090055644512177,
+      "num_tokens": 377207.0,
+      "step": 182
+    },
+    {
+      "entropy": 1.5881553888320923,
+      "epoch": 2.5774647887323945,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.5490307807922363,
+      "mean_token_accuracy": 0.6557108908891678,
+      "num_tokens": 379285.0,
+      "step": 183
+    },
+    {
+      "entropy": 1.8855535686016083,
+      "epoch": 2.591549295774648,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.8609915971755981,
+      "mean_token_accuracy": 0.6001175194978714,
+      "num_tokens": 381931.0,
+      "step": 184
+    },
+    {
+      "entropy": 1.5735717117786407,
+      "epoch": 2.6056338028169015,
+      "grad_norm": 8.875,
+      "learning_rate": 5e-05,
+      "loss": 1.5267468690872192,
+      "mean_token_accuracy": 0.6682861894369125,
+      "num_tokens": 383646.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.5233647227287292,
+      "epoch": 2.619718309859155,
+      "grad_norm": 10.0,
+      "learning_rate": 5e-05,
+      "loss": 1.5220509767532349,
+      "mean_token_accuracy": 0.6776503473520279,
+      "num_tokens": 385375.0,
+      "step": 186
+    },
+    {
+      "entropy": 1.8002847731113434,
+      "epoch": 2.6338028169014085,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.6597533226013184,
+      "mean_token_accuracy": 0.6361639946699142,
+      "num_tokens": 387083.0,
+      "step": 187
+    },
+    {
+      "entropy": 1.4071598052978516,
+      "epoch": 2.647887323943662,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.3929013013839722,
+      "mean_token_accuracy": 0.6784840375185013,
+      "num_tokens": 389755.0,
+      "step": 188
+    },
+    {
+      "entropy": 1.5969584584236145,
+      "epoch": 2.6619718309859155,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.5413727760314941,
+      "mean_token_accuracy": 0.6579905152320862,
+      "num_tokens": 392159.0,
+      "step": 189
+    },
+    {
+      "entropy": 1.4266707003116608,
+      "epoch": 2.676056338028169,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 1.4889869689941406,
+      "mean_token_accuracy": 0.6940827667713165,
+      "num_tokens": 394318.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.3000743389129639,
+      "epoch": 2.6901408450704225,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.2992713451385498,
+      "mean_token_accuracy": 0.7003718316555023,
+      "num_tokens": 396683.0,
+      "step": 191
+    },
+    {
+      "entropy": 1.454126387834549,
+      "epoch": 2.704225352112676,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 1.5633444786071777,
+      "mean_token_accuracy": 0.674123615026474,
+      "num_tokens": 398548.0,
+      "step": 192
+    },
+    {
+      "entropy": 1.2735390067100525,
+      "epoch": 2.7183098591549295,
+      "grad_norm": 7.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.2833411693572998,
+      "mean_token_accuracy": 0.7046811580657959,
+      "num_tokens": 401214.0,
+      "step": 193
+    },
+    {
+      "entropy": 1.6532493829727173,
+      "epoch": 2.732394366197183,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.6411373615264893,
+      "mean_token_accuracy": 0.6505441665649414,
+      "num_tokens": 403386.0,
+      "step": 194
+    },
+    {
+      "entropy": 1.3735451996326447,
+      "epoch": 2.7464788732394365,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.3553458452224731,
+      "mean_token_accuracy": 0.6837523132562637,
+      "num_tokens": 405696.0,
+      "step": 195
+    },
+    {
+      "entropy": 2.081565737724304,
+      "epoch": 2.76056338028169,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9327874183654785,
+      "mean_token_accuracy": 0.5979610085487366,
+      "num_tokens": 407659.0,
+      "step": 196
+    },
+    {
+      "entropy": 1.9338420629501343,
+      "epoch": 2.7746478873239435,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.8871653079986572,
+      "mean_token_accuracy": 0.5969646275043488,
+      "num_tokens": 410057.0,
+      "step": 197
+    },
+    {
+      "entropy": 1.5678223371505737,
+      "epoch": 2.788732394366197,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.6835041046142578,
+      "mean_token_accuracy": 0.6561465561389923,
+      "num_tokens": 412345.0,
+      "step": 198
+    },
+    {
+      "entropy": 1.349662259221077,
+      "epoch": 2.802816901408451,
+      "grad_norm": 8.5,
+      "learning_rate": 5e-05,
+      "loss": 1.4067068099975586,
+      "mean_token_accuracy": 0.6867142766714096,
+      "num_tokens": 414435.0,
+      "step": 199
+    },
+    {
+      "entropy": 1.4967040419578552,
+      "epoch": 2.816901408450704,
+      "grad_norm": 10.125,
+      "learning_rate": 5e-05,
+      "loss": 1.535017490386963,
+      "mean_token_accuracy": 0.6887214332818985,
+      "num_tokens": 416573.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.580811321735382,
+      "epoch": 2.830985915492958,
+      "grad_norm": 7.625,
+      "learning_rate": 5e-05,
+      "loss": 1.5120457410812378,
+      "mean_token_accuracy": 0.6450829207897186,
+      "num_tokens": 419194.0,
+      "step": 201
+    },
+    {
+      "entropy": 1.5417903065681458,
+      "epoch": 2.845070422535211,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.5814214944839478,
+      "mean_token_accuracy": 0.6863117665052414,
+      "num_tokens": 421333.0,
+      "step": 202
+    },
+    {
+      "entropy": 1.6860767006874084,
+      "epoch": 2.859154929577465,
+      "grad_norm": 8.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.757788896560669,
+      "mean_token_accuracy": 0.6364521831274033,
+      "num_tokens": 423578.0,
+      "step": 203
+    },
+    {
+      "entropy": 1.5526399314403534,
+      "epoch": 2.873239436619718,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.5645313262939453,
+      "mean_token_accuracy": 0.660129725933075,
+      "num_tokens": 426228.0,
+      "step": 204
+    },
+    {
+      "entropy": 1.574429988861084,
+      "epoch": 2.887323943661972,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.4116606712341309,
+      "mean_token_accuracy": 0.67585289478302,
+      "num_tokens": 428377.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.5494361817836761,
+      "epoch": 2.9014084507042255,
+      "grad_norm": 10.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.4719582796096802,
+      "mean_token_accuracy": 0.6877067238092422,
+      "num_tokens": 430248.0,
+      "step": 206
+    },
+    {
+      "entropy": 1.4029511511325836,
+      "epoch": 2.915492957746479,
+      "grad_norm": 9.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.3776860237121582,
+      "mean_token_accuracy": 0.6826284527778625,
+      "num_tokens": 432195.0,
+      "step": 207
+    },
+    {
+      "entropy": 1.3329160958528519,
+      "epoch": 2.9295774647887325,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.4112855195999146,
+      "mean_token_accuracy": 0.710347130894661,
+      "num_tokens": 433833.0,
+      "step": 208
+    },
+    {
+      "entropy": 1.3620904386043549,
+      "epoch": 2.943661971830986,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.4375736713409424,
+      "mean_token_accuracy": 0.66610486805439,
+      "num_tokens": 435846.0,
+      "step": 209
+    },
+    {
+      "entropy": 1.3039760887622833,
+      "epoch": 2.9577464788732395,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.4245851039886475,
+      "mean_token_accuracy": 0.6863886713981628,
+      "num_tokens": 437817.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.3174594566226006,
+      "epoch": 2.971830985915493,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.4897089004516602,
+      "mean_token_accuracy": 0.7268126755952835,
+      "num_tokens": 439001.0,
+      "step": 211
+    },
+    {
+      "entropy": 1.597790390253067,
+      "epoch": 2.9859154929577465,
+      "grad_norm": 10.0,
+      "learning_rate": 5e-05,
+      "loss": 1.637603998184204,
+      "mean_token_accuracy": 0.6423007696866989,
+      "num_tokens": 441067.0,
+      "step": 212
+    },
+    {
+      "entropy": 1.4755942821502686,
+      "epoch": 3.0,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.566728115081787,
+      "mean_token_accuracy": 0.6867347657680511,
+      "num_tokens": 442950.0,
+      "step": 213
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 1.742875314934153,
+      "eval_loss": 2.3794198036193848,
+      "eval_mean_token_accuracy": 0.5764347096564064,
+      "eval_num_tokens": 442950.0,
+      "eval_runtime": 18.96,
+      "eval_samples_per_second": 14.979,
+      "eval_steps_per_second": 3.745,
+      "step": 213
+    },
+    {
+      "entropy": 1.4477737545967102,
+      "epoch": 3.0140845070422535,
+      "grad_norm": 9.0,
+      "learning_rate": 5e-05,
+      "loss": 1.34397292137146,
+      "mean_token_accuracy": 0.7147080153226852,
+      "num_tokens": 445009.0,
+      "step": 214
+    },
+    {
+      "entropy": 1.203942894935608,
+      "epoch": 3.028169014084507,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 0.9816666841506958,
+      "mean_token_accuracy": 0.7748381495475769,
+      "num_tokens": 447153.0,
+      "step": 215
+    },
+    {
+      "entropy": 1.5787488222122192,
+      "epoch": 3.0422535211267605,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.270491600036621,
+      "mean_token_accuracy": 0.7053533643484116,
+      "num_tokens": 449145.0,
+      "step": 216
+    },
+    {
+      "entropy": 1.4443465173244476,
+      "epoch": 3.056338028169014,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.3180813789367676,
+      "mean_token_accuracy": 0.7140058428049088,
+      "num_tokens": 451094.0,
+      "step": 217
+    },
+    {
+      "entropy": 1.5492436438798904,
+      "epoch": 3.0704225352112675,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.3129041194915771,
+      "mean_token_accuracy": 0.7110538259148598,
+      "num_tokens": 452780.0,
+      "step": 218
+    },
+    {
+      "entropy": 1.2075490355491638,
+      "epoch": 3.084507042253521,
+      "grad_norm": 10.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.2063069343566895,
+      "mean_token_accuracy": 0.7200463265180588,
+      "num_tokens": 454790.0,
+      "step": 219
+    },
+    {
+      "entropy": 1.2461529076099396,
+      "epoch": 3.0985915492957745,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 1.1845942735671997,
+      "mean_token_accuracy": 0.7373121082782745,
+      "num_tokens": 456807.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.150038331747055,
+      "epoch": 3.112676056338028,
+      "grad_norm": 10.625,
+      "learning_rate": 5e-05,
+      "loss": 1.1360472440719604,
+      "mean_token_accuracy": 0.7540087252855301,
+      "num_tokens": 458710.0,
+      "step": 221
+    },
+    {
+      "entropy": 1.5197363197803497,
+      "epoch": 3.1267605633802815,
+      "grad_norm": 11.125,
+      "learning_rate": 5e-05,
+      "loss": 1.571465015411377,
+      "mean_token_accuracy": 0.6653782874345779,
+      "num_tokens": 460644.0,
+      "step": 222
+    },
+    {
+      "entropy": 1.2971323430538177,
+      "epoch": 3.140845070422535,
+      "grad_norm": 10.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.46250319480896,
+      "mean_token_accuracy": 0.6934572160243988,
+      "num_tokens": 462991.0,
+      "step": 223
+    },
+    {
+      "entropy": 1.0633235424757004,
+      "epoch": 3.1549295774647885,
+      "grad_norm": 10.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.1144837141036987,
+      "mean_token_accuracy": 0.7421760857105255,
+      "num_tokens": 465355.0,
+      "step": 224
+    },
+    {
+      "entropy": 1.211214318871498,
+      "epoch": 3.169014084507042,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.251718521118164,
+      "mean_token_accuracy": 0.7310685962438583,
+      "num_tokens": 467342.0,
+      "step": 225
+    },
+    {
+      "entropy": 1.2731620073318481,
+      "epoch": 3.183098591549296,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.3605529069900513,
+      "mean_token_accuracy": 0.7053728103637695,
+      "num_tokens": 469403.0,
+      "step": 226
+    },
+    {
+      "entropy": 1.0669567435979843,
+      "epoch": 3.1971830985915495,
+      "grad_norm": 10.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.1370317935943604,
+      "mean_token_accuracy": 0.769411489367485,
+      "num_tokens": 470658.0,
+      "step": 227
+    },
+    {
+      "entropy": 1.0720246136188507,
+      "epoch": 3.211267605633803,
+      "grad_norm": 11.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.1395004987716675,
+      "mean_token_accuracy": 0.7543797641992569,
+      "num_tokens": 472433.0,
+      "step": 228
+    },
+    {
+      "entropy": 1.6844939291477203,
+      "epoch": 3.2253521126760565,
+      "grad_norm": 11.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.691328525543213,
+      "mean_token_accuracy": 0.6531989872455597,
+      "num_tokens": 474324.0,
+      "step": 229
+    },
+    {
+      "entropy": 1.3551696240901947,
+      "epoch": 3.23943661971831,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.3602029085159302,
+      "mean_token_accuracy": 0.6766076982021332,
+      "num_tokens": 477006.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.081579566001892,
+      "epoch": 3.2535211267605635,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 1.0683220624923706,
+      "mean_token_accuracy": 0.7595697492361069,
+      "num_tokens": 479214.0,
+      "step": 231
+    },
+    {
+      "entropy": 0.9538451135158539,
+      "epoch": 3.267605633802817,
+      "grad_norm": 7.75,
+      "learning_rate": 5e-05,
+      "loss": 0.935756266117096,
+      "mean_token_accuracy": 0.7749744206666946,
+      "num_tokens": 481438.0,
+      "step": 232
+    },
+    {
+      "entropy": 1.195127934217453,
+      "epoch": 3.2816901408450705,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 1.1229381561279297,
+      "mean_token_accuracy": 0.7507655024528503,
+      "num_tokens": 483520.0,
+      "step": 233
+    },
+    {
+      "entropy": 1.4514926671981812,
+      "epoch": 3.295774647887324,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.409381628036499,
+      "mean_token_accuracy": 0.701470211148262,
+      "num_tokens": 485686.0,
+      "step": 234
+    },
+    {
+      "entropy": 1.6824813485145569,
+      "epoch": 3.3098591549295775,
+      "grad_norm": 10.125,
+      "learning_rate": 5e-05,
+      "loss": 1.4787794351577759,
+      "mean_token_accuracy": 0.6804617792367935,
+      "num_tokens": 487486.0,
+      "step": 235
+    },
+    {
+      "entropy": 1.413738265633583,
+      "epoch": 3.323943661971831,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 1.4716837406158447,
+      "mean_token_accuracy": 0.6935473531484604,
+      "num_tokens": 489588.0,
+      "step": 236
+    },
+    {
+      "entropy": 1.5309541523456573,
+      "epoch": 3.3380281690140845,
+      "grad_norm": 8.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.5352935791015625,
+      "mean_token_accuracy": 0.6691045314073563,
+      "num_tokens": 492236.0,
+      "step": 237
+    },
+    {
+      "entropy": 1.100858986377716,
+      "epoch": 3.352112676056338,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 1.0833566188812256,
+      "mean_token_accuracy": 0.753660261631012,
+      "num_tokens": 494406.0,
+      "step": 238
+    },
+    {
+      "entropy": 1.0815290808677673,
+      "epoch": 3.3661971830985915,
+      "grad_norm": 11.5,
+      "learning_rate": 5e-05,
+      "loss": 1.1363320350646973,
+      "mean_token_accuracy": 0.7735187858343124,
+      "num_tokens": 496057.0,
+      "step": 239
+    },
+    {
+      "entropy": 1.286641076207161,
+      "epoch": 3.380281690140845,
+      "grad_norm": 12.0,
+      "learning_rate": 5e-05,
+      "loss": 1.5121028423309326,
+      "mean_token_accuracy": 0.7167779356241226,
+      "num_tokens": 497871.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.8904624283313751,
+      "epoch": 3.3943661971830985,
+      "grad_norm": 11.1875,
+      "learning_rate": 5e-05,
+      "loss": 1.761491298675537,
+      "mean_token_accuracy": 0.6360357776284218,
+      "num_tokens": 499402.0,
+      "step": 241
+    },
+    {
+      "entropy": 1.3357311487197876,
+      "epoch": 3.408450704225352,
+      "grad_norm": 11.0,
+      "learning_rate": 5e-05,
+      "loss": 1.4104362726211548,
+      "mean_token_accuracy": 0.7005175352096558,
+      "num_tokens": 501478.0,
+      "step": 242
+    },
+    {
+      "entropy": 1.2808267772197723,
+      "epoch": 3.4225352112676055,
+      "grad_norm": 11.25,
+      "learning_rate": 5e-05,
+      "loss": 1.266588568687439,
+      "mean_token_accuracy": 0.7067604064941406,
+      "num_tokens": 503790.0,
+      "step": 243
+    },
+    {
+      "entropy": 1.3459688425064087,
+      "epoch": 3.436619718309859,
+      "grad_norm": 13.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.4315173625946045,
+      "mean_token_accuracy": 0.685220867395401,
+      "num_tokens": 505707.0,
+      "step": 244
+    },
+    {
+      "entropy": 1.1801058650016785,
+      "epoch": 3.4507042253521125,
+      "grad_norm": 10.125,
+      "learning_rate": 5e-05,
+      "loss": 1.1691138744354248,
+      "mean_token_accuracy": 0.735442042350769,
+      "num_tokens": 507931.0,
+      "step": 245
+    },
+    {
+      "entropy": 1.0448043197393417,
+      "epoch": 3.464788732394366,
+      "grad_norm": 11.375,
+      "learning_rate": 5e-05,
+      "loss": 1.108785629272461,
+      "mean_token_accuracy": 0.7781961262226105,
+      "num_tokens": 509377.0,
+      "step": 246
+    },
+    {
+      "entropy": 0.94132499396801,
+      "epoch": 3.4788732394366195,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 0.9574054479598999,
+      "mean_token_accuracy": 0.7875233143568039,
+      "num_tokens": 511026.0,
+      "step": 247
+    },
+    {
+      "entropy": 1.2185545563697815,
+      "epoch": 3.492957746478873,
+      "grad_norm": 10.25,
+      "learning_rate": 5e-05,
+      "loss": 1.1727116107940674,
+      "mean_token_accuracy": 0.7297279387712479,
+      "num_tokens": 513357.0,
+      "step": 248
+    },
+    {
+      "entropy": 1.4287467002868652,
+      "epoch": 3.507042253521127,
+      "grad_norm": 11.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.4134924411773682,
+      "mean_token_accuracy": 0.6829686760902405,
+      "num_tokens": 515279.0,
+      "step": 249
+    },
+    {
+      "entropy": 1.33033886551857,
+      "epoch": 3.52112676056338,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 1.2814573049545288,
+      "mean_token_accuracy": 0.7182295322418213,
+      "num_tokens": 517068.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1580873131752014,
+      "epoch": 3.535211267605634,
+      "grad_norm": 11.0,
+      "learning_rate": 5e-05,
+      "loss": 1.1515038013458252,
+      "mean_token_accuracy": 0.7479496151208878,
+      "num_tokens": 519048.0,
+      "step": 251
+    },
+    {
+      "entropy": 1.0777922421693802,
+      "epoch": 3.5492957746478875,
+      "grad_norm": 10.125,
+      "learning_rate": 5e-05,
+      "loss": 1.0803577899932861,
+      "mean_token_accuracy": 0.7433650195598602,
+      "num_tokens": 521042.0,
+      "step": 252
+    },
+    {
+      "entropy": 1.4103947579860687,
+      "epoch": 3.563380281690141,
+      "grad_norm": 11.75,
+      "learning_rate": 5e-05,
+      "loss": 1.364109992980957,
+      "mean_token_accuracy": 0.6804712861776352,
+      "num_tokens": 523330.0,
+      "step": 253
+    },
+    {
+      "entropy": 1.5412517488002777,
+      "epoch": 3.5774647887323945,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.5343971252441406,
+      "mean_token_accuracy": 0.6705016940832138,
+      "num_tokens": 525028.0,
+      "step": 254
+    },
+    {
+      "entropy": 1.2664999663829803,
+      "epoch": 3.591549295774648,
+      "grad_norm": 11.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.2529462575912476,
+      "mean_token_accuracy": 0.7088112682104111,
+      "num_tokens": 527594.0,
+      "step": 255
+    },
+    {
+      "entropy": 1.3230702579021454,
+      "epoch": 3.6056338028169015,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 1.1759625673294067,
+      "mean_token_accuracy": 0.7173673510551453,
+      "num_tokens": 530285.0,
+      "step": 256
+    },
+    {
+      "entropy": 1.1666917651891708,
+      "epoch": 3.619718309859155,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.1413341760635376,
+      "mean_token_accuracy": 0.7238663733005524,
+      "num_tokens": 533439.0,
+      "step": 257
+    },
+    {
+      "entropy": 1.34586563706398,
+      "epoch": 3.6338028169014085,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 1.2202277183532715,
+      "mean_token_accuracy": 0.7310525625944138,
+      "num_tokens": 535277.0,
+      "step": 258
+    },
+    {
+      "entropy": 1.1511282920837402,
+      "epoch": 3.647887323943662,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.1675046682357788,
+      "mean_token_accuracy": 0.7271726727485657,
+      "num_tokens": 537884.0,
+      "step": 259
+    },
+    {
+      "entropy": 1.1263611614704132,
+      "epoch": 3.6619718309859155,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.179438591003418,
+      "mean_token_accuracy": 0.7631877958774567,
+      "num_tokens": 539525.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.272725835442543,
+      "epoch": 3.676056338028169,
+      "grad_norm": 10.125,
+      "learning_rate": 5e-05,
+      "loss": 1.279167652130127,
+      "mean_token_accuracy": 0.7252255082130432,
+      "num_tokens": 541719.0,
+      "step": 261
+    },
+    {
+      "entropy": 1.546060025691986,
+      "epoch": 3.6901408450704225,
+      "grad_norm": 12.0,
+      "learning_rate": 5e-05,
+      "loss": 1.564497709274292,
+      "mean_token_accuracy": 0.6560944467782974,
+      "num_tokens": 544237.0,
+      "step": 262
+    },
+    {
+      "entropy": 1.0800508558750153,
+      "epoch": 3.704225352112676,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 1.1466948986053467,
+      "mean_token_accuracy": 0.7520655542612076,
+      "num_tokens": 546210.0,
+      "step": 263
+    },
+    {
+      "entropy": 1.3623918890953064,
+      "epoch": 3.7183098591549295,
+      "grad_norm": 10.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.3423826694488525,
+      "mean_token_accuracy": 0.7056058645248413,
+      "num_tokens": 548094.0,
+      "step": 264
+    },
+    {
+      "entropy": 1.399181753396988,
+      "epoch": 3.732394366197183,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 1.4637426137924194,
+      "mean_token_accuracy": 0.6957306265830994,
+      "num_tokens": 550057.0,
+      "step": 265
+    },
+    {
+      "entropy": 1.1494918912649155,
+      "epoch": 3.7464788732394365,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.188775897026062,
+      "mean_token_accuracy": 0.7327060103416443,
+      "num_tokens": 552414.0,
+      "step": 266
+    },
+    {
+      "entropy": 1.1738183051347733,
+      "epoch": 3.76056338028169,
+      "grad_norm": 10.125,
+      "learning_rate": 5e-05,
+      "loss": 1.0964633226394653,
+      "mean_token_accuracy": 0.7524174004793167,
+      "num_tokens": 554230.0,
+      "step": 267
+    },
+    {
+      "entropy": 1.4064239263534546,
+      "epoch": 3.7746478873239435,
+      "grad_norm": 11.625,
+      "learning_rate": 5e-05,
+      "loss": 1.3753869533538818,
+      "mean_token_accuracy": 0.6939396262168884,
+      "num_tokens": 556471.0,
+      "step": 268
+    },
+    {
+      "entropy": 1.7035263180732727,
+      "epoch": 3.788732394366197,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 1.6034808158874512,
+      "mean_token_accuracy": 0.6561361849308014,
+      "num_tokens": 558374.0,
+      "step": 269
+    },
+    {
+      "entropy": 1.1388080418109894,
+      "epoch": 3.802816901408451,
+      "grad_norm": 9.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.143500804901123,
+      "mean_token_accuracy": 0.7669709026813507,
+      "num_tokens": 559899.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.314527541399002,
+      "epoch": 3.816901408450704,
+      "grad_norm": 11.625,
+      "learning_rate": 5e-05,
+      "loss": 1.2044060230255127,
+      "mean_token_accuracy": 0.7112587243318558,
+      "num_tokens": 561730.0,
+      "step": 271
+    },
+    {
+      "entropy": 1.6000914573669434,
+      "epoch": 3.830985915492958,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 1.4867759943008423,
+      "mean_token_accuracy": 0.6724018305540085,
+      "num_tokens": 563776.0,
+      "step": 272
+    },
+    {
+      "entropy": 1.0925718545913696,
+      "epoch": 3.845070422535211,
+      "grad_norm": 10.0,
+      "learning_rate": 5e-05,
+      "loss": 1.1063774824142456,
+      "mean_token_accuracy": 0.7417843043804169,
+      "num_tokens": 565957.0,
+      "step": 273
+    },
+    {
+      "entropy": 1.4033282101154327,
+      "epoch": 3.859154929577465,
+      "grad_norm": 9.625,
+      "learning_rate": 5e-05,
+      "loss": 1.347365140914917,
+      "mean_token_accuracy": 0.6811056584119797,
+      "num_tokens": 568649.0,
+      "step": 274
+    },
+    {
+      "entropy": 1.0864711850881577,
+      "epoch": 3.873239436619718,
+      "grad_norm": 10.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.1229323148727417,
+      "mean_token_accuracy": 0.7628998160362244,
+      "num_tokens": 570311.0,
+      "step": 275
+    },
+    {
+      "entropy": 1.3399913907051086,
+      "epoch": 3.887323943661972,
+      "grad_norm": 10.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.341964840888977,
+      "mean_token_accuracy": 0.712143212556839,
+      "num_tokens": 572240.0,
+      "step": 276
+    },
+    {
+      "entropy": 0.9606586545705795,
+      "epoch": 3.9014084507042255,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 0.9399880170822144,
+      "mean_token_accuracy": 0.7931267023086548,
+      "num_tokens": 573718.0,
+      "step": 277
+    },
+    {
+      "entropy": 1.5906201899051666,
+      "epoch": 3.915492957746479,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 1.5526992082595825,
+      "mean_token_accuracy": 0.6785695552825928,
+      "num_tokens": 575547.0,
+      "step": 278
+    },
+    {
+      "entropy": 1.3323252499103546,
+      "epoch": 3.9295774647887325,
+      "grad_norm": 9.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.261247158050537,
+      "mean_token_accuracy": 0.7029483765363693,
+      "num_tokens": 577890.0,
+      "step": 279
+    },
+    {
+      "entropy": 1.3561639338731766,
+      "epoch": 3.943661971830986,
+      "grad_norm": 11.625,
+      "learning_rate": 5e-05,
+      "loss": 1.3369463682174683,
+      "mean_token_accuracy": 0.7009438425302505,
+      "num_tokens": 580036.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.2692861258983612,
+      "epoch": 3.9577464788732395,
+      "grad_norm": 9.6875,
+      "learning_rate": 5e-05,
+      "loss": 1.2595934867858887,
+      "mean_token_accuracy": 0.6921688318252563,
+      "num_tokens": 582633.0,
+      "step": 281
+    },
+    {
+      "entropy": 1.2151632905006409,
+      "epoch": 3.971830985915493,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 1.2210090160369873,
+      "mean_token_accuracy": 0.7068206667900085,
+      "num_tokens": 585083.0,
+      "step": 282
+    },
+    {
+      "entropy": 1.551920771598816,
+      "epoch": 3.9859154929577465,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.5452065467834473,
+      "mean_token_accuracy": 0.6577051281929016,
+      "num_tokens": 587674.0,
+      "step": 283
+    },
+    {
+      "entropy": 1.184539407491684,
+      "epoch": 4.0,
+      "grad_norm": 9.5,
+      "learning_rate": 5e-05,
+      "loss": 1.2013723850250244,
+      "mean_token_accuracy": 0.7067449688911438,
+      "num_tokens": 590600.0,
+      "step": 284
+    },
+    {
+      "epoch": 4.0,
+      "eval_entropy": 1.4757034711434807,
+      "eval_loss": 2.6069748401641846,
+      "eval_mean_token_accuracy": 0.5698243828726487,
+      "eval_num_tokens": 590600.0,
+      "eval_runtime": 18.85,
+      "eval_samples_per_second": 15.066,
+      "eval_steps_per_second": 3.767,
+      "step": 284
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 284,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 617869407353856.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-284/training_args.bin b/checkpoint-284/training_args.bin
new file mode 100644
index 0000000..0b099d6
--- /dev/null
+++ b/checkpoint-284/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
+size 5585
diff --git a/checkpoint-71/chat_template.jinja b/checkpoint-71/chat_template.jinja
new file mode 100644
index 0000000..1117055
--- /dev/null
+++ b/checkpoint-71/chat_template.jinja
@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}
diff --git a/checkpoint-71/config.json b/checkpoint-71/config.json
new file mode 100644
index 0000000..436e93d
--- /dev/null
+++ b/checkpoint-71/config.json
@@ -0,0 +1,62 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 512,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_bidirectional_attention": false,
+  "use_cache": false,
+  "vocab_size": 262144
+}
diff --git a/checkpoint-71/generation_config.json b/checkpoint-71/generation_config.json
new file mode 100644
index 0000000..c652aad
--- /dev/null
+++ b/checkpoint-71/generation_config.json
@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.0.0"
+}
diff --git a/checkpoint-71/model.safetensors b/checkpoint-71/model.safetensors
new file mode 100644
index 0000000..5fb0d37
--- /dev/null
+++ b/checkpoint-71/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce662878ad6e9761f44d14c64b8ad018edf716822dba4c94e827070456a6a17b
+size 536223056
diff --git a/checkpoint-71/optimizer.pt b/checkpoint-71/optimizer.pt
new file mode 100644
index 0000000..75ce0b2
--- /dev/null
+++ b/checkpoint-71/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a90c0c913beec5aee54988304a77e9b75b4259d9d1d6af43ff219239b2b331c
+size 1072591115
diff --git a/checkpoint-71/rng_state.pth b/checkpoint-71/rng_state.pth
new file mode 100644
index 0000000..1feba1a
--- /dev/null
+++ b/checkpoint-71/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+size 14645
diff --git a/checkpoint-71/scheduler.pt b/checkpoint-71/scheduler.pt
new file mode 100644
index 0000000..f8be3a2
--- /dev/null
+++ b/checkpoint-71/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b6cac40310cacf8cbaf2f5446f5795764e3a9cd1001f02bd65f1eb2fb5fd2ed
+size 1465
diff --git a/checkpoint-71/tokenizer.json b/checkpoint-71/tokenizer.json
new file mode 100644
index 0000000..899af07
--- /dev/null
+++ b/checkpoint-71/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
+size 33384443
diff --git a/checkpoint-71/tokenizer_config.json b/checkpoint-71/tokenizer_config.json
new file mode 100644
index 0000000..7202584
--- /dev/null
+++ b/checkpoint-71/tokenizer_config.json
@@ -0,0 +1,24 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}
diff --git a/checkpoint-71/trainer_state.json b/checkpoint-71/trainer_state.json
new file mode 100644
index 0000000..ceb7590
--- /dev/null
+++ b/checkpoint-71/trainer_state.json
@@ -0,0 +1,755 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 71,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4938980638980865,
+      "epoch": 0.014084507042253521,
+      "grad_norm": 89.5,
+      "learning_rate": 5e-05,
+      "loss": 4.519379615783691,
+      "mean_token_accuracy": 0.3811451345682144,
+      "num_tokens": 2953.0,
+      "step": 1
+    },
+    {
+      "entropy": 3.6362282037734985,
+      "epoch": 0.028169014084507043,
+      "grad_norm": 29.125,
+      "learning_rate": 5e-05,
+      "loss": 3.7604172229766846,
+      "mean_token_accuracy": 0.39103876799345016,
+      "num_tokens": 5196.0,
+      "step": 2
+    },
+    {
+      "entropy": 3.796808362007141,
+      "epoch": 0.04225352112676056,
+      "grad_norm": 18.75,
+      "learning_rate": 5e-05,
+      "loss": 3.3663859367370605,
+      "mean_token_accuracy": 0.42154909670352936,
+      "num_tokens": 7537.0,
+      "step": 3
+    },
+    {
+      "entropy": 3.071028172969818,
+      "epoch": 0.056338028169014086,
+      "grad_norm": 14.5625,
+      "learning_rate": 5e-05,
+      "loss": 3.1916425228118896,
+      "mean_token_accuracy": 0.4548730403184891,
+      "num_tokens": 9584.0,
+      "step": 4
+    },
+    {
+      "entropy": 2.7734431624412537,
+      "epoch": 0.07042253521126761,
+      "grad_norm": 15.0,
+      "learning_rate": 5e-05,
+      "loss": 2.661928653717041,
+      "mean_token_accuracy": 0.5473615825176239,
+      "num_tokens": 11344.0,
+      "step": 5
+    },
+    {
+      "entropy": 2.4827571511268616,
+      "epoch": 0.08450704225352113,
+      "grad_norm": 13.6875,
+      "learning_rate": 5e-05,
+      "loss": 2.691732168197632,
+      "mean_token_accuracy": 0.5095682442188263,
+      "num_tokens": 13289.0,
+      "step": 6
+    },
+    {
+      "entropy": 2.6064285039901733,
+      "epoch": 0.09859154929577464,
+      "grad_norm": 12.625,
+      "learning_rate": 5e-05,
+      "loss": 2.7455577850341797,
+      "mean_token_accuracy": 0.5099757686257362,
+      "num_tokens": 15664.0,
+      "step": 7
+    },
+    {
+      "entropy": 2.4781495332717896,
+      "epoch": 0.11267605633802817,
+      "grad_norm": 12.75,
+      "learning_rate": 5e-05,
+      "loss": 2.5589702129364014,
+      "mean_token_accuracy": 0.5618840456008911,
+      "num_tokens": 17331.0,
+      "step": 8
+    },
+    {
+      "entropy": 2.512305438518524,
+      "epoch": 0.1267605633802817,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.5893146991729736,
+      "mean_token_accuracy": 0.5188388600945473,
+      "num_tokens": 19724.0,
+      "step": 9
+    },
+    {
+      "entropy": 2.187403589487076,
+      "epoch": 0.14084507042253522,
+      "grad_norm": 11.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4727907180786133,
+      "mean_token_accuracy": 0.6085146963596344,
+      "num_tokens": 21334.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.8956857919692993,
+      "epoch": 0.15492957746478872,
+      "grad_norm": 10.1875,
+      "learning_rate": 5e-05,
+      "loss": 2.8553805351257324,
+      "mean_token_accuracy": 0.4754949361085892,
+      "num_tokens": 23770.0,
+      "step": 11
+    },
+    {
+      "entropy": 2.4213987588882446,
+      "epoch": 0.16901408450704225,
+      "grad_norm": 9.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4139676094055176,
+      "mean_token_accuracy": 0.5475910231471062,
+      "num_tokens": 26233.0,
+      "step": 12
+    },
+    {
+      "entropy": 2.668797016143799,
+      "epoch": 0.18309859154929578,
+      "grad_norm": 12.25,
+      "learning_rate": 5e-05,
+      "loss": 2.769174337387085,
+      "mean_token_accuracy": 0.5351328551769257,
+      "num_tokens": 27692.0,
+      "step": 13
+    },
+    {
+      "entropy": 2.544685661792755,
+      "epoch": 0.19718309859154928,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.4870781898498535,
+      "mean_token_accuracy": 0.514883853495121,
+      "num_tokens": 30217.0,
+      "step": 14
+    },
+    {
+      "entropy": 2.5978153944015503,
+      "epoch": 0.2112676056338028,
+      "grad_norm": 10.875,
+      "learning_rate": 5e-05,
+      "loss": 2.639450788497925,
+      "mean_token_accuracy": 0.5468774288892746,
+      "num_tokens": 31969.0,
+      "step": 15
+    },
+    {
+      "entropy": 2.452792704105377,
+      "epoch": 0.22535211267605634,
+      "grad_norm": 11.5,
+      "learning_rate": 5e-05,
+      "loss": 2.5285415649414062,
+      "mean_token_accuracy": 0.5533104464411736,
+      "num_tokens": 34160.0,
+      "step": 16
+    },
+    {
+      "entropy": 2.234907388687134,
+      "epoch": 0.23943661971830985,
+      "grad_norm": 12.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.2553577423095703,
+      "mean_token_accuracy": 0.5958019495010376,
+      "num_tokens": 35899.0,
+      "step": 17
+    },
+    {
+      "entropy": 2.2781192660331726,
+      "epoch": 0.2535211267605634,
+      "grad_norm": 10.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4401230812072754,
+      "mean_token_accuracy": 0.5632342845201492,
+      "num_tokens": 37693.0,
+      "step": 18
+    },
+    {
+      "entropy": 2.292062520980835,
+      "epoch": 0.2676056338028169,
+      "grad_norm": 10.5,
+      "learning_rate": 5e-05,
+      "loss": 2.53348445892334,
+      "mean_token_accuracy": 0.5441952645778656,
+      "num_tokens": 39793.0,
+      "step": 19
+    },
+    {
+      "entropy": 2.5250023007392883,
+      "epoch": 0.28169014084507044,
+      "grad_norm": 13.5,
+      "learning_rate": 5e-05,
+      "loss": 2.7173304557800293,
+      "mean_token_accuracy": 0.4843936040997505,
+      "num_tokens": 42035.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.3936269879341125,
+      "epoch": 0.29577464788732394,
+      "grad_norm": 10.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.397752046585083,
+      "mean_token_accuracy": 0.5632070526480675,
+      "num_tokens": 43606.0,
+      "step": 21
+    },
+    {
+      "entropy": 2.7215799689292908,
+      "epoch": 0.30985915492957744,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.7266643047332764,
+      "mean_token_accuracy": 0.4814189076423645,
+      "num_tokens": 46160.0,
+      "step": 22
+    },
+    {
+      "entropy": 2.4695653319358826,
+      "epoch": 0.323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3790149688720703,
+      "mean_token_accuracy": 0.5413414835929871,
+      "num_tokens": 48749.0,
+      "step": 23
+    },
+    {
+      "entropy": 2.3166028261184692,
+      "epoch": 0.3380281690140845,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.274803638458252,
+      "mean_token_accuracy": 0.5993590205907822,
+      "num_tokens": 50770.0,
+      "step": 24
+    },
+    {
+      "entropy": 2.285377472639084,
+      "epoch": 0.352112676056338,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.2308878898620605,
+      "mean_token_accuracy": 0.5828141123056412,
+      "num_tokens": 53004.0,
+      "step": 25
+    },
+    {
+      "entropy": 2.1157060265541077,
+      "epoch": 0.36619718309859156,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1033847332000732,
+      "mean_token_accuracy": 0.5867631435394287,
+      "num_tokens": 55108.0,
+      "step": 26
+    },
+    {
+      "entropy": 2.5209829807281494,
+      "epoch": 0.38028169014084506,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.3607499599456787,
+      "mean_token_accuracy": 0.5426046550273895,
+      "num_tokens": 57080.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.974183738231659,
+      "epoch": 0.39436619718309857,
+      "grad_norm": 7.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.9418365955352783,
+      "mean_token_accuracy": 0.6255736351013184,
+      "num_tokens": 59237.0,
+      "step": 28
+    },
+    {
+      "entropy": 2.252374231815338,
+      "epoch": 0.4084507042253521,
+      "grad_norm": 9.875,
+      "learning_rate": 5e-05,
+      "loss": 2.228358507156372,
+      "mean_token_accuracy": 0.5917280316352844,
+      "num_tokens": 60850.0,
+      "step": 29
+    },
+    {
+      "entropy": 2.327530801296234,
+      "epoch": 0.4225352112676056,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.3329389095306396,
+      "mean_token_accuracy": 0.5718153715133667,
+      "num_tokens": 62734.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.2620502710342407,
+      "epoch": 0.43661971830985913,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.1953892707824707,
+      "mean_token_accuracy": 0.5675001591444016,
+      "num_tokens": 64637.0,
+      "step": 31
+    },
+    {
+      "entropy": 2.445267140865326,
+      "epoch": 0.4507042253521127,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.501152276992798,
+      "mean_token_accuracy": 0.5355894565582275,
+      "num_tokens": 66880.0,
+      "step": 32
+    },
+    {
+      "entropy": 2.3873718976974487,
+      "epoch": 0.4647887323943662,
+      "grad_norm": 9.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.3468880653381348,
+      "mean_token_accuracy": 0.5372899621725082,
+      "num_tokens": 68623.0,
+      "step": 33
+    },
+    {
+      "entropy": 2.0545941591262817,
+      "epoch": 0.4788732394366197,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.1947593688964844,
+      "mean_token_accuracy": 0.6112199202179909,
+      "num_tokens": 70769.0,
+      "step": 34
+    },
+    {
+      "entropy": 2.350032150745392,
+      "epoch": 0.49295774647887325,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.418266773223877,
+      "mean_token_accuracy": 0.5104246437549591,
+      "num_tokens": 73653.0,
+      "step": 35
+    },
+    {
+      "entropy": 2.237062931060791,
+      "epoch": 0.5070422535211268,
+      "grad_norm": 10.75,
+      "learning_rate": 5e-05,
+      "loss": 2.4755356311798096,
+      "mean_token_accuracy": 0.5747656747698784,
+      "num_tokens": 75524.0,
+      "step": 36
+    },
+    {
+      "entropy": 2.0994813442230225,
+      "epoch": 0.5211267605633803,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 2.044315814971924,
+      "mean_token_accuracy": 0.6056438237428665,
+      "num_tokens": 77246.0,
+      "step": 37
+    },
+    {
+      "entropy": 2.5556156635284424,
+      "epoch": 0.5352112676056338,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.446904182434082,
+      "mean_token_accuracy": 0.5177429020404816,
+      "num_tokens": 79970.0,
+      "step": 38
+    },
+    {
+      "entropy": 2.337540030479431,
+      "epoch": 0.5492957746478874,
+      "grad_norm": 8.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.216331720352173,
+      "mean_token_accuracy": 0.58082115650177,
+      "num_tokens": 81876.0,
+      "step": 39
+    },
+    {
+      "entropy": 2.043498247861862,
+      "epoch": 0.5633802816901409,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 1.9643354415893555,
+      "mean_token_accuracy": 0.607293888926506,
+      "num_tokens": 84298.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.4852016270160675,
+      "epoch": 0.5774647887323944,
+      "grad_norm": 8.75,
+      "learning_rate": 5e-05,
+      "loss": 2.571437358856201,
+      "mean_token_accuracy": 0.5233406946063042,
+      "num_tokens": 86291.0,
+      "step": 41
+    },
+    {
+      "entropy": 2.5456899404525757,
+      "epoch": 0.5915492957746479,
+      "grad_norm": 9.125,
+      "learning_rate": 5e-05,
+      "loss": 2.4434070587158203,
+      "mean_token_accuracy": 0.5472028106451035,
+      "num_tokens": 88194.0,
+      "step": 42
+    },
+    {
+      "entropy": 2.2191686630249023,
+      "epoch": 0.6056338028169014,
+      "grad_norm": 9.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.225029468536377,
+      "mean_token_accuracy": 0.6006355881690979,
+      "num_tokens": 89922.0,
+      "step": 43
+    },
+    {
+      "entropy": 2.2295450568199158,
+      "epoch": 0.6197183098591549,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.4000232219696045,
+      "mean_token_accuracy": 0.580116793513298,
+      "num_tokens": 92171.0,
+      "step": 44
+    },
+    {
+      "entropy": 2.2142042219638824,
+      "epoch": 0.6338028169014085,
+      "grad_norm": 8.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2619423866271973,
+      "mean_token_accuracy": 0.5818447023630142,
+      "num_tokens": 94362.0,
+      "step": 45
+    },
+    {
+      "entropy": 2.3695861101150513,
+      "epoch": 0.647887323943662,
+      "grad_norm": 7.21875,
+      "learning_rate": 5e-05,
+      "loss": 2.4540324211120605,
+      "mean_token_accuracy": 0.5298221409320831,
+      "num_tokens": 96559.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.9680090248584747,
+      "epoch": 0.6619718309859155,
+      "grad_norm": 8.9375,
+      "learning_rate": 5e-05,
+      "loss": 1.9608612060546875,
+      "mean_token_accuracy": 0.6151116490364075,
+      "num_tokens": 99018.0,
+      "step": 47
+    },
+    {
+      "entropy": 2.449136793613434,
+      "epoch": 0.676056338028169,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.364119529724121,
+      "mean_token_accuracy": 0.5657317340373993,
+      "num_tokens": 100885.0,
+      "step": 48
+    },
+    {
+      "entropy": 2.109535813331604,
+      "epoch": 0.6901408450704225,
+      "grad_norm": 6.875,
+      "learning_rate": 5e-05,
+      "loss": 2.1192779541015625,
+      "mean_token_accuracy": 0.5970411896705627,
+      "num_tokens": 103357.0,
+      "step": 49
+    },
+    {
+      "entropy": 2.3080650568008423,
+      "epoch": 0.704225352112676,
+      "grad_norm": 8.0625,
+      "learning_rate": 5e-05,
+      "loss": 2.351897716522217,
+      "mean_token_accuracy": 0.5307685062289238,
+      "num_tokens": 105870.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.1463415920734406,
+      "epoch": 0.7183098591549296,
+      "grad_norm": 7.5,
+      "learning_rate": 5e-05,
+      "loss": 2.2693376541137695,
+      "mean_token_accuracy": 0.5934874191880226,
+      "num_tokens": 107995.0,
+      "step": 51
+    },
+    {
+      "entropy": 2.1181687712669373,
+      "epoch": 0.7323943661971831,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 2.1629841327667236,
+      "mean_token_accuracy": 0.6063478291034698,
+      "num_tokens": 110358.0,
+      "step": 52
+    },
+    {
+      "entropy": 2.0169546008110046,
+      "epoch": 0.7464788732394366,
+      "grad_norm": 7.78125,
+      "learning_rate": 5e-05,
+      "loss": 1.817868709564209,
+      "mean_token_accuracy": 0.6273332387208939,
+      "num_tokens": 112267.0,
+      "step": 53
+    },
+    {
+      "entropy": 2.189443975687027,
+      "epoch": 0.7605633802816901,
+      "grad_norm": 9.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.3292086124420166,
+      "mean_token_accuracy": 0.6041209101676941,
+      "num_tokens": 113899.0,
+      "step": 54
+    },
+    {
+      "entropy": 2.1946000456809998,
+      "epoch": 0.7746478873239436,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.2774949073791504,
+      "mean_token_accuracy": 0.5620942413806915,
+      "num_tokens": 116015.0,
+      "step": 55
+    },
+    {
+      "entropy": 2.1172711849212646,
+      "epoch": 0.7887323943661971,
+      "grad_norm": 7.90625,
+      "learning_rate": 5e-05,
+      "loss": 2.1236391067504883,
+      "mean_token_accuracy": 0.5911316424608231,
+      "num_tokens": 118044.0,
+      "step": 56
+    },
+    {
+      "entropy": 2.144031971693039,
+      "epoch": 0.8028169014084507,
+      "grad_norm": 8.8125,
+      "learning_rate": 5e-05,
+      "loss": 2.187140703201294,
+      "mean_token_accuracy": 0.5624879449605942,
+      "num_tokens": 120180.0,
+      "step": 57
+    },
+    {
+      "entropy": 2.181178390979767,
+      "epoch": 0.8169014084507042,
+      "grad_norm": 10.3125,
+      "learning_rate": 5e-05,
+      "loss": 2.113279104232788,
+      "mean_token_accuracy": 0.6004882454872131,
+      "num_tokens": 121676.0,
+      "step": 58
+    },
+    {
+      "entropy": 2.314671218395233,
+      "epoch": 0.8309859154929577,
+      "grad_norm": 9.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.331024169921875,
+      "mean_token_accuracy": 0.5824068263173103,
+      "num_tokens": 123161.0,
+      "step": 59
+    },
+    {
+      "entropy": 2.0048855245113373,
+      "epoch": 0.8450704225352113,
+      "grad_norm": 9.375,
+      "learning_rate": 5e-05,
+      "loss": 2.070265054702759,
+      "mean_token_accuracy": 0.6141902804374695,
+      "num_tokens": 125016.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.966452568769455,
+      "epoch": 0.8591549295774648,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 1.9488521814346313,
+      "mean_token_accuracy": 0.5854089111089706,
+      "num_tokens": 127349.0,
+      "step": 61
+    },
+    {
+      "entropy": 2.1519170999526978,
+      "epoch": 0.8732394366197183,
+      "grad_norm": 7.5625,
+      "learning_rate": 5e-05,
+      "loss": 2.1743807792663574,
+      "mean_token_accuracy": 0.5636171847581863,
+      "num_tokens": 129742.0,
+      "step": 62
+    },
+    {
+      "entropy": 2.7546687722206116,
+      "epoch": 0.8873239436619719,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.7343056201934814,
+      "mean_token_accuracy": 0.496803916990757,
+      "num_tokens": 131743.0,
+      "step": 63
+    },
+    {
+      "entropy": 2.421773910522461,
+      "epoch": 0.9014084507042254,
+      "grad_norm": 7.96875,
+      "learning_rate": 5e-05,
+      "loss": 2.6149866580963135,
+      "mean_token_accuracy": 0.5249136164784431,
+      "num_tokens": 134002.0,
+      "step": 64
+    },
+    {
+      "entropy": 2.0106727480888367,
+      "epoch": 0.9154929577464789,
+      "grad_norm": 8.4375,
+      "learning_rate": 5e-05,
+      "loss": 1.940914273262024,
+      "mean_token_accuracy": 0.5855439230799675,
+      "num_tokens": 136120.0,
+      "step": 65
+    },
+    {
+      "entropy": 2.1143648624420166,
+      "epoch": 0.9295774647887324,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 2.2469482421875,
+      "mean_token_accuracy": 0.6145937740802765,
+      "num_tokens": 137997.0,
+      "step": 66
+    },
+    {
+      "entropy": 2.1894670128822327,
+      "epoch": 0.9436619718309859,
+      "grad_norm": 8.25,
+      "learning_rate": 5e-05,
+      "loss": 2.20756459236145,
+      "mean_token_accuracy": 0.5872255340218544,
+      "num_tokens": 139904.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.75501748919487,
+      "epoch": 0.9577464788732394,
+      "grad_norm": 9.3125,
+      "learning_rate": 5e-05,
+      "loss": 1.9068934917449951,
+      "mean_token_accuracy": 0.6642919778823853,
+      "num_tokens": 141166.0,
+      "step": 68
+    },
+    {
+      "entropy": 2.204477220773697,
+      "epoch": 0.971830985915493,
+      "grad_norm": 7.875,
+      "learning_rate": 5e-05,
+      "loss": 2.286216974258423,
+      "mean_token_accuracy": 0.5354761853814125,
+      "num_tokens": 143738.0,
+      "step": 69
+    },
+    {
+      "entropy": 2.1219914853572845,
+      "epoch": 0.9859154929577465,
+      "grad_norm": 8.625,
+      "learning_rate": 5e-05,
+      "loss": 2.274585723876953,
+      "mean_token_accuracy": 0.609759122133255,
+      "num_tokens": 145439.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.9260511100292206,
+      "epoch": 1.0,
+      "grad_norm": 8.375,
+      "learning_rate": 5e-05,
+      "loss": 1.8763841390609741,
+      "mean_token_accuracy": 0.5979989916086197,
+      "num_tokens": 147650.0,
+      "step": 71
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 2.1684498711371085,
+      "eval_loss": 2.284580707550049,
+      "eval_mean_token_accuracy": 0.5757196218195096,
+      "eval_num_tokens": 147650.0,
+      "eval_runtime": 18.8201,
+      "eval_samples_per_second": 15.09,
+      "eval_steps_per_second": 3.773,
+      "step": 71
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 284,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 153041317847040.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-71/training_args.bin b/checkpoint-71/training_args.bin
new file mode 100644
index 0000000..0b099d6
--- /dev/null
+++ b/checkpoint-71/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
+size 5585
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..436e93d
--- /dev/null
+++ b/config.json
@@ -0,0 +1,62 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 512,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_bidirectional_attention": false,
+  "use_cache": false,
+  "vocab_size": 262144
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000..c652aad
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.0.0"
+}
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000..0aae080
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d96c91306eb76ff6917e36d262afbcc696836520af2951f30d4fe02f9f50ad8
+size 536223056
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000..899af07
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
+size 33384443
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..7202584
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,24 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000..0b099d6
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
+size 5585