From eb1a43fb862266d495a526de0b017d95127e1b05 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 13 Jun 2026 11:02:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: reaperdoesntknow/DiStil-Qwen3-1.7B-uncensored Source: Original Platform --- .gitattributes | 36 + README.md | 126 + chat_template.jinja | 89 + config.json | 63 + ...t.tfevents.1774711157.c23bde7bdfe8.10063.0 | 3 + generation_config.json | 12 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 29 + trainer_state (2).json | 5176 +++++++++++++++++ 10 files changed, 5540 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 events.out.tfevents.1774711157.c23bde7bdfe8.10063.0 create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 trainer_state (2).json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..9149be2 --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +--- +base_model: +- reaperdoesntknow/TopologicalQwen +tags: +- convergentintel +- edge +- distillation +- knowledge-distillation + +--- + +# DiStil-Qwen3-1.7B-uncensored + +**Uncensored Distillation of Qwen3-1.7B — Alignment-Free Capability Transfer** + +*Convergent Intelligence LLC: Research Division* + +--- + +## What This Is + +DiStil-Qwen3-1.7B-uncensored is a 1.7B parameter model produced by distilling Qwen3 with uncensored SFT data, removing alignment-imposed refusal behaviors while preserving the base model's reasoning and generation capabilities. The goal is a model that responds to the prompt as given rather than filtering through safety heuristics that often misfire on legitimate technical, analytical, and research queries. + +This is the base model in a distillation chain: +- **DiStil-Qwen3-1.7B-uncensored** ← you are here +- → [Disctil-Qwen3-1.7B](https://huggingface.co/reaperdoesntknow/Disctil-Qwen3-1.7B) (DISC-informed refinement) + +## Architecture + +| Parameter | Value | +|-----------|-------| +| Architecture | Qwen3ForCausalLM | +| Parameters | ~2.03B (1.7B effective) | +| Hidden Size | 2048 | +| Layers | 28 | +| Attention Heads | 16 (Q) / 8 (KV) — GQA | +| Intermediate | 6144 | +| Context Length | 40,960 tokens | +| Vocabulary | 151,936 | + +## Training + +Supervised fine-tuning using TRL on uncensored instruction data. The training preserves the base Qwen3 architecture and tokenizer while shifting the model's response distribution away from refusal patterns. No architectural modifications — this is a pure SFT intervention on the response surface. + +## Usage + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model = AutoModelForCausalLM.from_pretrained( + "reaperdoesntknow/DiStil-Qwen3-1.7B-uncensored", + torch_dtype="auto", + device_map="auto" +) +tokenizer = AutoTokenizer.from_pretrained("reaperdoesntknow/DiStil-Qwen3-1.7B-uncensored") + +messages = [{"role": "user", "content": "Explain the tradeoffs between alignment training and capability preservation in small language models."}] +text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +inputs = tokenizer(text, return_tensors="pt").to(model.device) +output = model.generate(**inputs, max_new_tokens=512, temperature=0.7, top_p=0.9) +print(tokenizer.decode(output[0], skip_special_tokens=True)) +``` + + +## Mathematical Foundations: Discrepancy Calculus (DISC) + +This model is part of a distillation chain built on Discrepancy Calculus — a measure-theoretic framework where the teacher's output distribution is decomposed via the Mesh Fundamental Identity into smooth (AC), jump, and Cantor components. The discrepancy operator $Df(x) = \lim_{\varepsilon \downarrow 0} \frac{1}{\varepsilon} \int_x^{x+\varepsilon} \frac{|f(t) - f(x)|}{|t - x|} dt$ quantifies local structural mismatch that standard KL divergence averages away. + +Full theory: *"On the Formal Analysis of Discrepancy Calculus"* (Colca, 2026; Convergent Intelligence LLC: Research Division). Full methodology: [Structure Over Scale (DOI: 10.57967/hf/8165)](https://doi.org/10.57967/hf/8165). + +## Related Models + +| Model | Description | Downloads | +|-------|-------------|-----------| +| [Disctil-Qwen3-1.7B](https://huggingface.co/reaperdoesntknow/Disctil-Qwen3-1.7B) | DISC-informed refinement of this model | 286 | +| [DistilQwen3-1.7B-uncensored](https://huggingface.co/reaperdoesntknow/DistilQwen3-1.7B-uncensored) | Parallel distillation variant | 351 | +| [DistilQwen3-1.7B-uncensored-GGUF](https://huggingface.co/reaperdoesntknow/DistilQwen3-1.7B-uncensored-GGUF) | Quantized for edge deployment | 239 | +| [TopologicalQwen](https://huggingface.co/reaperdoesntknow/TopologicalQwen) | Topology-aware distillation (TKD) | 622 | + +**[DistilQwen Collection](https://huggingface.co/collections/reaperdoesntknow/distilqwen-69bf40ec669117e3f069ef1c)** — Full proof-weighted distillation series + +Full methodology: [Structure Over Scale (DOI: 10.57967/hf/8165)](https://doi.org/10.57967/hf/8165) + +## Citation + +```bibtex +@misc{colca2026distiluncensored, + title={DiStil-Qwen3-1.7B-uncensored: Alignment-Free Capability Transfer}, + author={Colca, Roy S.}, + year={2026}, + publisher={HuggingFace}, + url={https://huggingface.co/reaperdoesntknow/DiStil-Qwen3-1.7B-uncensored}, + note={Convergent Intelligence LLC: Research Division} +} +``` + +--- + + + + +--- + +## From the Convergent Intelligence Portfolio + +**[DistilQwen Collection](https://huggingface.co/collections/reaperdoesntknow/distilqwen-69bf40ec669117e3f069ef1c)** — Our only BF16 series. Proof-weighted distillation from Qwen3-30B-A3B → 1.7B and 0.6B on H100. Three teacher variants (Instruct, Thinking, Coder), nine models, 2,788 combined downloads. The rest of the portfolio proves structure beats scale on CPU. This collection shows what happens when you give the methodology real hardware. + +Top model: [Qwen3-1.7B-Coder-Distilled-SFT](https://huggingface.co/reaperdoesntknow/Qwen3-1.7B-Coder-Distilled-SFT) — 508 downloads + +Full methodology: [Structure Over Scale (DOI: 10.57967/hf/8165)](https://doi.org/10.57967/hf/8165) + +*Convergent Intelligence LLC: Research Division* + + + +*Convergent Intelligence LLC: Research Division* +*"Where classical analysis fails to see, we begin."* + +--- +Part of the [reaperdoesntknow research portfolio](https://huggingface.co/reaperdoesntknow) — 49 models, 22,598 total downloads | Last refreshed: 2026-03-30 12:05 UTC + + + +--- + +*Last updated: 2026-03-31 by Convergent Intelligence LLC: Research Division* diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..1782cca --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.0.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/events.out.tfevents.1774711157.c23bde7bdfe8.10063.0 b/events.out.tfevents.1774711157.c23bde7bdfe8.10063.0 new file mode 100644 index 0000000..7e28187 --- /dev/null +++ b/events.out.tfevents.1774711157.c23bde7bdfe8.10063.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab24fbf638a33240d9c5e8f5608c78d6f02ac58fac33035ff17642a9127a5969 +size 201489 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c33fb76 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.0.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..3a3b243 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:120d82c4313a59c34bdc26106821e29f5df3f96fe5c3e1234e05c92454e5bb70 +size 4063515640 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..7d75d3b --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/trainer_state (2).json b/trainer_state (2).json new file mode 100644 index 0000000..692824a --- /dev/null +++ b/trainer_state (2).json @@ -0,0 +1,5176 @@ +{ + "best_global_step": 512, + "best_metric": 0.5158528089523315, + "best_model_checkpoint": "Disctil-Qwen3-1.7B/checkpoint-512", + "epoch": 0.22394751230180426, + "eval_steps": 256, + "global_step": 512, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.27786555141210556, + "epoch": 0.00043739748496446145, + "grad_norm": 21.25, + "learning_rate": 0.0, + "loss": 0.901656985282898, + "mean_token_accuracy": 0.8019306063652039, + "num_tokens": 8192.0, + "step": 1 + }, + { + "entropy": 0.32270805910229683, + "epoch": 0.0008747949699289229, + "grad_norm": 20.75, + "learning_rate": 7.8125e-08, + "loss": 0.9285624027252197, + "mean_token_accuracy": 0.7913001030683517, + "num_tokens": 16384.0, + "step": 2 + }, + { + "entropy": 0.3030129224061966, + "epoch": 0.0013121924548933845, + "grad_norm": 23.375, + "learning_rate": 1.5625e-07, + "loss": 0.9610694646835327, + "mean_token_accuracy": 0.7899560034275055, + "num_tokens": 24576.0, + "step": 3 + }, + { + "entropy": 0.3043339401483536, + "epoch": 0.0017495899398578458, + "grad_norm": 22.0, + "learning_rate": 2.3437500000000003e-07, + "loss": 0.9333600997924805, + "mean_token_accuracy": 0.7908113300800323, + "num_tokens": 32768.0, + "step": 4 + }, + { + "entropy": 0.30834493041038513, + "epoch": 0.002186987424822307, + "grad_norm": 21.75, + "learning_rate": 3.125e-07, + "loss": 0.9301238059997559, + "mean_token_accuracy": 0.7915444821119308, + "num_tokens": 40960.0, + "step": 5 + }, + { + "entropy": 0.3122338131070137, + "epoch": 0.002624384909786769, + "grad_norm": 22.5, + "learning_rate": 3.90625e-07, + "loss": 0.9689892530441284, + "mean_token_accuracy": 0.7847018390893936, + "num_tokens": 49152.0, + "step": 6 + }, + { + "entropy": 0.2893872708082199, + "epoch": 0.0030617823947512303, + "grad_norm": 21.0, + "learning_rate": 4.6875000000000006e-07, + "loss": 0.9168861508369446, + "mean_token_accuracy": 0.8013196438550949, + "num_tokens": 57344.0, + "step": 7 + }, + { + "entropy": 0.3050616979598999, + "epoch": 0.0034991798797156916, + "grad_norm": 21.875, + "learning_rate": 5.468750000000001e-07, + "loss": 0.9516887664794922, + "mean_token_accuracy": 0.7850684374570847, + "num_tokens": 65536.0, + "step": 8 + }, + { + "entropy": 0.31983715295791626, + "epoch": 0.003936577364680153, + "grad_norm": 23.375, + "learning_rate": 6.25e-07, + "loss": 0.9918694496154785, + "mean_token_accuracy": 0.7829912006855011, + "num_tokens": 73728.0, + "step": 9 + }, + { + "entropy": 0.32186224311590195, + "epoch": 0.004373974849644614, + "grad_norm": 21.0, + "learning_rate": 7.03125e-07, + "loss": 0.9560595750808716, + "mean_token_accuracy": 0.7919110506772995, + "num_tokens": 81920.0, + "step": 10 + }, + { + "entropy": 0.32521846890449524, + "epoch": 0.004811372334609076, + "grad_norm": 22.875, + "learning_rate": 7.8125e-07, + "loss": 0.9732818007469177, + "mean_token_accuracy": 0.7860459387302399, + "num_tokens": 90112.0, + "step": 11 + }, + { + "entropy": 0.31955306977033615, + "epoch": 0.005248769819573538, + "grad_norm": 21.75, + "learning_rate": 8.59375e-07, + "loss": 0.9713459014892578, + "mean_token_accuracy": 0.7805474102497101, + "num_tokens": 98304.0, + "step": 12 + }, + { + "entropy": 0.30780235305428505, + "epoch": 0.005686167304537999, + "grad_norm": 20.625, + "learning_rate": 9.375000000000001e-07, + "loss": 0.9195499420166016, + "mean_token_accuracy": 0.7969208359718323, + "num_tokens": 106496.0, + "step": 13 + }, + { + "entropy": 0.33620841801166534, + "epoch": 0.0061235647895024605, + "grad_norm": 21.25, + "learning_rate": 1.0156250000000001e-06, + "loss": 0.9759924411773682, + "mean_token_accuracy": 0.7779259830713272, + "num_tokens": 114652.0, + "step": 14 + }, + { + "entropy": 0.30951154232025146, + "epoch": 0.0065609622744669215, + "grad_norm": 20.0, + "learning_rate": 1.0937500000000001e-06, + "loss": 0.8884122967720032, + "mean_token_accuracy": 0.7992424219846725, + "num_tokens": 122844.0, + "step": 15 + }, + { + "entropy": 0.3126643896102905, + "epoch": 0.006998359759431383, + "grad_norm": 20.5, + "learning_rate": 1.1718750000000001e-06, + "loss": 0.9235544204711914, + "mean_token_accuracy": 0.7932551354169846, + "num_tokens": 131036.0, + "step": 16 + }, + { + "entropy": 0.32217270135879517, + "epoch": 0.007435757244395845, + "grad_norm": 22.25, + "learning_rate": 1.25e-06, + "loss": 0.9568185210227966, + "mean_token_accuracy": 0.7825024574995041, + "num_tokens": 139228.0, + "step": 17 + }, + { + "entropy": 0.28684747219085693, + "epoch": 0.007873154729360307, + "grad_norm": 18.25, + "learning_rate": 1.328125e-06, + "loss": 0.8255479335784912, + "mean_token_accuracy": 0.8137829899787903, + "num_tokens": 147420.0, + "step": 18 + }, + { + "entropy": 0.3058253973722458, + "epoch": 0.008310552214324767, + "grad_norm": 20.0, + "learning_rate": 1.40625e-06, + "loss": 0.8979604244232178, + "mean_token_accuracy": 0.7972874045372009, + "num_tokens": 155612.0, + "step": 19 + }, + { + "entropy": 0.38854775577783585, + "epoch": 0.008747949699289229, + "grad_norm": 22.625, + "learning_rate": 1.484375e-06, + "loss": 1.0705394744873047, + "mean_token_accuracy": 0.7572091966867447, + "num_tokens": 163804.0, + "step": 20 + }, + { + "entropy": 0.30834781378507614, + "epoch": 0.00918534718425369, + "grad_norm": 20.375, + "learning_rate": 1.5625e-06, + "loss": 0.9028609395027161, + "mean_token_accuracy": 0.7923997938632965, + "num_tokens": 171996.0, + "step": 21 + }, + { + "entropy": 0.32601282000541687, + "epoch": 0.009622744669218152, + "grad_norm": 20.625, + "learning_rate": 1.640625e-06, + "loss": 0.9245335459709167, + "mean_token_accuracy": 0.7817693054676056, + "num_tokens": 180188.0, + "step": 22 + }, + { + "entropy": 0.30635514110326767, + "epoch": 0.010060142154182614, + "grad_norm": 19.875, + "learning_rate": 1.71875e-06, + "loss": 0.9333948493003845, + "mean_token_accuracy": 0.7876344174146652, + "num_tokens": 188380.0, + "step": 23 + }, + { + "entropy": 0.3130311816930771, + "epoch": 0.010497539639147076, + "grad_norm": 19.625, + "learning_rate": 1.796875e-06, + "loss": 0.8786435723304749, + "mean_token_accuracy": 0.801075279712677, + "num_tokens": 196572.0, + "step": 24 + }, + { + "entropy": 0.3683779239654541, + "epoch": 0.010934937124111536, + "grad_norm": 21.0, + "learning_rate": 1.8750000000000003e-06, + "loss": 0.9972445964813232, + "mean_token_accuracy": 0.7724829018115997, + "num_tokens": 204764.0, + "step": 25 + }, + { + "entropy": 0.34924668073654175, + "epoch": 0.011372334609075998, + "grad_norm": 22.125, + "learning_rate": 1.953125e-06, + "loss": 1.0172064304351807, + "mean_token_accuracy": 0.7709844559431076, + "num_tokens": 212677.0, + "step": 26 + }, + { + "entropy": 0.3614501357078552, + "epoch": 0.01180973209404046, + "grad_norm": 19.625, + "learning_rate": 2.0312500000000002e-06, + "loss": 0.9596425294876099, + "mean_token_accuracy": 0.7776148617267609, + "num_tokens": 220869.0, + "step": 27 + }, + { + "entropy": 0.31008924543857574, + "epoch": 0.012247129579004921, + "grad_norm": 18.25, + "learning_rate": 2.109375e-06, + "loss": 0.8647151589393616, + "mean_token_accuracy": 0.7933773398399353, + "num_tokens": 229061.0, + "step": 28 + }, + { + "entropy": 0.29904749244451523, + "epoch": 0.012684527063969383, + "grad_norm": 18.875, + "learning_rate": 2.1875000000000002e-06, + "loss": 0.8574439287185669, + "mean_token_accuracy": 0.804007813334465, + "num_tokens": 237253.0, + "step": 29 + }, + { + "entropy": 0.378295473754406, + "epoch": 0.013121924548933843, + "grad_norm": 19.75, + "learning_rate": 2.265625e-06, + "loss": 0.9890425205230713, + "mean_token_accuracy": 0.7633186727762222, + "num_tokens": 245445.0, + "step": 30 + }, + { + "entropy": 0.360466867685318, + "epoch": 0.013559322033898305, + "grad_norm": 18.0, + "learning_rate": 2.3437500000000002e-06, + "loss": 0.9291175603866577, + "mean_token_accuracy": 0.7820213884115219, + "num_tokens": 253493.0, + "step": 31 + }, + { + "entropy": 0.30008650943636894, + "epoch": 0.013996719518862766, + "grad_norm": 16.875, + "learning_rate": 2.421875e-06, + "loss": 0.7958075404167175, + "mean_token_accuracy": 0.8091397881507874, + "num_tokens": 261685.0, + "step": 32 + }, + { + "entropy": 0.3233437053859234, + "epoch": 0.014434117003827228, + "grad_norm": 16.75, + "learning_rate": 2.5e-06, + "loss": 0.8411202430725098, + "mean_token_accuracy": 0.793621689081192, + "num_tokens": 269877.0, + "step": 33 + }, + { + "entropy": 0.30822817981243134, + "epoch": 0.01487151448879169, + "grad_norm": 16.75, + "learning_rate": 2.5781250000000004e-06, + "loss": 0.8265758752822876, + "mean_token_accuracy": 0.8031525015830994, + "num_tokens": 278069.0, + "step": 34 + }, + { + "entropy": 0.3806365579366684, + "epoch": 0.01530891197375615, + "grad_norm": 15.4375, + "learning_rate": 2.65625e-06, + "loss": 0.8731443881988525, + "mean_token_accuracy": 0.7812805324792862, + "num_tokens": 286261.0, + "step": 35 + }, + { + "entropy": 0.28861086070537567, + "epoch": 0.015746309458720614, + "grad_norm": 14.0625, + "learning_rate": 2.7343750000000004e-06, + "loss": 0.7268953323364258, + "mean_token_accuracy": 0.822182759642601, + "num_tokens": 294432.0, + "step": 36 + }, + { + "entropy": 0.333795465528965, + "epoch": 0.016183706943685074, + "grad_norm": 15.875, + "learning_rate": 2.8125e-06, + "loss": 0.8444324731826782, + "mean_token_accuracy": 0.7978983372449875, + "num_tokens": 302624.0, + "step": 37 + }, + { + "entropy": 0.33112896978855133, + "epoch": 0.016621104428649534, + "grad_norm": 15.75, + "learning_rate": 2.8906250000000004e-06, + "loss": 0.8359007835388184, + "mean_token_accuracy": 0.792033240199089, + "num_tokens": 310816.0, + "step": 38 + }, + { + "entropy": 0.34320055693387985, + "epoch": 0.017058501913613997, + "grad_norm": 13.0, + "learning_rate": 2.96875e-06, + "loss": 0.7701590061187744, + "mean_token_accuracy": 0.8037634491920471, + "num_tokens": 319008.0, + "step": 39 + }, + { + "entropy": 0.42401332408189774, + "epoch": 0.017495899398578457, + "grad_norm": 14.3125, + "learning_rate": 3.0468750000000004e-06, + "loss": 0.8930312395095825, + "mean_token_accuracy": 0.7701613008975983, + "num_tokens": 327200.0, + "step": 40 + }, + { + "entropy": 0.3282608240842819, + "epoch": 0.01793329688354292, + "grad_norm": 12.875, + "learning_rate": 3.125e-06, + "loss": 0.7595839500427246, + "mean_token_accuracy": 0.8081848919391632, + "num_tokens": 335128.0, + "step": 41 + }, + { + "entropy": 0.3341097980737686, + "epoch": 0.01837069436850738, + "grad_norm": 12.6875, + "learning_rate": 3.2031250000000004e-06, + "loss": 0.7940071821212769, + "mean_token_accuracy": 0.8011974543333054, + "num_tokens": 343320.0, + "step": 42 + }, + { + "entropy": 0.35623009502887726, + "epoch": 0.01880809185347184, + "grad_norm": 13.5625, + "learning_rate": 3.28125e-06, + "loss": 0.7961429357528687, + "mean_token_accuracy": 0.7959433048963547, + "num_tokens": 351512.0, + "step": 43 + }, + { + "entropy": 0.33236806094646454, + "epoch": 0.019245489338436304, + "grad_norm": 12.8125, + "learning_rate": 3.3593750000000003e-06, + "loss": 0.7723078727722168, + "mean_token_accuracy": 0.8073069304227829, + "num_tokens": 359704.0, + "step": 44 + }, + { + "entropy": 0.45774705708026886, + "epoch": 0.019682886823400764, + "grad_norm": 13.9375, + "learning_rate": 3.4375e-06, + "loss": 0.9575591087341309, + "mean_token_accuracy": 0.757942333817482, + "num_tokens": 367896.0, + "step": 45 + }, + { + "entropy": 0.35400258004665375, + "epoch": 0.020120284308365228, + "grad_norm": 10.375, + "learning_rate": 3.5156250000000003e-06, + "loss": 0.719309389591217, + "mean_token_accuracy": 0.814393937587738, + "num_tokens": 376088.0, + "step": 46 + }, + { + "entropy": 0.3176649585366249, + "epoch": 0.020557681793329688, + "grad_norm": 9.4375, + "learning_rate": 3.59375e-06, + "loss": 0.6449992656707764, + "mean_token_accuracy": 0.8264907151460648, + "num_tokens": 384280.0, + "step": 47 + }, + { + "entropy": 0.3971637114882469, + "epoch": 0.02099507927829415, + "grad_norm": 11.4375, + "learning_rate": 3.6718750000000003e-06, + "loss": 0.8187000155448914, + "mean_token_accuracy": 0.7884897291660309, + "num_tokens": 392472.0, + "step": 48 + }, + { + "entropy": 0.42229485511779785, + "epoch": 0.02143247676325861, + "grad_norm": 11.6875, + "learning_rate": 3.7500000000000005e-06, + "loss": 0.846123456954956, + "mean_token_accuracy": 0.7745601385831833, + "num_tokens": 400664.0, + "step": 49 + }, + { + "entropy": 0.3986642137169838, + "epoch": 0.02186987424822307, + "grad_norm": 10.625, + "learning_rate": 3.828125000000001e-06, + "loss": 0.7735276222229004, + "mean_token_accuracy": 0.7872678488492966, + "num_tokens": 408856.0, + "step": 50 + }, + { + "entropy": 0.43453069031238556, + "epoch": 0.022307271733187535, + "grad_norm": 11.5625, + "learning_rate": 3.90625e-06, + "loss": 0.8254950046539307, + "mean_token_accuracy": 0.7809139937162399, + "num_tokens": 417048.0, + "step": 51 + }, + { + "entropy": 0.37958449125289917, + "epoch": 0.022744669218151995, + "grad_norm": 8.75, + "learning_rate": 3.984375e-06, + "loss": 0.6892186403274536, + "mean_token_accuracy": 0.813049852848053, + "num_tokens": 425240.0, + "step": 52 + }, + { + "entropy": 0.4048294723033905, + "epoch": 0.02318206670311646, + "grad_norm": 8.8125, + "learning_rate": 4.0625000000000005e-06, + "loss": 0.7329984307289124, + "mean_token_accuracy": 0.798020526766777, + "num_tokens": 433432.0, + "step": 53 + }, + { + "entropy": 0.42267417162656784, + "epoch": 0.02361946418808092, + "grad_norm": 9.1875, + "learning_rate": 4.140625000000001e-06, + "loss": 0.7617641687393188, + "mean_token_accuracy": 0.7957284301519394, + "num_tokens": 441608.0, + "step": 54 + }, + { + "entropy": 0.44952575117349625, + "epoch": 0.02405686167304538, + "grad_norm": 8.125, + "learning_rate": 4.21875e-06, + "loss": 0.7307311296463013, + "mean_token_accuracy": 0.787634402513504, + "num_tokens": 449800.0, + "step": 55 + }, + { + "entropy": 0.45996539294719696, + "epoch": 0.024494259158009842, + "grad_norm": 7.4375, + "learning_rate": 4.296875e-06, + "loss": 0.7325446605682373, + "mean_token_accuracy": 0.7894672453403473, + "num_tokens": 457992.0, + "step": 56 + }, + { + "entropy": 0.46281183511018753, + "epoch": 0.024931656642974302, + "grad_norm": 7.78125, + "learning_rate": 4.3750000000000005e-06, + "loss": 0.7531175017356873, + "mean_token_accuracy": 0.7916666567325592, + "num_tokens": 466184.0, + "step": 57 + }, + { + "entropy": 0.40494948625564575, + "epoch": 0.025369054127938766, + "grad_norm": 6.46875, + "learning_rate": 4.453125000000001e-06, + "loss": 0.6567135453224182, + "mean_token_accuracy": 0.8132942169904709, + "num_tokens": 474376.0, + "step": 58 + }, + { + "entropy": 0.44023270159959793, + "epoch": 0.025806451612903226, + "grad_norm": 7.25, + "learning_rate": 4.53125e-06, + "loss": 0.703899621963501, + "mean_token_accuracy": 0.8019305914640427, + "num_tokens": 482568.0, + "step": 59 + }, + { + "entropy": 0.3879058510065079, + "epoch": 0.026243849097867686, + "grad_norm": 6.15625, + "learning_rate": 4.609375e-06, + "loss": 0.6011914014816284, + "mean_token_accuracy": 0.8233137875795364, + "num_tokens": 490760.0, + "step": 60 + }, + { + "entropy": 0.41236213594675064, + "epoch": 0.02668124658283215, + "grad_norm": 5.46875, + "learning_rate": 4.6875000000000004e-06, + "loss": 0.5771687626838684, + "mean_token_accuracy": 0.8264907002449036, + "num_tokens": 498952.0, + "step": 61 + }, + { + "entropy": 0.4559241756796837, + "epoch": 0.02711864406779661, + "grad_norm": 5.59375, + "learning_rate": 4.765625000000001e-06, + "loss": 0.6721391081809998, + "mean_token_accuracy": 0.8046199232339859, + "num_tokens": 507024.0, + "step": 62 + }, + { + "entropy": 0.40718207508325577, + "epoch": 0.027556041552761073, + "grad_norm": 5.15625, + "learning_rate": 4.84375e-06, + "loss": 0.5817515850067139, + "mean_token_accuracy": 0.8244134932756424, + "num_tokens": 515216.0, + "step": 63 + }, + { + "entropy": 0.4880864918231964, + "epoch": 0.027993439037725533, + "grad_norm": 5.1875, + "learning_rate": 4.921875e-06, + "loss": 0.6458011865615845, + "mean_token_accuracy": 0.8033017814159393, + "num_tokens": 523359.0, + "step": 64 + }, + { + "entropy": 0.4802205115556717, + "epoch": 0.028430836522689993, + "grad_norm": 4.8125, + "learning_rate": 5e-06, + "loss": 0.6212886571884155, + "mean_token_accuracy": 0.8052297234535217, + "num_tokens": 531551.0, + "step": 65 + }, + { + "entropy": 0.5558610707521439, + "epoch": 0.028868234007654456, + "grad_norm": 5.6875, + "learning_rate": 4.999938531594062e-06, + "loss": 0.7324199676513672, + "mean_token_accuracy": 0.7729716449975967, + "num_tokens": 539743.0, + "step": 66 + }, + { + "entropy": 0.4111747443675995, + "epoch": 0.029305631492618917, + "grad_norm": 4.0, + "learning_rate": 4.999754129398938e-06, + "loss": 0.5279664993286133, + "mean_token_accuracy": 0.8387096673250198, + "num_tokens": 547935.0, + "step": 67 + }, + { + "entropy": 0.49969246983528137, + "epoch": 0.02974302897758338, + "grad_norm": 4.21875, + "learning_rate": 4.999446802482556e-06, + "loss": 0.6269458532333374, + "mean_token_accuracy": 0.807062566280365, + "num_tokens": 556127.0, + "step": 68 + }, + { + "entropy": 0.533087246119976, + "epoch": 0.03018042646254784, + "grad_norm": 4.46875, + "learning_rate": 4.999016565957633e-06, + "loss": 0.6664482355117798, + "mean_token_accuracy": 0.7987536638975143, + "num_tokens": 564319.0, + "step": 69 + }, + { + "entropy": 0.4802093654870987, + "epoch": 0.0306178239475123, + "grad_norm": 3.984375, + "learning_rate": 4.998463440980931e-06, + "loss": 0.5956555604934692, + "mean_token_accuracy": 0.8218475133180618, + "num_tokens": 572511.0, + "step": 70 + }, + { + "entropy": 0.5158685892820358, + "epoch": 0.031055221432476764, + "grad_norm": 4.3125, + "learning_rate": 4.997787454752217e-06, + "loss": 0.6313604116439819, + "mean_token_accuracy": 0.8016862124204636, + "num_tokens": 580703.0, + "step": 71 + }, + { + "entropy": 0.49100108444690704, + "epoch": 0.03149261891744123, + "grad_norm": 3.8125, + "learning_rate": 4.996988640512931e-06, + "loss": 0.595371663570404, + "mean_token_accuracy": 0.8114613741636276, + "num_tokens": 588895.0, + "step": 72 + }, + { + "entropy": 0.5480513125658035, + "epoch": 0.031930016402405684, + "grad_norm": 3.90625, + "learning_rate": 4.996067037544542e-06, + "loss": 0.643286406993866, + "mean_token_accuracy": 0.7959433048963547, + "num_tokens": 597087.0, + "step": 73 + }, + { + "entropy": 0.5591509789228439, + "epoch": 0.03236741388737015, + "grad_norm": 4.625, + "learning_rate": 4.995022691166621e-06, + "loss": 0.6653096675872803, + "mean_token_accuracy": 0.7919110357761383, + "num_tokens": 605279.0, + "step": 74 + }, + { + "entropy": 0.5032752007246017, + "epoch": 0.03280481137233461, + "grad_norm": 3.640625, + "learning_rate": 4.993855652734616e-06, + "loss": 0.5787771940231323, + "mean_token_accuracy": 0.8167155385017395, + "num_tokens": 613471.0, + "step": 75 + }, + { + "entropy": 0.4510410279035568, + "epoch": 0.03324220885729907, + "grad_norm": 3.453125, + "learning_rate": 4.992565979637318e-06, + "loss": 0.5281908512115479, + "mean_token_accuracy": 0.8304007798433304, + "num_tokens": 621663.0, + "step": 76 + }, + { + "entropy": 0.4650597348809242, + "epoch": 0.03367960634226353, + "grad_norm": 3.640625, + "learning_rate": 4.9911537352940485e-06, + "loss": 0.5683005452156067, + "mean_token_accuracy": 0.8246578723192215, + "num_tokens": 629855.0, + "step": 77 + }, + { + "entropy": 0.5157822743058205, + "epoch": 0.034117003827227994, + "grad_norm": 3.9375, + "learning_rate": 4.989618989151533e-06, + "loss": 0.5880601406097412, + "mean_token_accuracy": 0.8147605061531067, + "num_tokens": 638047.0, + "step": 78 + }, + { + "entropy": 0.58241768181324, + "epoch": 0.03455440131219246, + "grad_norm": 3.859375, + "learning_rate": 4.987961816680493e-06, + "loss": 0.6643019914627075, + "mean_token_accuracy": 0.794232651591301, + "num_tokens": 646239.0, + "step": 79 + }, + { + "entropy": 0.46477261185646057, + "epoch": 0.034991798797156914, + "grad_norm": 3.03125, + "learning_rate": 4.9861822993719255e-06, + "loss": 0.5226079225540161, + "mean_token_accuracy": 0.8370988517999649, + "num_tokens": 654199.0, + "step": 80 + }, + { + "entropy": 0.4996022880077362, + "epoch": 0.03542919628212138, + "grad_norm": 3.4375, + "learning_rate": 4.984280524733107e-06, + "loss": 0.5533886551856995, + "mean_token_accuracy": 0.8238108903169632, + "num_tokens": 662335.0, + "step": 81 + }, + { + "entropy": 0.5934857875108719, + "epoch": 0.03586659376708584, + "grad_norm": 3.6875, + "learning_rate": 4.98225658628328e-06, + "loss": 0.6762368679046631, + "mean_token_accuracy": 0.7827468067407608, + "num_tokens": 670527.0, + "step": 82 + }, + { + "entropy": 0.563569039106369, + "epoch": 0.0363039912520503, + "grad_norm": 3.390625, + "learning_rate": 4.980110583549062e-06, + "loss": 0.6458903551101685, + "mean_token_accuracy": 0.7992424219846725, + "num_tokens": 678719.0, + "step": 83 + }, + { + "entropy": 0.4253915399312973, + "epoch": 0.03674138873701476, + "grad_norm": 2.6875, + "learning_rate": 4.977842622059546e-06, + "loss": 0.47188764810562134, + "mean_token_accuracy": 0.8451857268810272, + "num_tokens": 686911.0, + "step": 84 + }, + { + "entropy": 0.5138464272022247, + "epoch": 0.037178786221979225, + "grad_norm": 3.140625, + "learning_rate": 4.9754528133411144e-06, + "loss": 0.5898129940032959, + "mean_token_accuracy": 0.8066959828138351, + "num_tokens": 695103.0, + "step": 85 + }, + { + "entropy": 0.5432897508144379, + "epoch": 0.03761618370694368, + "grad_norm": 3.140625, + "learning_rate": 4.972941274911953e-06, + "loss": 0.6102941632270813, + "mean_token_accuracy": 0.8013196438550949, + "num_tokens": 703295.0, + "step": 86 + }, + { + "entropy": 0.4696696624159813, + "epoch": 0.038053581191908145, + "grad_norm": 3.109375, + "learning_rate": 4.970308130276273e-06, + "loss": 0.5391623973846436, + "mean_token_accuracy": 0.8216031193733215, + "num_tokens": 711487.0, + "step": 87 + }, + { + "entropy": 0.45806846022605896, + "epoch": 0.03849097867687261, + "grad_norm": 2.90625, + "learning_rate": 4.967553508918236e-06, + "loss": 0.5263965129852295, + "mean_token_accuracy": 0.8305229693651199, + "num_tokens": 719679.0, + "step": 88 + }, + { + "entropy": 0.49067409336566925, + "epoch": 0.03892837616183707, + "grad_norm": 3.0, + "learning_rate": 4.96467754629559e-06, + "loss": 0.5333690047264099, + "mean_token_accuracy": 0.8266129046678543, + "num_tokens": 727871.0, + "step": 89 + }, + { + "entropy": 0.5586400255560875, + "epoch": 0.03936577364680153, + "grad_norm": 3.296875, + "learning_rate": 4.961680383833005e-06, + "loss": 0.6310930848121643, + "mean_token_accuracy": 0.7937438935041428, + "num_tokens": 736063.0, + "step": 90 + }, + { + "entropy": 0.6168518960475922, + "epoch": 0.03980317113176599, + "grad_norm": 3.296875, + "learning_rate": 4.958562168915121e-06, + "loss": 0.7125385999679565, + "mean_token_accuracy": 0.7805473953485489, + "num_tokens": 744255.0, + "step": 91 + }, + { + "entropy": 0.5408408343791962, + "epoch": 0.040240568616730456, + "grad_norm": 3.0, + "learning_rate": 4.955323054879298e-06, + "loss": 0.6216639280319214, + "mean_token_accuracy": 0.8088954240083694, + "num_tokens": 752447.0, + "step": 92 + }, + { + "entropy": 0.445939376950264, + "epoch": 0.04067796610169491, + "grad_norm": 2.640625, + "learning_rate": 4.9519632010080765e-06, + "loss": 0.4946935772895813, + "mean_token_accuracy": 0.8440860360860825, + "num_tokens": 760639.0, + "step": 93 + }, + { + "entropy": 0.346243679523468, + "epoch": 0.041115363586659376, + "grad_norm": 2.578125, + "learning_rate": 4.948482772521346e-06, + "loss": 0.38514432311058044, + "mean_token_accuracy": 0.8690126985311508, + "num_tokens": 768831.0, + "step": 94 + }, + { + "entropy": 0.5287810266017914, + "epoch": 0.04155276107162384, + "grad_norm": 2.90625, + "learning_rate": 4.944881940568219e-06, + "loss": 0.5804294943809509, + "mean_token_accuracy": 0.8090175986289978, + "num_tokens": 777023.0, + "step": 95 + }, + { + "entropy": 0.4812530502676964, + "epoch": 0.0419901585565883, + "grad_norm": 2.75, + "learning_rate": 4.941160882218617e-06, + "loss": 0.5441983938217163, + "mean_token_accuracy": 0.8217253088951111, + "num_tokens": 785215.0, + "step": 96 + }, + { + "entropy": 0.5249937623739243, + "epoch": 0.04242755604155276, + "grad_norm": 3.171875, + "learning_rate": 4.937319780454559e-06, + "loss": 0.6077675819396973, + "mean_token_accuracy": 0.8069403767585754, + "num_tokens": 793407.0, + "step": 97 + }, + { + "entropy": 0.5760547369718552, + "epoch": 0.04286495352651722, + "grad_norm": 3.0, + "learning_rate": 4.933358824161167e-06, + "loss": 0.6388422250747681, + "mean_token_accuracy": 0.7965542525053024, + "num_tokens": 801599.0, + "step": 98 + }, + { + "entropy": 0.4914955049753189, + "epoch": 0.043302351011481686, + "grad_norm": 2.734375, + "learning_rate": 4.929278208117378e-06, + "loss": 0.5518084168434143, + "mean_token_accuracy": 0.8212365508079529, + "num_tokens": 809791.0, + "step": 99 + }, + { + "entropy": 0.4663372188806534, + "epoch": 0.04373974849644614, + "grad_norm": 2.703125, + "learning_rate": 4.925078132986361e-06, + "loss": 0.521884560585022, + "mean_token_accuracy": 0.8260019570589066, + "num_tokens": 817983.0, + "step": 100 + }, + { + "entropy": 0.5335247218608856, + "epoch": 0.04417714598141061, + "grad_norm": 2.859375, + "learning_rate": 4.920758805305654e-06, + "loss": 0.5998464822769165, + "mean_token_accuracy": 0.8064516186714172, + "num_tokens": 826175.0, + "step": 101 + }, + { + "entropy": 0.5472059100866318, + "epoch": 0.04461454346637507, + "grad_norm": 2.953125, + "learning_rate": 4.9163204374770085e-06, + "loss": 0.6169013977050781, + "mean_token_accuracy": 0.8025415539741516, + "num_tokens": 834367.0, + "step": 102 + }, + { + "entropy": 0.615875169634819, + "epoch": 0.04505194095133953, + "grad_norm": 2.9375, + "learning_rate": 4.91176324775594e-06, + "loss": 0.6760657429695129, + "mean_token_accuracy": 0.7895894348621368, + "num_tokens": 842559.0, + "step": 103 + }, + { + "entropy": 0.36748453229665756, + "epoch": 0.04548933843630399, + "grad_norm": 2.34375, + "learning_rate": 4.907087460240999e-06, + "loss": 0.4150247275829315, + "mean_token_accuracy": 0.8599706739187241, + "num_tokens": 850751.0, + "step": 104 + }, + { + "entropy": 0.5654869377613068, + "epoch": 0.045926735921268454, + "grad_norm": 3.109375, + "learning_rate": 4.9022933048627496e-06, + "loss": 0.6229053735733032, + "mean_token_accuracy": 0.7930107563734055, + "num_tokens": 858943.0, + "step": 105 + }, + { + "entropy": 0.4413799047470093, + "epoch": 0.04636413340623292, + "grad_norm": 2.46875, + "learning_rate": 4.897381017372462e-06, + "loss": 0.45984703302383423, + "mean_token_accuracy": 0.8453079164028168, + "num_tokens": 867135.0, + "step": 106 + }, + { + "entropy": 0.4523310959339142, + "epoch": 0.046801530891197374, + "grad_norm": 2.53125, + "learning_rate": 4.8923508393305224e-06, + "loss": 0.4898549020290375, + "mean_token_accuracy": 0.8416422307491302, + "num_tokens": 875327.0, + "step": 107 + }, + { + "entropy": 0.4744609370827675, + "epoch": 0.04723892837616184, + "grad_norm": 2.765625, + "learning_rate": 4.887203018094552e-06, + "loss": 0.5368300676345825, + "mean_token_accuracy": 0.8267350792884827, + "num_tokens": 883519.0, + "step": 108 + }, + { + "entropy": 0.5006208494305611, + "epoch": 0.0476763258611263, + "grad_norm": 2.640625, + "learning_rate": 4.881937806807241e-06, + "loss": 0.5572711229324341, + "mean_token_accuracy": 0.821480929851532, + "num_tokens": 891711.0, + "step": 109 + }, + { + "entropy": 0.43279464542865753, + "epoch": 0.04811372334609076, + "grad_norm": 2.859375, + "learning_rate": 4.876555464383908e-06, + "loss": 0.4780406355857849, + "mean_token_accuracy": 0.8428641259670258, + "num_tokens": 899903.0, + "step": 110 + }, + { + "entropy": 0.49086206406354904, + "epoch": 0.04855112083105522, + "grad_norm": 2.640625, + "learning_rate": 4.871056255499758e-06, + "loss": 0.5436443090438843, + "mean_token_accuracy": 0.8207478076219559, + "num_tokens": 908095.0, + "step": 111 + }, + { + "entropy": 0.42109110951423645, + "epoch": 0.048988518316019684, + "grad_norm": 2.6875, + "learning_rate": 4.8654404505768735e-06, + "loss": 0.48846113681793213, + "mean_token_accuracy": 0.8423753827810287, + "num_tokens": 916287.0, + "step": 112 + }, + { + "entropy": 0.45168935507535934, + "epoch": 0.04942591580098414, + "grad_norm": 2.453125, + "learning_rate": 4.859708325770919e-06, + "loss": 0.5054396390914917, + "mean_token_accuracy": 0.8401759564876556, + "num_tokens": 924479.0, + "step": 113 + }, + { + "entropy": 0.4531315267086029, + "epoch": 0.049863313285948604, + "grad_norm": 2.71875, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.504815936088562, + "mean_token_accuracy": 0.8376099616289139, + "num_tokens": 932671.0, + "step": 114 + }, + { + "entropy": 0.48239656537771225, + "epoch": 0.05030071077091307, + "grad_norm": 2.625, + "learning_rate": 4.84789624971857e-06, + "loss": 0.5212786197662354, + "mean_token_accuracy": 0.8241691142320633, + "num_tokens": 940863.0, + "step": 115 + }, + { + "entropy": 0.4362593963742256, + "epoch": 0.05073810825587753, + "grad_norm": 2.5625, + "learning_rate": 4.841816879327764e-06, + "loss": 0.4883577823638916, + "mean_token_accuracy": 0.8366324603557587, + "num_tokens": 949055.0, + "step": 116 + }, + { + "entropy": 0.4954916387796402, + "epoch": 0.05117550574084199, + "grad_norm": 2.640625, + "learning_rate": 4.835622350736499e-06, + "loss": 0.5549702048301697, + "mean_token_accuracy": 0.8211143612861633, + "num_tokens": 957247.0, + "step": 117 + }, + { + "entropy": 0.457555316388607, + "epoch": 0.05161290322580645, + "grad_norm": 2.46875, + "learning_rate": 4.8293129685590164e-06, + "loss": 0.5015179514884949, + "mean_token_accuracy": 0.832722395658493, + "num_tokens": 965439.0, + "step": 118 + }, + { + "entropy": 0.4725177064538002, + "epoch": 0.052050300710770915, + "grad_norm": 2.546875, + "learning_rate": 4.822889043057446e-06, + "loss": 0.5209950804710388, + "mean_token_accuracy": 0.8273460417985916, + "num_tokens": 973631.0, + "step": 119 + }, + { + "entropy": 0.4489220380783081, + "epoch": 0.05248769819573537, + "grad_norm": 2.421875, + "learning_rate": 4.816350890126556e-06, + "loss": 0.4854542315006256, + "mean_token_accuracy": 0.8404203355312347, + "num_tokens": 981823.0, + "step": 120 + }, + { + "entropy": 0.47007766366004944, + "epoch": 0.052925095680699835, + "grad_norm": 2.671875, + "learning_rate": 4.809698831278217e-06, + "loss": 0.5187719464302063, + "mean_token_accuracy": 0.8310117423534393, + "num_tokens": 990015.0, + "step": 121 + }, + { + "entropy": 0.4113807678222656, + "epoch": 0.0533624931656643, + "grad_norm": 2.3125, + "learning_rate": 4.802933193625593e-06, + "loss": 0.4729251563549042, + "mean_token_accuracy": 0.8432306945323944, + "num_tokens": 998207.0, + "step": 122 + }, + { + "entropy": 0.40296216309070587, + "epoch": 0.05379989065062876, + "grad_norm": 2.5, + "learning_rate": 4.796054309867053e-06, + "loss": 0.4630545973777771, + "mean_token_accuracy": 0.8541055619716644, + "num_tokens": 1006399.0, + "step": 123 + }, + { + "entropy": 0.5370416194200516, + "epoch": 0.05423728813559322, + "grad_norm": 2.890625, + "learning_rate": 4.789062518269812e-06, + "loss": 0.6142182350158691, + "mean_token_accuracy": 0.8086510300636292, + "num_tokens": 1014591.0, + "step": 124 + }, + { + "entropy": 0.41779524832963943, + "epoch": 0.05467468562055768, + "grad_norm": 2.21875, + "learning_rate": 4.781958162653298e-06, + "loss": 0.4485539197921753, + "mean_token_accuracy": 0.851783961057663, + "num_tokens": 1022783.0, + "step": 125 + }, + { + "entropy": 0.47237853705883026, + "epoch": 0.055112083105522146, + "grad_norm": 2.5625, + "learning_rate": 4.774741592372242e-06, + "loss": 0.5322825908660889, + "mean_token_accuracy": 0.8250244408845901, + "num_tokens": 1030975.0, + "step": 126 + }, + { + "entropy": 0.5153854340314865, + "epoch": 0.0555494805904866, + "grad_norm": 2.65625, + "learning_rate": 4.767413162299501e-06, + "loss": 0.5750604271888733, + "mean_token_accuracy": 0.8148826956748962, + "num_tokens": 1039167.0, + "step": 127 + }, + { + "entropy": 0.48488766700029373, + "epoch": 0.055986878075451066, + "grad_norm": 2.40625, + "learning_rate": 4.759973232808609e-06, + "loss": 0.5179052352905273, + "mean_token_accuracy": 0.8273460417985916, + "num_tokens": 1047359.0, + "step": 128 + }, + { + "entropy": 0.45713143050670624, + "epoch": 0.05642427556041553, + "grad_norm": 2.390625, + "learning_rate": 4.752422169756048e-06, + "loss": 0.4798344373703003, + "mean_token_accuracy": 0.8387096673250198, + "num_tokens": 1055551.0, + "step": 129 + }, + { + "entropy": 0.5863286033272743, + "epoch": 0.056861673045379986, + "grad_norm": 2.96875, + "learning_rate": 4.744760344463267e-06, + "loss": 0.6585489511489868, + "mean_token_accuracy": 0.7889784872531891, + "num_tokens": 1063743.0, + "step": 130 + }, + { + "entropy": 0.5283961147069931, + "epoch": 0.05729907053034445, + "grad_norm": 2.546875, + "learning_rate": 4.736988133698416e-06, + "loss": 0.5767723917961121, + "mean_token_accuracy": 0.8143939226865768, + "num_tokens": 1071935.0, + "step": 131 + }, + { + "entropy": 0.4788113385438919, + "epoch": 0.05773646801530891, + "grad_norm": 2.640625, + "learning_rate": 4.729105919657818e-06, + "loss": 0.5062345862388611, + "mean_token_accuracy": 0.8274682313203812, + "num_tokens": 1080127.0, + "step": 132 + }, + { + "entropy": 0.4851013496518135, + "epoch": 0.058173865500273376, + "grad_norm": 2.578125, + "learning_rate": 4.721114089947181e-06, + "loss": 0.5487649440765381, + "mean_token_accuracy": 0.8216031342744827, + "num_tokens": 1088319.0, + "step": 133 + }, + { + "entropy": 0.3871802017092705, + "epoch": 0.05861126298523783, + "grad_norm": 2.40625, + "learning_rate": 4.713013037562531e-06, + "loss": 0.4369054436683655, + "mean_token_accuracy": 0.8519765138626099, + "num_tokens": 1096268.0, + "step": 134 + }, + { + "entropy": 0.47404175996780396, + "epoch": 0.0590486604702023, + "grad_norm": 2.5, + "learning_rate": 4.704803160870888e-06, + "loss": 0.5088684558868408, + "mean_token_accuracy": 0.8313782960176468, + "num_tokens": 1104460.0, + "step": 135 + }, + { + "entropy": 0.578920029103756, + "epoch": 0.05948605795516676, + "grad_norm": 2.71875, + "learning_rate": 4.69648486359068e-06, + "loss": 0.635509729385376, + "mean_token_accuracy": 0.8016862124204636, + "num_tokens": 1112652.0, + "step": 136 + }, + { + "entropy": 0.5153625160455704, + "epoch": 0.05992345544013122, + "grad_norm": 2.5625, + "learning_rate": 4.6880585547718845e-06, + "loss": 0.5665484070777893, + "mean_token_accuracy": 0.816837728023529, + "num_tokens": 1120844.0, + "step": 137 + }, + { + "entropy": 0.49980155378580093, + "epoch": 0.06036085292509568, + "grad_norm": 2.3125, + "learning_rate": 4.67952464877592e-06, + "loss": 0.5389302968978882, + "mean_token_accuracy": 0.8242912888526917, + "num_tokens": 1129036.0, + "step": 138 + }, + { + "entropy": 0.550069123506546, + "epoch": 0.060798250410060144, + "grad_norm": 2.890625, + "learning_rate": 4.670883565255264e-06, + "loss": 0.6364497542381287, + "mean_token_accuracy": 0.8036412447690964, + "num_tokens": 1137228.0, + "step": 139 + }, + { + "entropy": 0.4852010905742645, + "epoch": 0.0612356478950246, + "grad_norm": 2.390625, + "learning_rate": 4.662135729132821e-06, + "loss": 0.5219293832778931, + "mean_token_accuracy": 0.8240469396114349, + "num_tokens": 1145420.0, + "step": 140 + }, + { + "entropy": 0.4792797490954399, + "epoch": 0.061673045379989064, + "grad_norm": 2.5625, + "learning_rate": 4.653281570581023e-06, + "loss": 0.5293768048286438, + "mean_token_accuracy": 0.8225806355476379, + "num_tokens": 1153612.0, + "step": 141 + }, + { + "entropy": 0.48140496760606766, + "epoch": 0.06211044286495353, + "grad_norm": 2.359375, + "learning_rate": 4.644321525000681e-06, + "loss": 0.5096234679222107, + "mean_token_accuracy": 0.834921807050705, + "num_tokens": 1161804.0, + "step": 142 + }, + { + "entropy": 0.49927058815956116, + "epoch": 0.06254784034991799, + "grad_norm": 2.625, + "learning_rate": 4.635256032999569e-06, + "loss": 0.5446697473526001, + "mean_token_accuracy": 0.8187927603721619, + "num_tokens": 1169996.0, + "step": 143 + }, + { + "entropy": 0.4959957152605057, + "epoch": 0.06298523783488245, + "grad_norm": 2.640625, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.5632919669151306, + "mean_token_accuracy": 0.8185483813285828, + "num_tokens": 1178188.0, + "step": 144 + }, + { + "entropy": 0.49702586978673935, + "epoch": 0.0634226353198469, + "grad_norm": 2.46875, + "learning_rate": 4.6168104980707105e-06, + "loss": 0.5302911996841431, + "mean_token_accuracy": 0.826979473233223, + "num_tokens": 1186380.0, + "step": 145 + }, + { + "entropy": 0.5061358213424683, + "epoch": 0.06386003280481137, + "grad_norm": 2.640625, + "learning_rate": 4.607431362197067e-06, + "loss": 0.5666874647140503, + "mean_token_accuracy": 0.8173264861106873, + "num_tokens": 1194572.0, + "step": 146 + }, + { + "entropy": 0.43493766337633133, + "epoch": 0.06429743028977583, + "grad_norm": 2.328125, + "learning_rate": 4.597948593966256e-06, + "loss": 0.44672006368637085, + "mean_token_accuracy": 0.8514173924922943, + "num_tokens": 1202764.0, + "step": 147 + }, + { + "entropy": 0.48091938346624374, + "epoch": 0.0647348277747403, + "grad_norm": 2.4375, + "learning_rate": 4.5883626596907945e-06, + "loss": 0.5102735757827759, + "mean_token_accuracy": 0.836754634976387, + "num_tokens": 1210956.0, + "step": 148 + }, + { + "entropy": 0.44341713935136795, + "epoch": 0.06517222525970476, + "grad_norm": 2.265625, + "learning_rate": 4.578674030756364e-06, + "loss": 0.46256667375564575, + "mean_token_accuracy": 0.8445747792720795, + "num_tokens": 1219148.0, + "step": 149 + }, + { + "entropy": 0.45063044130802155, + "epoch": 0.06560962274466922, + "grad_norm": 2.359375, + "learning_rate": 4.568883183598623e-06, + "loss": 0.4687718152999878, + "mean_token_accuracy": 0.8429749459028244, + "num_tokens": 1227312.0, + "step": 150 + }, + { + "entropy": 0.46204202622175217, + "epoch": 0.06604702022963369, + "grad_norm": 2.328125, + "learning_rate": 4.558990599679787e-06, + "loss": 0.5018566846847534, + "mean_token_accuracy": 0.831867054104805, + "num_tokens": 1235504.0, + "step": 151 + }, + { + "entropy": 0.44746626913547516, + "epoch": 0.06648441771459813, + "grad_norm": 2.40625, + "learning_rate": 4.548996765464947e-06, + "loss": 0.4644359350204468, + "mean_token_accuracy": 0.8445747792720795, + "num_tokens": 1243696.0, + "step": 152 + }, + { + "entropy": 0.4651805981993675, + "epoch": 0.0669218151995626, + "grad_norm": 2.46875, + "learning_rate": 4.538902172398151e-06, + "loss": 0.5121830105781555, + "mean_token_accuracy": 0.8330889493227005, + "num_tokens": 1251888.0, + "step": 153 + }, + { + "entropy": 0.4517085552215576, + "epoch": 0.06735921268452706, + "grad_norm": 2.5625, + "learning_rate": 4.528707316878233e-06, + "loss": 0.511009693145752, + "mean_token_accuracy": 0.837854340672493, + "num_tokens": 1260080.0, + "step": 154 + }, + { + "entropy": 0.4772433340549469, + "epoch": 0.06779661016949153, + "grad_norm": 2.40625, + "learning_rate": 4.518412700234407e-06, + "loss": 0.523794412612915, + "mean_token_accuracy": 0.8319481462240219, + "num_tokens": 1268224.0, + "step": 155 + }, + { + "entropy": 0.4500661939382553, + "epoch": 0.06823400765445599, + "grad_norm": 2.484375, + "learning_rate": 4.508018828701613e-06, + "loss": 0.4889807105064392, + "mean_token_accuracy": 0.8389540612697601, + "num_tokens": 1276416.0, + "step": 156 + }, + { + "entropy": 0.5382272824645042, + "epoch": 0.06867140513942045, + "grad_norm": 2.671875, + "learning_rate": 4.4975262133956235e-06, + "loss": 0.6048404574394226, + "mean_token_accuracy": 0.8051075339317322, + "num_tokens": 1284608.0, + "step": 157 + }, + { + "entropy": 0.4137018471956253, + "epoch": 0.06910880262438492, + "grad_norm": 2.34375, + "learning_rate": 4.486935370287907e-06, + "loss": 0.4585399627685547, + "mean_token_accuracy": 0.8515395969152451, + "num_tokens": 1292800.0, + "step": 158 + }, + { + "entropy": 0.5151834562420845, + "epoch": 0.06954620010934937, + "grad_norm": 2.78125, + "learning_rate": 4.476246820180259e-06, + "loss": 0.5867331027984619, + "mean_token_accuracy": 0.8187927752733231, + "num_tokens": 1300992.0, + "step": 159 + }, + { + "entropy": 0.4778127670288086, + "epoch": 0.06998359759431383, + "grad_norm": 2.390625, + "learning_rate": 4.465461088679189e-06, + "loss": 0.5396682024002075, + "mean_token_accuracy": 0.8280728608369827, + "num_tokens": 1309112.0, + "step": 160 + }, + { + "entropy": 0.5015709027647972, + "epoch": 0.07042099507927829, + "grad_norm": 2.484375, + "learning_rate": 4.454578706170075e-06, + "loss": 0.5446302890777588, + "mean_token_accuracy": 0.8217253088951111, + "num_tokens": 1317304.0, + "step": 161 + }, + { + "entropy": 0.4737255424261093, + "epoch": 0.07085839256424276, + "grad_norm": 2.578125, + "learning_rate": 4.44360020779108e-06, + "loss": 0.5138781666755676, + "mean_token_accuracy": 0.829912006855011, + "num_tokens": 1325496.0, + "step": 162 + }, + { + "entropy": 0.49487005919218063, + "epoch": 0.07129579004920722, + "grad_norm": 2.53125, + "learning_rate": 4.432526133406843e-06, + "loss": 0.5455970764160156, + "mean_token_accuracy": 0.8183040022850037, + "num_tokens": 1333688.0, + "step": 163 + }, + { + "entropy": 0.4824206978082657, + "epoch": 0.07173318753417168, + "grad_norm": 2.328125, + "learning_rate": 4.421357027581921e-06, + "loss": 0.5247961282730103, + "mean_token_accuracy": 0.8318670690059662, + "num_tokens": 1341880.0, + "step": 164 + }, + { + "entropy": 0.5007349699735641, + "epoch": 0.07217058501913615, + "grad_norm": 2.625, + "learning_rate": 4.410093439554019e-06, + "loss": 0.5320008993148804, + "mean_token_accuracy": 0.8262404948472977, + "num_tokens": 1349828.0, + "step": 165 + }, + { + "entropy": 0.45331911742687225, + "epoch": 0.0726079825041006, + "grad_norm": 2.375, + "learning_rate": 4.398735923206978e-06, + "loss": 0.5013130903244019, + "mean_token_accuracy": 0.8371212184429169, + "num_tokens": 1358020.0, + "step": 166 + }, + { + "entropy": 0.5055417120456696, + "epoch": 0.07304537998906506, + "grad_norm": 2.5625, + "learning_rate": 4.38728503704354e-06, + "loss": 0.5585504174232483, + "mean_token_accuracy": 0.820992186665535, + "num_tokens": 1366212.0, + "step": 167 + }, + { + "entropy": 0.5267198234796524, + "epoch": 0.07348277747402952, + "grad_norm": 2.5, + "learning_rate": 4.375741344157879e-06, + "loss": 0.5661095976829529, + "mean_token_accuracy": 0.8141495585441589, + "num_tokens": 1374404.0, + "step": 168 + }, + { + "entropy": 0.4898260533809662, + "epoch": 0.07392017495899399, + "grad_norm": 2.4375, + "learning_rate": 4.364105412207914e-06, + "loss": 0.5348383188247681, + "mean_token_accuracy": 0.8258797526359558, + "num_tokens": 1382596.0, + "step": 169 + }, + { + "entropy": 0.5083483383059502, + "epoch": 0.07435757244395845, + "grad_norm": 2.515625, + "learning_rate": 4.352377813387398e-06, + "loss": 0.5432155132293701, + "mean_token_accuracy": 0.8250244408845901, + "num_tokens": 1390788.0, + "step": 170 + }, + { + "entropy": 0.44939737766981125, + "epoch": 0.07479496992892291, + "grad_norm": 2.46875, + "learning_rate": 4.340559124397774e-06, + "loss": 0.500606119632721, + "mean_token_accuracy": 0.8351661711931229, + "num_tokens": 1398980.0, + "step": 171 + }, + { + "entropy": 0.44741733372211456, + "epoch": 0.07523236741388736, + "grad_norm": 2.546875, + "learning_rate": 4.32864992641982e-06, + "loss": 0.4959651231765747, + "mean_token_accuracy": 0.8425872474908829, + "num_tokens": 1407145.0, + "step": 172 + }, + { + "entropy": 0.5106001943349838, + "epoch": 0.07566976489885183, + "grad_norm": 2.59375, + "learning_rate": 4.316650805085068e-06, + "loss": 0.5437954068183899, + "mean_token_accuracy": 0.821130782365799, + "num_tokens": 1415243.0, + "step": 173 + }, + { + "entropy": 0.5708731412887573, + "epoch": 0.07610716238381629, + "grad_norm": 2.671875, + "learning_rate": 4.304562350447008e-06, + "loss": 0.6295499801635742, + "mean_token_accuracy": 0.8030303120613098, + "num_tokens": 1423435.0, + "step": 174 + }, + { + "entropy": 0.45462463796138763, + "epoch": 0.07654455986878075, + "grad_norm": 2.25, + "learning_rate": 4.292385156952069e-06, + "loss": 0.48111745715141296, + "mean_token_accuracy": 0.8400537669658661, + "num_tokens": 1431627.0, + "step": 175 + }, + { + "entropy": 0.6211910545825958, + "epoch": 0.07698195735374522, + "grad_norm": 2.625, + "learning_rate": 4.2801198234103895e-06, + "loss": 0.669766902923584, + "mean_token_accuracy": 0.7941104620695114, + "num_tokens": 1439819.0, + "step": 176 + }, + { + "entropy": 0.4480922445654869, + "epoch": 0.07741935483870968, + "grad_norm": 2.375, + "learning_rate": 4.267766952966369e-06, + "loss": 0.4825879633426666, + "mean_token_accuracy": 0.8418866097927094, + "num_tokens": 1448011.0, + "step": 177 + }, + { + "entropy": 0.5084129646420479, + "epoch": 0.07785675232367414, + "grad_norm": 2.734375, + "learning_rate": 4.255327153069014e-06, + "loss": 0.574237048625946, + "mean_token_accuracy": 0.8087732046842575, + "num_tokens": 1456203.0, + "step": 178 + }, + { + "entropy": 0.4354983791708946, + "epoch": 0.0782941498086386, + "grad_norm": 2.296875, + "learning_rate": 4.242801035442059e-06, + "loss": 0.47676539421081543, + "mean_token_accuracy": 0.8424975723028183, + "num_tokens": 1464395.0, + "step": 179 + }, + { + "entropy": 0.5039181187748909, + "epoch": 0.07873154729360306, + "grad_norm": 2.578125, + "learning_rate": 4.2301892160538895e-06, + "loss": 0.5764544010162354, + "mean_token_accuracy": 0.8211143761873245, + "num_tokens": 1472587.0, + "step": 180 + }, + { + "entropy": 0.4349247217178345, + "epoch": 0.07916894477856752, + "grad_norm": 2.265625, + "learning_rate": 4.217492315087255e-06, + "loss": 0.4684079885482788, + "mean_token_accuracy": 0.8488514274358749, + "num_tokens": 1480779.0, + "step": 181 + }, + { + "entropy": 0.529822438955307, + "epoch": 0.07960634226353198, + "grad_norm": 2.578125, + "learning_rate": 4.204710956908763e-06, + "loss": 0.592373251914978, + "mean_token_accuracy": 0.8134164214134216, + "num_tokens": 1488971.0, + "step": 182 + }, + { + "entropy": 0.5234254226088524, + "epoch": 0.08004373974849645, + "grad_norm": 2.484375, + "learning_rate": 4.191845770038186e-06, + "loss": 0.5723234415054321, + "mean_token_accuracy": 0.8096285313367844, + "num_tokens": 1497163.0, + "step": 183 + }, + { + "entropy": 0.5915178805589676, + "epoch": 0.08048113723346091, + "grad_norm": 2.6875, + "learning_rate": 4.178897387117547e-06, + "loss": 0.6336113214492798, + "mean_token_accuracy": 0.7989980429410934, + "num_tokens": 1505355.0, + "step": 184 + }, + { + "entropy": 0.5129002556204796, + "epoch": 0.08091853471842538, + "grad_norm": 2.53125, + "learning_rate": 4.1658664448800105e-06, + "loss": 0.5581027269363403, + "mean_token_accuracy": 0.8189149498939514, + "num_tokens": 1513547.0, + "step": 185 + }, + { + "entropy": 0.5568132996559143, + "epoch": 0.08135593220338982, + "grad_norm": 2.546875, + "learning_rate": 4.152753584118576e-06, + "loss": 0.5956103801727295, + "mean_token_accuracy": 0.8057184815406799, + "num_tokens": 1521739.0, + "step": 186 + }, + { + "entropy": 0.5218602269887924, + "epoch": 0.08179332968835429, + "grad_norm": 2.546875, + "learning_rate": 4.139559449654561e-06, + "loss": 0.5501708984375, + "mean_token_accuracy": 0.8205034136772156, + "num_tokens": 1529931.0, + "step": 187 + }, + { + "entropy": 0.48786159604787827, + "epoch": 0.08223072717331875, + "grad_norm": 2.359375, + "learning_rate": 4.1262846903058966e-06, + "loss": 0.5217365622520447, + "mean_token_accuracy": 0.8286901414394379, + "num_tokens": 1538123.0, + "step": 188 + }, + { + "entropy": 0.5472189486026764, + "epoch": 0.08266812465828322, + "grad_norm": 2.46875, + "learning_rate": 4.11292995885522e-06, + "loss": 0.5948308706283569, + "mean_token_accuracy": 0.8060850501060486, + "num_tokens": 1546315.0, + "step": 189 + }, + { + "entropy": 0.4940583184361458, + "epoch": 0.08310552214324768, + "grad_norm": 2.34375, + "learning_rate": 4.099495912017773e-06, + "loss": 0.5291944742202759, + "mean_token_accuracy": 0.8277125954627991, + "num_tokens": 1554507.0, + "step": 190 + }, + { + "entropy": 0.5450269728899002, + "epoch": 0.08354291962821214, + "grad_norm": 2.421875, + "learning_rate": 4.085983210409114e-06, + "loss": 0.5771312117576599, + "mean_token_accuracy": 0.8102394938468933, + "num_tokens": 1562699.0, + "step": 191 + }, + { + "entropy": 0.4809708967804909, + "epoch": 0.0839803171131766, + "grad_norm": 2.296875, + "learning_rate": 4.072392518512623e-06, + "loss": 0.515668511390686, + "mean_token_accuracy": 0.8312561064958572, + "num_tokens": 1570891.0, + "step": 192 + }, + { + "entropy": 0.5454308390617371, + "epoch": 0.08441771459814106, + "grad_norm": 2.40625, + "learning_rate": 4.058724504646834e-06, + "loss": 0.5704600811004639, + "mean_token_accuracy": 0.8162267804145813, + "num_tokens": 1579083.0, + "step": 193 + }, + { + "entropy": 0.46287182718515396, + "epoch": 0.08485511208310552, + "grad_norm": 2.296875, + "learning_rate": 4.044979840932567e-06, + "loss": 0.4835647940635681, + "mean_token_accuracy": 0.8400537520647049, + "num_tokens": 1587275.0, + "step": 194 + }, + { + "entropy": 0.4392080307006836, + "epoch": 0.08529250956806998, + "grad_norm": 2.359375, + "learning_rate": 4.031159203259876e-06, + "loss": 0.47868117690086365, + "mean_token_accuracy": 0.8413367718458176, + "num_tokens": 1595298.0, + "step": 195 + }, + { + "entropy": 0.4842827618122101, + "epoch": 0.08572990705303445, + "grad_norm": 2.453125, + "learning_rate": 4.0172632712548145e-06, + "loss": 0.5347608327865601, + "mean_token_accuracy": 0.8262463212013245, + "num_tokens": 1603490.0, + "step": 196 + }, + { + "entropy": 0.42982233315706253, + "epoch": 0.08616730453799891, + "grad_norm": 2.46875, + "learning_rate": 4.003292728246015e-06, + "loss": 0.4687855839729309, + "mean_token_accuracy": 0.8500733077526093, + "num_tokens": 1611682.0, + "step": 197 + }, + { + "entropy": 0.47412050515413284, + "epoch": 0.08660470202296337, + "grad_norm": 2.390625, + "learning_rate": 3.989248261231084e-06, + "loss": 0.4940436780452728, + "mean_token_accuracy": 0.8384652882814407, + "num_tokens": 1619874.0, + "step": 198 + }, + { + "entropy": 0.4639420583844185, + "epoch": 0.08704209950792782, + "grad_norm": 2.296875, + "learning_rate": 3.975130560842821e-06, + "loss": 0.4897710680961609, + "mean_token_accuracy": 0.8346655517816544, + "num_tokens": 1628017.0, + "step": 199 + }, + { + "entropy": 0.4265002906322479, + "epoch": 0.08747949699289229, + "grad_norm": 2.1875, + "learning_rate": 3.960940321315257e-06, + "loss": 0.4600346088409424, + "mean_token_accuracy": 0.8492179811000824, + "num_tokens": 1636209.0, + "step": 200 + }, + { + "entropy": 0.3981725126504898, + "epoch": 0.08791689447785675, + "grad_norm": 2.1875, + "learning_rate": 3.946678240449515e-06, + "loss": 0.43623173236846924, + "mean_token_accuracy": 0.8528836667537689, + "num_tokens": 1644401.0, + "step": 201 + }, + { + "entropy": 0.5264689773321152, + "epoch": 0.08835429196282121, + "grad_norm": 2.828125, + "learning_rate": 3.9323450195794954e-06, + "loss": 0.5827093720436096, + "mean_token_accuracy": 0.8156800270080566, + "num_tokens": 1652489.0, + "step": 202 + }, + { + "entropy": 0.4846732020378113, + "epoch": 0.08879168944778568, + "grad_norm": 2.390625, + "learning_rate": 3.9179413635373895e-06, + "loss": 0.5261396169662476, + "mean_token_accuracy": 0.8269794881343842, + "num_tokens": 1660681.0, + "step": 203 + }, + { + "entropy": 0.5061644613742828, + "epoch": 0.08922908693275014, + "grad_norm": 2.390625, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.5423262119293213, + "mean_token_accuracy": 0.8249022364616394, + "num_tokens": 1668873.0, + "step": 204 + }, + { + "entropy": 0.5438003838062286, + "epoch": 0.0896664844177146, + "grad_norm": 2.5625, + "learning_rate": 3.888925582549006e-06, + "loss": 0.5897356867790222, + "mean_token_accuracy": 0.8076735138893127, + "num_tokens": 1677065.0, + "step": 205 + }, + { + "entropy": 0.4519123062491417, + "epoch": 0.09010388190267905, + "grad_norm": 2.234375, + "learning_rate": 3.874314884445771e-06, + "loss": 0.4760674238204956, + "mean_token_accuracy": 0.8390762358903885, + "num_tokens": 1685257.0, + "step": 206 + }, + { + "entropy": 0.5100989043712616, + "epoch": 0.09054127938764352, + "grad_norm": 2.375, + "learning_rate": 3.859636604786372e-06, + "loss": 0.5384048223495483, + "mean_token_accuracy": 0.8235581666231155, + "num_tokens": 1693449.0, + "step": 207 + }, + { + "entropy": 0.4979291707277298, + "epoch": 0.09097867687260798, + "grad_norm": 2.359375, + "learning_rate": 3.8448914653711725e-06, + "loss": 0.5454040169715881, + "mean_token_accuracy": 0.82233627140522, + "num_tokens": 1701641.0, + "step": 208 + }, + { + "entropy": 0.48866020888090134, + "epoch": 0.09141607435757244, + "grad_norm": 2.390625, + "learning_rate": 3.830080191288342e-06, + "loss": 0.5353090167045593, + "mean_token_accuracy": 0.8250244408845901, + "num_tokens": 1709833.0, + "step": 209 + }, + { + "entropy": 0.4369449093937874, + "epoch": 0.09185347184253691, + "grad_norm": 2.21875, + "learning_rate": 3.815203510878209e-06, + "loss": 0.47227001190185547, + "mean_token_accuracy": 0.8410312831401825, + "num_tokens": 1718025.0, + "step": 210 + }, + { + "entropy": 0.5231678262352943, + "epoch": 0.09229086932750137, + "grad_norm": 2.546875, + "learning_rate": 3.8002621556974367e-06, + "loss": 0.5623139142990112, + "mean_token_accuracy": 0.8194037079811096, + "num_tokens": 1726217.0, + "step": 211 + }, + { + "entropy": 0.47718561440706253, + "epoch": 0.09272826681246583, + "grad_norm": 2.4375, + "learning_rate": 3.785256860483054e-06, + "loss": 0.5391553640365601, + "mean_token_accuracy": 0.8256353586912155, + "num_tokens": 1734409.0, + "step": 212 + }, + { + "entropy": 0.48632175475358963, + "epoch": 0.09316566429743028, + "grad_norm": 2.28125, + "learning_rate": 3.770188363116324e-06, + "loss": 0.5253764986991882, + "mean_token_accuracy": 0.8268572837114334, + "num_tokens": 1742601.0, + "step": 213 + }, + { + "entropy": 0.5466743558645248, + "epoch": 0.09360306178239475, + "grad_norm": 2.4375, + "learning_rate": 3.755057404586455e-06, + "loss": 0.5637357234954834, + "mean_token_accuracy": 0.8114613890647888, + "num_tokens": 1750793.0, + "step": 214 + }, + { + "entropy": 0.4229523092508316, + "epoch": 0.09404045926735921, + "grad_norm": 2.28125, + "learning_rate": 3.7398647289541703e-06, + "loss": 0.46079084277153015, + "mean_token_accuracy": 0.8492179811000824, + "num_tokens": 1758985.0, + "step": 215 + }, + { + "entropy": 0.5077691823244095, + "epoch": 0.09447785675232367, + "grad_norm": 2.40625, + "learning_rate": 3.724611083315109e-06, + "loss": 0.5458195209503174, + "mean_token_accuracy": 0.8192094713449478, + "num_tokens": 1767017.0, + "step": 216 + }, + { + "entropy": 0.45738635212183, + "epoch": 0.09491525423728814, + "grad_norm": 2.203125, + "learning_rate": 3.7092972177630998e-06, + "loss": 0.49977749586105347, + "mean_token_accuracy": 0.8341886550188065, + "num_tokens": 1775209.0, + "step": 217 + }, + { + "entropy": 0.5932013764977455, + "epoch": 0.0953526517222526, + "grad_norm": 2.515625, + "learning_rate": 3.6939238853532657e-06, + "loss": 0.6106526851654053, + "mean_token_accuracy": 0.8014418482780457, + "num_tokens": 1783401.0, + "step": 218 + }, + { + "entropy": 0.49210917949676514, + "epoch": 0.09579004920721707, + "grad_norm": 2.578125, + "learning_rate": 3.6784918420649952e-06, + "loss": 0.5386967658996582, + "mean_token_accuracy": 0.8255131840705872, + "num_tokens": 1791593.0, + "step": 219 + }, + { + "entropy": 0.4963834807276726, + "epoch": 0.09622744669218151, + "grad_norm": 2.28125, + "learning_rate": 3.663001846764769e-06, + "loss": 0.5275002717971802, + "mean_token_accuracy": 0.8225806504487991, + "num_tokens": 1799785.0, + "step": 220 + }, + { + "entropy": 0.4301696792244911, + "epoch": 0.09666484417714598, + "grad_norm": 2.3125, + "learning_rate": 3.6474546611688446e-06, + "loss": 0.46401843428611755, + "mean_token_accuracy": 0.8456361144781113, + "num_tokens": 1807918.0, + "step": 221 + }, + { + "entropy": 0.456448033452034, + "epoch": 0.09710224166211044, + "grad_norm": 2.34375, + "learning_rate": 3.631851049805793e-06, + "loss": 0.49123865365982056, + "mean_token_accuracy": 0.8367546498775482, + "num_tokens": 1816110.0, + "step": 222 + }, + { + "entropy": 0.5055864825844765, + "epoch": 0.0975396391470749, + "grad_norm": 2.34375, + "learning_rate": 3.6161917799789076e-06, + "loss": 0.5433682799339294, + "mean_token_accuracy": 0.8196480870246887, + "num_tokens": 1824302.0, + "step": 223 + }, + { + "entropy": 0.48989594727754593, + "epoch": 0.09797703663203937, + "grad_norm": 2.46875, + "learning_rate": 3.600477621728473e-06, + "loss": 0.5149828195571899, + "mean_token_accuracy": 0.8290567100048065, + "num_tokens": 1832494.0, + "step": 224 + }, + { + "entropy": 0.461464524269104, + "epoch": 0.09841443411700383, + "grad_norm": 2.140625, + "learning_rate": 3.5847093477938955e-06, + "loss": 0.46980535984039307, + "mean_token_accuracy": 0.843352884054184, + "num_tokens": 1840686.0, + "step": 225 + }, + { + "entropy": 0.5267182663083076, + "epoch": 0.09885183160196828, + "grad_norm": 2.5, + "learning_rate": 3.5688877335757055e-06, + "loss": 0.5791686773300171, + "mean_token_accuracy": 0.8118279427289963, + "num_tokens": 1848878.0, + "step": 226 + }, + { + "entropy": 0.5815688967704773, + "epoch": 0.09928922908693275, + "grad_norm": 2.421875, + "learning_rate": 3.553013557097428e-06, + "loss": 0.6133584380149841, + "mean_token_accuracy": 0.8036650568246841, + "num_tokens": 1856831.0, + "step": 227 + }, + { + "entropy": 0.4044683054089546, + "epoch": 0.09972662657189721, + "grad_norm": 2.234375, + "learning_rate": 3.5370875989673198e-06, + "loss": 0.43988388776779175, + "mean_token_accuracy": 0.851725623011589, + "num_tokens": 1864824.0, + "step": 228 + }, + { + "entropy": 0.44032787531614304, + "epoch": 0.10016402405686167, + "grad_norm": 2.328125, + "learning_rate": 3.521110642339991e-06, + "loss": 0.4767438769340515, + "mean_token_accuracy": 0.843352884054184, + "num_tokens": 1873016.0, + "step": 229 + }, + { + "entropy": 0.5057295858860016, + "epoch": 0.10060142154182614, + "grad_norm": 2.234375, + "learning_rate": 3.505083472877884e-06, + "loss": 0.5249795913696289, + "mean_token_accuracy": 0.8291788697242737, + "num_tokens": 1881208.0, + "step": 230 + }, + { + "entropy": 0.46067339181900024, + "epoch": 0.1010388190267906, + "grad_norm": 2.359375, + "learning_rate": 3.4890068787126475e-06, + "loss": 0.4979941248893738, + "mean_token_accuracy": 0.8371212184429169, + "num_tokens": 1889400.0, + "step": 231 + }, + { + "entropy": 0.3977827876806259, + "epoch": 0.10147621651175506, + "grad_norm": 2.3125, + "learning_rate": 3.472881650406373e-06, + "loss": 0.4426778256893158, + "mean_token_accuracy": 0.8523949086666107, + "num_tokens": 1897592.0, + "step": 232 + }, + { + "entropy": 0.49042895436286926, + "epoch": 0.10191361399671951, + "grad_norm": 2.34375, + "learning_rate": 3.4567085809127247e-06, + "loss": 0.5123292207717896, + "mean_token_accuracy": 0.8297898322343826, + "num_tokens": 1905784.0, + "step": 233 + }, + { + "entropy": 0.5105108842253685, + "epoch": 0.10235101148168398, + "grad_norm": 2.46875, + "learning_rate": 3.4404884655379433e-06, + "loss": 0.5438479781150818, + "mean_token_accuracy": 0.8196481019258499, + "num_tokens": 1913976.0, + "step": 234 + }, + { + "entropy": 0.4192558228969574, + "epoch": 0.10278840896664844, + "grad_norm": 2.1875, + "learning_rate": 3.424222101901738e-06, + "loss": 0.4592979848384857, + "mean_token_accuracy": 0.8531280606985092, + "num_tokens": 1922168.0, + "step": 235 + }, + { + "entropy": 0.49979063868522644, + "epoch": 0.1032258064516129, + "grad_norm": 2.359375, + "learning_rate": 3.4079102898980633e-06, + "loss": 0.5456841588020325, + "mean_token_accuracy": 0.8219696879386902, + "num_tokens": 1930360.0, + "step": 236 + }, + { + "entropy": 0.42070452123880386, + "epoch": 0.10366320393657737, + "grad_norm": 2.09375, + "learning_rate": 3.391553831655783e-06, + "loss": 0.44820719957351685, + "mean_token_accuracy": 0.8532502502202988, + "num_tokens": 1938552.0, + "step": 237 + }, + { + "entropy": 0.5203416794538498, + "epoch": 0.10410060142154183, + "grad_norm": 2.390625, + "learning_rate": 3.375153531499231e-06, + "loss": 0.5365148186683655, + "mean_token_accuracy": 0.8222140669822693, + "num_tokens": 1946744.0, + "step": 238 + }, + { + "entropy": 0.42554354667663574, + "epoch": 0.1045379989065063, + "grad_norm": 2.15625, + "learning_rate": 3.358710195908653e-06, + "loss": 0.44316649436950684, + "mean_token_accuracy": 0.8528836667537689, + "num_tokens": 1954936.0, + "step": 239 + }, + { + "entropy": 0.49205484241247177, + "epoch": 0.10497539639147074, + "grad_norm": 2.578125, + "learning_rate": 3.3422246334805504e-06, + "loss": 0.5142548084259033, + "mean_token_accuracy": 0.8328445851802826, + "num_tokens": 1963128.0, + "step": 240 + }, + { + "entropy": 0.45800766348838806, + "epoch": 0.1054127938764352, + "grad_norm": 2.328125, + "learning_rate": 3.3256976548879183e-06, + "loss": 0.4986850917339325, + "mean_token_accuracy": 0.8334555178880692, + "num_tokens": 1971320.0, + "step": 241 + }, + { + "entropy": 0.5307494774460793, + "epoch": 0.10585019136139967, + "grad_norm": 2.4375, + "learning_rate": 3.309130072840379e-06, + "loss": 0.5546558499336243, + "mean_token_accuracy": 0.8191862851381302, + "num_tokens": 1979406.0, + "step": 242 + }, + { + "entropy": 0.4573250040411949, + "epoch": 0.10628758884636413, + "grad_norm": 2.25, + "learning_rate": 3.292522702044221e-06, + "loss": 0.4930002689361572, + "mean_token_accuracy": 0.8427419364452362, + "num_tokens": 1987598.0, + "step": 243 + }, + { + "entropy": 0.5074465125799179, + "epoch": 0.1067249863313286, + "grad_norm": 2.3125, + "learning_rate": 3.27587635916233e-06, + "loss": 0.5355194211006165, + "mean_token_accuracy": 0.8251466304063797, + "num_tokens": 1995790.0, + "step": 244 + }, + { + "entropy": 0.44894687086343765, + "epoch": 0.10716238381629306, + "grad_norm": 2.25, + "learning_rate": 3.259191862774037e-06, + "loss": 0.4787830412387848, + "mean_token_accuracy": 0.8409090936183929, + "num_tokens": 2003982.0, + "step": 245 + }, + { + "entropy": 0.5497879087924957, + "epoch": 0.10759978130125752, + "grad_norm": 2.40625, + "learning_rate": 3.242470033334859e-06, + "loss": 0.605748176574707, + "mean_token_accuracy": 0.8096285462379456, + "num_tokens": 2012174.0, + "step": 246 + }, + { + "entropy": 0.4496011435985565, + "epoch": 0.10803717878622197, + "grad_norm": 2.296875, + "learning_rate": 3.225711693136156e-06, + "loss": 0.47699061036109924, + "mean_token_accuracy": 0.836754634976387, + "num_tokens": 2020366.0, + "step": 247 + }, + { + "entropy": 0.4718461260199547, + "epoch": 0.10847457627118644, + "grad_norm": 2.359375, + "learning_rate": 3.2089176662646946e-06, + "loss": 0.5072636604309082, + "mean_token_accuracy": 0.8390762507915497, + "num_tokens": 2028558.0, + "step": 248 + }, + { + "entropy": 0.4522283151745796, + "epoch": 0.1089119737561509, + "grad_norm": 2.25, + "learning_rate": 3.1920887785621233e-06, + "loss": 0.48412400484085083, + "mean_token_accuracy": 0.8443304002285004, + "num_tokens": 2036750.0, + "step": 249 + }, + { + "entropy": 0.48243827372789383, + "epoch": 0.10934937124111536, + "grad_norm": 2.203125, + "learning_rate": 3.175225857584364e-06, + "loss": 0.5176639556884766, + "mean_token_accuracy": 0.8324780166149139, + "num_tokens": 2044942.0, + "step": 250 + }, + { + "entropy": 0.43758024275302887, + "epoch": 0.10978676872607983, + "grad_norm": 2.40625, + "learning_rate": 3.1583297325609117e-06, + "loss": 0.4812491536140442, + "mean_token_accuracy": 0.8454301208257675, + "num_tokens": 2053134.0, + "step": 251 + }, + { + "entropy": 0.5806743204593658, + "epoch": 0.11022416621104429, + "grad_norm": 2.484375, + "learning_rate": 3.1414012343540667e-06, + "loss": 0.6051475405693054, + "mean_token_accuracy": 0.8000977486371994, + "num_tokens": 2061326.0, + "step": 252 + }, + { + "entropy": 0.4813562035560608, + "epoch": 0.11066156369600874, + "grad_norm": 2.40625, + "learning_rate": 3.1244411954180677e-06, + "loss": 0.4888189136981964, + "mean_token_accuracy": 0.8377321511507034, + "num_tokens": 2069518.0, + "step": 253 + }, + { + "entropy": 0.4688641130924225, + "epoch": 0.1110989611809732, + "grad_norm": 2.359375, + "learning_rate": 3.10745044975816e-06, + "loss": 0.4884192943572998, + "mean_token_accuracy": 0.8410312831401825, + "num_tokens": 2077710.0, + "step": 254 + }, + { + "entropy": 0.5002786964178085, + "epoch": 0.11153635866593767, + "grad_norm": 2.21875, + "learning_rate": 3.0904298328895865e-06, + "loss": 0.53147292137146, + "mean_token_accuracy": 0.8268572837114334, + "num_tokens": 2085902.0, + "step": 255 + }, + { + "entropy": 0.5001499354839325, + "epoch": 0.11197375615090213, + "grad_norm": 2.34375, + "learning_rate": 3.073380181796495e-06, + "loss": 0.5351055264472961, + "mean_token_accuracy": 0.8217253237962723, + "num_tokens": 2094094.0, + "step": 256 + }, + { + "epoch": 0.11197375615090213, + "eval_entropy": 0.4864933586810246, + "eval_loss": 0.5185804963111877, + "eval_mean_token_accuracy": 0.8292105375242628, + "eval_num_tokens": 2094094.0, + "eval_runtime": 32.8305, + "eval_samples_per_second": 29.333, + "eval_steps_per_second": 3.686, + "step": 256 + }, + { + "entropy": 0.5339047610759735, + "epoch": 0.1124111536358666, + "grad_norm": 2.453125, + "learning_rate": 3.056302334890786e-06, + "loss": 0.5549373030662537, + "mean_token_accuracy": 0.8195258975028992, + "num_tokens": 2102286.0, + "step": 257 + }, + { + "entropy": 0.5548300221562386, + "epoch": 0.11284855112083106, + "grad_norm": 2.421875, + "learning_rate": 3.039197131970881e-06, + "loss": 0.5810383558273315, + "mean_token_accuracy": 0.8132295906543732, + "num_tokens": 2110449.0, + "step": 258 + }, + { + "entropy": 0.48320460319519043, + "epoch": 0.11328594860579552, + "grad_norm": 2.28125, + "learning_rate": 3.022065414180425e-06, + "loss": 0.5107712745666504, + "mean_token_accuracy": 0.8238025456666946, + "num_tokens": 2118641.0, + "step": 259 + }, + { + "entropy": 0.500133715569973, + "epoch": 0.11372334609075997, + "grad_norm": 2.3125, + "learning_rate": 3.0049080239669243e-06, + "loss": 0.5098925232887268, + "mean_token_accuracy": 0.8319892585277557, + "num_tokens": 2126833.0, + "step": 260 + }, + { + "entropy": 0.4652687683701515, + "epoch": 0.11416074357572444, + "grad_norm": 2.390625, + "learning_rate": 2.9877258050403214e-06, + "loss": 0.5014001727104187, + "mean_token_accuracy": 0.8366324454545975, + "num_tokens": 2135025.0, + "step": 261 + }, + { + "entropy": 0.4709179848432541, + "epoch": 0.1145981410606889, + "grad_norm": 2.203125, + "learning_rate": 2.9705196023315007e-06, + "loss": 0.4898775815963745, + "mean_token_accuracy": 0.8367546498775482, + "num_tokens": 2143217.0, + "step": 262 + }, + { + "entropy": 0.4747740998864174, + "epoch": 0.11503553854565336, + "grad_norm": 2.34375, + "learning_rate": 2.9532902619507465e-06, + "loss": 0.5130186676979065, + "mean_token_accuracy": 0.8319892436265945, + "num_tokens": 2151409.0, + "step": 263 + }, + { + "entropy": 0.4799908697605133, + "epoch": 0.11547293603061783, + "grad_norm": 2.546875, + "learning_rate": 2.9360386311461276e-06, + "loss": 0.5306564569473267, + "mean_token_accuracy": 0.826979473233223, + "num_tokens": 2159601.0, + "step": 264 + }, + { + "entropy": 0.4681060016155243, + "epoch": 0.11591033351558229, + "grad_norm": 2.328125, + "learning_rate": 2.9187655582618413e-06, + "loss": 0.5099412202835083, + "mean_token_accuracy": 0.8284457474946976, + "num_tokens": 2167793.0, + "step": 265 + }, + { + "entropy": 0.4734398275613785, + "epoch": 0.11634773100054675, + "grad_norm": 2.296875, + "learning_rate": 2.901471892696493e-06, + "loss": 0.49233460426330566, + "mean_token_accuracy": 0.8350439816713333, + "num_tokens": 2175985.0, + "step": 266 + }, + { + "entropy": 0.5704035013914108, + "epoch": 0.1167851284855112, + "grad_norm": 2.65625, + "learning_rate": 2.8841584848613254e-06, + "loss": 0.6443942785263062, + "mean_token_accuracy": 0.79178886115551, + "num_tokens": 2184177.0, + "step": 267 + }, + { + "entropy": 0.5196888893842697, + "epoch": 0.11722252597047567, + "grad_norm": 2.390625, + "learning_rate": 2.8668261861384045e-06, + "loss": 0.5377823114395142, + "mean_token_accuracy": 0.821480929851532, + "num_tokens": 2192369.0, + "step": 268 + }, + { + "entropy": 0.4297257959842682, + "epoch": 0.11765992345544013, + "grad_norm": 2.140625, + "learning_rate": 2.849475848838749e-06, + "loss": 0.4501199722290039, + "mean_token_accuracy": 0.8475073277950287, + "num_tokens": 2200561.0, + "step": 269 + }, + { + "entropy": 0.5126072689890862, + "epoch": 0.1180973209404046, + "grad_norm": 2.34375, + "learning_rate": 2.83210832616042e-06, + "loss": 0.5289058089256287, + "mean_token_accuracy": 0.8217253237962723, + "num_tokens": 2208753.0, + "step": 270 + }, + { + "entropy": 0.539338693022728, + "epoch": 0.11853471842536906, + "grad_norm": 2.4375, + "learning_rate": 2.814724472146564e-06, + "loss": 0.5616109371185303, + "mean_token_accuracy": 0.8209921717643738, + "num_tokens": 2216945.0, + "step": 271 + }, + { + "entropy": 0.43294807523489, + "epoch": 0.11897211591033352, + "grad_norm": 2.234375, + "learning_rate": 2.7973251416434176e-06, + "loss": 0.451716810464859, + "mean_token_accuracy": 0.8498289287090302, + "num_tokens": 2225137.0, + "step": 272 + }, + { + "entropy": 0.47712908685207367, + "epoch": 0.11940951339529798, + "grad_norm": 2.328125, + "learning_rate": 2.7799111902582697e-06, + "loss": 0.5194007754325867, + "mean_token_accuracy": 0.8256353735923767, + "num_tokens": 2233329.0, + "step": 273 + }, + { + "entropy": 0.47110161930322647, + "epoch": 0.11984691088026243, + "grad_norm": 2.3125, + "learning_rate": 2.762483474317387e-06, + "loss": 0.49832338094711304, + "mean_token_accuracy": 0.8338220864534378, + "num_tokens": 2241521.0, + "step": 274 + }, + { + "entropy": 0.49556952714920044, + "epoch": 0.1202843083652269, + "grad_norm": 2.4375, + "learning_rate": 2.7450428508239024e-06, + "loss": 0.5383248925209045, + "mean_token_accuracy": 0.8240469396114349, + "num_tokens": 2249713.0, + "step": 275 + }, + { + "entropy": 0.43116144835948944, + "epoch": 0.12072170585019136, + "grad_norm": 2.28125, + "learning_rate": 2.7275901774156753e-06, + "loss": 0.47572997212409973, + "mean_token_accuracy": 0.847318559885025, + "num_tokens": 2257797.0, + "step": 276 + }, + { + "entropy": 0.5458333194255829, + "epoch": 0.12115910333515582, + "grad_norm": 2.375, + "learning_rate": 2.710126312323119e-06, + "loss": 0.5907471776008606, + "mean_token_accuracy": 0.8092619776725769, + "num_tokens": 2265989.0, + "step": 277 + }, + { + "entropy": 0.4849350154399872, + "epoch": 0.12159650082012029, + "grad_norm": 2.390625, + "learning_rate": 2.6926521143269914e-06, + "loss": 0.5167871713638306, + "mean_token_accuracy": 0.8285679519176483, + "num_tokens": 2274181.0, + "step": 278 + }, + { + "entropy": 0.48804011195898056, + "epoch": 0.12203389830508475, + "grad_norm": 2.375, + "learning_rate": 2.6751684427161684e-06, + "loss": 0.5357515215873718, + "mean_token_accuracy": 0.8296676427125931, + "num_tokens": 2282373.0, + "step": 279 + }, + { + "entropy": 0.4341088831424713, + "epoch": 0.1224712957900492, + "grad_norm": 2.28125, + "learning_rate": 2.6576761572453903e-06, + "loss": 0.461492657661438, + "mean_token_accuracy": 0.8447287380695343, + "num_tokens": 2290388.0, + "step": 280 + }, + { + "entropy": 0.39888056367635727, + "epoch": 0.12290869327501366, + "grad_norm": 2.1875, + "learning_rate": 2.6401761180929798e-06, + "loss": 0.4248638451099396, + "mean_token_accuracy": 0.8599706888198853, + "num_tokens": 2298580.0, + "step": 281 + }, + { + "entropy": 0.39297056198120117, + "epoch": 0.12334609075997813, + "grad_norm": 2.15625, + "learning_rate": 2.6226691858185454e-06, + "loss": 0.40902504324913025, + "mean_token_accuracy": 0.8653470277786255, + "num_tokens": 2306772.0, + "step": 282 + }, + { + "entropy": 0.4313372001051903, + "epoch": 0.12378348824494259, + "grad_norm": 2.140625, + "learning_rate": 2.6051562213206633e-06, + "loss": 0.44663628935813904, + "mean_token_accuracy": 0.8475073277950287, + "num_tokens": 2314964.0, + "step": 283 + }, + { + "entropy": 0.47445718944072723, + "epoch": 0.12422088572990705, + "grad_norm": 2.34375, + "learning_rate": 2.587638085794541e-06, + "loss": 0.5170879364013672, + "mean_token_accuracy": 0.8340664654970169, + "num_tokens": 2323156.0, + "step": 284 + }, + { + "entropy": 0.44119589775800705, + "epoch": 0.12465828321487152, + "grad_norm": 2.125, + "learning_rate": 2.5701156406896726e-06, + "loss": 0.4654120206832886, + "mean_token_accuracy": 0.8454301059246063, + "num_tokens": 2331348.0, + "step": 285 + }, + { + "entropy": 0.5091964900493622, + "epoch": 0.12509568069983598, + "grad_norm": 2.453125, + "learning_rate": 2.5525897476674722e-06, + "loss": 0.5454074144363403, + "mean_token_accuracy": 0.81659334897995, + "num_tokens": 2339540.0, + "step": 286 + }, + { + "entropy": 0.48265135288238525, + "epoch": 0.12553307818480045, + "grad_norm": 2.453125, + "learning_rate": 2.535061268558906e-06, + "loss": 0.5154399275779724, + "mean_token_accuracy": 0.8330889493227005, + "num_tokens": 2347732.0, + "step": 287 + }, + { + "entropy": 0.47054049372673035, + "epoch": 0.1259704756697649, + "grad_norm": 2.265625, + "learning_rate": 2.5175310653221092e-06, + "loss": 0.4928542673587799, + "mean_token_accuracy": 0.8404203355312347, + "num_tokens": 2355924.0, + "step": 288 + }, + { + "entropy": 0.5776143297553062, + "epoch": 0.12640787315472937, + "grad_norm": 2.46875, + "learning_rate": 2.5e-06, + "loss": 0.6230844259262085, + "mean_token_accuracy": 0.7988758534193039, + "num_tokens": 2364116.0, + "step": 289 + }, + { + "entropy": 0.5465107038617134, + "epoch": 0.1268452706396938, + "grad_norm": 2.46875, + "learning_rate": 2.482468934677891e-06, + "loss": 0.5890681743621826, + "mean_token_accuracy": 0.8060850352048874, + "num_tokens": 2372308.0, + "step": 290 + }, + { + "entropy": 0.47898172587156296, + "epoch": 0.12728266812465827, + "grad_norm": 2.28125, + "learning_rate": 2.464938731441095e-06, + "loss": 0.5131503939628601, + "mean_token_accuracy": 0.8317448645830154, + "num_tokens": 2380500.0, + "step": 291 + }, + { + "entropy": 0.5422700941562653, + "epoch": 0.12772006560962273, + "grad_norm": 2.609375, + "learning_rate": 2.447410252332528e-06, + "loss": 0.5898529887199402, + "mean_token_accuracy": 0.8073069453239441, + "num_tokens": 2388692.0, + "step": 292 + }, + { + "entropy": 0.49293333292007446, + "epoch": 0.1281574630945872, + "grad_norm": 2.421875, + "learning_rate": 2.429884359310328e-06, + "loss": 0.5128275156021118, + "mean_token_accuracy": 0.8294232487678528, + "num_tokens": 2396884.0, + "step": 293 + }, + { + "entropy": 0.435122549533844, + "epoch": 0.12859486057955166, + "grad_norm": 2.328125, + "learning_rate": 2.41236191420546e-06, + "loss": 0.47749215364456177, + "mean_token_accuracy": 0.8449413478374481, + "num_tokens": 2405076.0, + "step": 294 + }, + { + "entropy": 0.4070725217461586, + "epoch": 0.12903225806451613, + "grad_norm": 2.1875, + "learning_rate": 2.394843778679338e-06, + "loss": 0.44448477029800415, + "mean_token_accuracy": 0.8543499559164047, + "num_tokens": 2413268.0, + "step": 295 + }, + { + "entropy": 0.4112040773034096, + "epoch": 0.1294696555494806, + "grad_norm": 2.125, + "learning_rate": 2.377330814181455e-06, + "loss": 0.4309145510196686, + "mean_token_accuracy": 0.8550830781459808, + "num_tokens": 2421460.0, + "step": 296 + }, + { + "entropy": 0.5157284215092659, + "epoch": 0.12990705303444505, + "grad_norm": 2.5, + "learning_rate": 2.3598238819070206e-06, + "loss": 0.5801538228988647, + "mean_token_accuracy": 0.8158602267503738, + "num_tokens": 2429652.0, + "step": 297 + }, + { + "entropy": 0.4528968632221222, + "epoch": 0.13034445051940952, + "grad_norm": 2.171875, + "learning_rate": 2.3423238427546105e-06, + "loss": 0.4707263112068176, + "mean_token_accuracy": 0.8461632430553436, + "num_tokens": 2437844.0, + "step": 298 + }, + { + "entropy": 0.5282165333628654, + "epoch": 0.13078184800437398, + "grad_norm": 2.3125, + "learning_rate": 2.3248315572838316e-06, + "loss": 0.5393767356872559, + "mean_token_accuracy": 0.8227028399705887, + "num_tokens": 2446036.0, + "step": 299 + }, + { + "entropy": 0.5359873697161674, + "epoch": 0.13121924548933844, + "grad_norm": 2.5, + "learning_rate": 2.30734788567301e-06, + "loss": 0.5740209817886353, + "mean_token_accuracy": 0.8135386258363724, + "num_tokens": 2454228.0, + "step": 300 + }, + { + "entropy": 0.5769957453012466, + "epoch": 0.1316566429743029, + "grad_norm": 2.640625, + "learning_rate": 2.2898736876768816e-06, + "loss": 0.6145251393318176, + "mean_token_accuracy": 0.8029080927371979, + "num_tokens": 2462420.0, + "step": 301 + }, + { + "entropy": 0.42022210359573364, + "epoch": 0.13209404045926737, + "grad_norm": 2.34375, + "learning_rate": 2.272409822584325e-06, + "loss": 0.4709186851978302, + "mean_token_accuracy": 0.8477517068386078, + "num_tokens": 2470612.0, + "step": 302 + }, + { + "entropy": 0.5215721800923347, + "epoch": 0.13253143794423183, + "grad_norm": 2.65625, + "learning_rate": 2.2549571491760985e-06, + "loss": 0.5673218965530396, + "mean_token_accuracy": 0.81659334897995, + "num_tokens": 2478804.0, + "step": 303 + }, + { + "entropy": 0.45102396607398987, + "epoch": 0.13296883542919627, + "grad_norm": 2.21875, + "learning_rate": 2.237516525682614e-06, + "loss": 0.48201784491539, + "mean_token_accuracy": 0.8437194675207138, + "num_tokens": 2486996.0, + "step": 304 + }, + { + "entropy": 0.42183125764131546, + "epoch": 0.13340623291416073, + "grad_norm": 2.140625, + "learning_rate": 2.2200888097417308e-06, + "loss": 0.44949376583099365, + "mean_token_accuracy": 0.8519061654806137, + "num_tokens": 2495188.0, + "step": 305 + }, + { + "entropy": 0.5075167715549469, + "epoch": 0.1338436303991252, + "grad_norm": 2.453125, + "learning_rate": 2.2026748583565824e-06, + "loss": 0.5390216112136841, + "mean_token_accuracy": 0.8218998908996582, + "num_tokens": 2503172.0, + "step": 306 + }, + { + "entropy": 0.5761678591370583, + "epoch": 0.13428102788408966, + "grad_norm": 2.75, + "learning_rate": 2.1852755278534373e-06, + "loss": 0.6333814859390259, + "mean_token_accuracy": 0.7954545468091965, + "num_tokens": 2511364.0, + "step": 307 + }, + { + "entropy": 0.4733138978481293, + "epoch": 0.13471842536905412, + "grad_norm": 2.515625, + "learning_rate": 2.167891673839581e-06, + "loss": 0.5121445059776306, + "mean_token_accuracy": 0.8380987197160721, + "num_tokens": 2519556.0, + "step": 308 + }, + { + "entropy": 0.5125866606831551, + "epoch": 0.1351558228540186, + "grad_norm": 2.515625, + "learning_rate": 2.1505241511612522e-06, + "loss": 0.5592678785324097, + "mean_token_accuracy": 0.819892480969429, + "num_tokens": 2527748.0, + "step": 309 + }, + { + "entropy": 0.4545166566967964, + "epoch": 0.13559322033898305, + "grad_norm": 2.34375, + "learning_rate": 2.133173813861596e-06, + "loss": 0.4897732734680176, + "mean_token_accuracy": 0.8401759415864944, + "num_tokens": 2535940.0, + "step": 310 + }, + { + "entropy": 0.4526529014110565, + "epoch": 0.13603061782394751, + "grad_norm": 2.234375, + "learning_rate": 2.1158415151386746e-06, + "loss": 0.48374703526496887, + "mean_token_accuracy": 0.8405425250530243, + "num_tokens": 2544132.0, + "step": 311 + }, + { + "entropy": 0.5908744037151337, + "epoch": 0.13646801530891198, + "grad_norm": 2.578125, + "learning_rate": 2.098528107303508e-06, + "loss": 0.6271101832389832, + "mean_token_accuracy": 0.796676442027092, + "num_tokens": 2552324.0, + "step": 312 + }, + { + "entropy": 0.49712150543928146, + "epoch": 0.13690541279387644, + "grad_norm": 2.46875, + "learning_rate": 2.0812344417381595e-06, + "loss": 0.5376288890838623, + "mean_token_accuracy": 0.8271016776561737, + "num_tokens": 2560516.0, + "step": 313 + }, + { + "entropy": 0.5063036307692528, + "epoch": 0.1373428102788409, + "grad_norm": 2.40625, + "learning_rate": 2.0639613688538733e-06, + "loss": 0.5550069808959961, + "mean_token_accuracy": 0.8277125954627991, + "num_tokens": 2568708.0, + "step": 314 + }, + { + "entropy": 0.49224531650543213, + "epoch": 0.13778020776380537, + "grad_norm": 2.484375, + "learning_rate": 2.0467097380492547e-06, + "loss": 0.5338283777236938, + "mean_token_accuracy": 0.826735109090805, + "num_tokens": 2576900.0, + "step": 315 + }, + { + "entropy": 0.45498689264059067, + "epoch": 0.13821760524876983, + "grad_norm": 2.25, + "learning_rate": 2.0294803976685006e-06, + "loss": 0.4753957986831665, + "mean_token_accuracy": 0.8456840515136719, + "num_tokens": 2584810.0, + "step": 316 + }, + { + "entropy": 0.4325580522418022, + "epoch": 0.13865500273373427, + "grad_norm": 2.34375, + "learning_rate": 2.01227419495968e-06, + "loss": 0.4696834087371826, + "mean_token_accuracy": 0.8431084901094437, + "num_tokens": 2593002.0, + "step": 317 + }, + { + "entropy": 0.5289476662874222, + "epoch": 0.13909240021869873, + "grad_norm": 2.296875, + "learning_rate": 1.9950919760330757e-06, + "loss": 0.5645922422409058, + "mean_token_accuracy": 0.8153714537620544, + "num_tokens": 2601194.0, + "step": 318 + }, + { + "entropy": 0.4893813133239746, + "epoch": 0.1395297977036632, + "grad_norm": 2.28125, + "learning_rate": 1.9779345858195757e-06, + "loss": 0.5062471628189087, + "mean_token_accuracy": 0.8324780017137527, + "num_tokens": 2609386.0, + "step": 319 + }, + { + "entropy": 0.5512858629226685, + "epoch": 0.13996719518862766, + "grad_norm": 2.296875, + "learning_rate": 1.960802868029119e-06, + "loss": 0.5786765813827515, + "mean_token_accuracy": 0.8102394938468933, + "num_tokens": 2617578.0, + "step": 320 + }, + { + "entropy": 0.45561161637306213, + "epoch": 0.14040459267359212, + "grad_norm": 2.265625, + "learning_rate": 1.9436976651092143e-06, + "loss": 0.476117879152298, + "mean_token_accuracy": 0.8421309888362885, + "num_tokens": 2625770.0, + "step": 321 + }, + { + "entropy": 0.525338664650917, + "epoch": 0.14084199015855658, + "grad_norm": 2.359375, + "learning_rate": 1.926619818203506e-06, + "loss": 0.563406229019165, + "mean_token_accuracy": 0.8158602118492126, + "num_tokens": 2633962.0, + "step": 322 + }, + { + "entropy": 0.49841468036174774, + "epoch": 0.14127938764352105, + "grad_norm": 2.421875, + "learning_rate": 1.909570167110415e-06, + "loss": 0.5303950309753418, + "mean_token_accuracy": 0.8271016478538513, + "num_tokens": 2642154.0, + "step": 323 + }, + { + "entropy": 0.4283083751797676, + "epoch": 0.1417167851284855, + "grad_norm": 2.140625, + "learning_rate": 1.8925495502418407e-06, + "loss": 0.457165002822876, + "mean_token_accuracy": 0.8508064597845078, + "num_tokens": 2650346.0, + "step": 324 + }, + { + "entropy": 0.4319509118795395, + "epoch": 0.14215418261344998, + "grad_norm": 2.28125, + "learning_rate": 1.8755588045819325e-06, + "loss": 0.4763701856136322, + "mean_token_accuracy": 0.8459232449531555, + "num_tokens": 2658502.0, + "step": 325 + }, + { + "entropy": 0.5478535592556, + "epoch": 0.14259158009841444, + "grad_norm": 2.546875, + "learning_rate": 1.8585987656459337e-06, + "loss": 0.6074569821357727, + "mean_token_accuracy": 0.8050237745046616, + "num_tokens": 2666657.0, + "step": 326 + }, + { + "entropy": 0.5283005684614182, + "epoch": 0.1430289775833789, + "grad_norm": 2.265625, + "learning_rate": 1.841670267439088e-06, + "loss": 0.5483436584472656, + "mean_token_accuracy": 0.8168377429246902, + "num_tokens": 2674849.0, + "step": 327 + }, + { + "entropy": 0.43779096752405167, + "epoch": 0.14346637506834337, + "grad_norm": 2.28125, + "learning_rate": 1.8247741424156373e-06, + "loss": 0.4743049740791321, + "mean_token_accuracy": 0.8424975425004959, + "num_tokens": 2683041.0, + "step": 328 + }, + { + "entropy": 0.5099033936858177, + "epoch": 0.14390377255330783, + "grad_norm": 2.421875, + "learning_rate": 1.8079112214378769e-06, + "loss": 0.5568172931671143, + "mean_token_accuracy": 0.8189149498939514, + "num_tokens": 2691233.0, + "step": 329 + }, + { + "entropy": 0.4858943372964859, + "epoch": 0.1443411700382723, + "grad_norm": 2.34375, + "learning_rate": 1.7910823337353062e-06, + "loss": 0.5260504484176636, + "mean_token_accuracy": 0.8301564007997513, + "num_tokens": 2699425.0, + "step": 330 + }, + { + "entropy": 0.5257250666618347, + "epoch": 0.14477856752323673, + "grad_norm": 2.40625, + "learning_rate": 1.7742883068638447e-06, + "loss": 0.5469223856925964, + "mean_token_accuracy": 0.8225806504487991, + "num_tokens": 2707617.0, + "step": 331 + }, + { + "entropy": 0.42773452401161194, + "epoch": 0.1452159650082012, + "grad_norm": 2.21875, + "learning_rate": 1.7575299666651413e-06, + "loss": 0.4414147138595581, + "mean_token_accuracy": 0.8527614921331406, + "num_tokens": 2715809.0, + "step": 332 + }, + { + "entropy": 0.500542514026165, + "epoch": 0.14565336249316566, + "grad_norm": 2.25, + "learning_rate": 1.7408081372259633e-06, + "loss": 0.5075960159301758, + "mean_token_accuracy": 0.8308895230293274, + "num_tokens": 2724001.0, + "step": 333 + }, + { + "entropy": 0.4560134708881378, + "epoch": 0.14609075997813012, + "grad_norm": 2.296875, + "learning_rate": 1.7241236408376702e-06, + "loss": 0.48843127489089966, + "mean_token_accuracy": 0.8379765450954437, + "num_tokens": 2732193.0, + "step": 334 + }, + { + "entropy": 0.5988139659166336, + "epoch": 0.14652815746309458, + "grad_norm": 2.46875, + "learning_rate": 1.7074772979557802e-06, + "loss": 0.6490610241889954, + "mean_token_accuracy": 0.7976539582014084, + "num_tokens": 2740385.0, + "step": 335 + }, + { + "entropy": 0.3993486016988754, + "epoch": 0.14696555494805905, + "grad_norm": 1.9765625, + "learning_rate": 1.6908699271596213e-06, + "loss": 0.42430931329727173, + "mean_token_accuracy": 0.8637585490942001, + "num_tokens": 2748577.0, + "step": 336 + }, + { + "entropy": 0.48627281934022903, + "epoch": 0.1474029524330235, + "grad_norm": 2.203125, + "learning_rate": 1.6743023451120831e-06, + "loss": 0.5066972374916077, + "mean_token_accuracy": 0.8312561064958572, + "num_tokens": 2756769.0, + "step": 337 + }, + { + "entropy": 0.4686634689569473, + "epoch": 0.14784034991798797, + "grad_norm": 2.359375, + "learning_rate": 1.6577753665194502e-06, + "loss": 0.4975348711013794, + "mean_token_accuracy": 0.8268572986125946, + "num_tokens": 2764961.0, + "step": 338 + }, + { + "entropy": 0.4240003898739815, + "epoch": 0.14827774740295244, + "grad_norm": 2.046875, + "learning_rate": 1.6412898040913471e-06, + "loss": 0.44500935077667236, + "mean_token_accuracy": 0.8515395820140839, + "num_tokens": 2773153.0, + "step": 339 + }, + { + "entropy": 0.5179468542337418, + "epoch": 0.1487151448879169, + "grad_norm": 2.5625, + "learning_rate": 1.6248464685007692e-06, + "loss": 0.5472849607467651, + "mean_token_accuracy": 0.8209921717643738, + "num_tokens": 2781345.0, + "step": 340 + }, + { + "entropy": 0.45416006445884705, + "epoch": 0.14915254237288136, + "grad_norm": 2.21875, + "learning_rate": 1.6084461683442176e-06, + "loss": 0.47359973192214966, + "mean_token_accuracy": 0.8380987346172333, + "num_tokens": 2789537.0, + "step": 341 + }, + { + "entropy": 0.5557916834950447, + "epoch": 0.14958993985784583, + "grad_norm": 2.453125, + "learning_rate": 1.5920897101019384e-06, + "loss": 0.586639940738678, + "mean_token_accuracy": 0.811950147151947, + "num_tokens": 2797729.0, + "step": 342 + }, + { + "entropy": 0.419264979660511, + "epoch": 0.1500273373428103, + "grad_norm": 2.171875, + "learning_rate": 1.5757778980982627e-06, + "loss": 0.4477095305919647, + "mean_token_accuracy": 0.8466520011425018, + "num_tokens": 2805921.0, + "step": 343 + }, + { + "entropy": 0.4657460153102875, + "epoch": 0.15046473482777473, + "grad_norm": 2.203125, + "learning_rate": 1.5595115344620575e-06, + "loss": 0.49677377939224243, + "mean_token_accuracy": 0.8355327546596527, + "num_tokens": 2814113.0, + "step": 344 + }, + { + "entropy": 0.42323917895555496, + "epoch": 0.1509021323127392, + "grad_norm": 2.140625, + "learning_rate": 1.5432914190872757e-06, + "loss": 0.4586298167705536, + "mean_token_accuracy": 0.8498289436101913, + "num_tokens": 2822305.0, + "step": 345 + }, + { + "entropy": 0.4431135877966881, + "epoch": 0.15133952979770365, + "grad_norm": 2.375, + "learning_rate": 1.5271183495936273e-06, + "loss": 0.49553415179252625, + "mean_token_accuracy": 0.8379765301942825, + "num_tokens": 2830497.0, + "step": 346 + }, + { + "entropy": 0.5104446560144424, + "epoch": 0.15177692728266812, + "grad_norm": 2.40625, + "learning_rate": 1.5109931212873535e-06, + "loss": 0.5375692844390869, + "mean_token_accuracy": 0.8218475133180618, + "num_tokens": 2838689.0, + "step": 347 + }, + { + "entropy": 0.5319401547312737, + "epoch": 0.15221432476763258, + "grad_norm": 2.5, + "learning_rate": 1.494916527122116e-06, + "loss": 0.587618887424469, + "mean_token_accuracy": 0.8161045908927917, + "num_tokens": 2846881.0, + "step": 348 + }, + { + "entropy": 0.5874071270227432, + "epoch": 0.15265172225259704, + "grad_norm": 2.71875, + "learning_rate": 1.47888935766001e-06, + "loss": 0.6408997774124146, + "mean_token_accuracy": 0.7987536638975143, + "num_tokens": 2855073.0, + "step": 349 + }, + { + "entropy": 0.5707217156887054, + "epoch": 0.1530891197375615, + "grad_norm": 2.484375, + "learning_rate": 1.4629124010326808e-06, + "loss": 0.6072824001312256, + "mean_token_accuracy": 0.8013196438550949, + "num_tokens": 2863265.0, + "step": 350 + }, + { + "entropy": 0.47589151561260223, + "epoch": 0.15352651722252597, + "grad_norm": 2.28125, + "learning_rate": 1.446986442902574e-06, + "loss": 0.5004896521568298, + "mean_token_accuracy": 0.8291788846254349, + "num_tokens": 2871457.0, + "step": 351 + }, + { + "entropy": 0.43671607226133347, + "epoch": 0.15396391470749043, + "grad_norm": 2.296875, + "learning_rate": 1.4311122664242955e-06, + "loss": 0.46961554884910583, + "mean_token_accuracy": 0.8429863154888153, + "num_tokens": 2879649.0, + "step": 352 + }, + { + "entropy": 0.4778784140944481, + "epoch": 0.1544013121924549, + "grad_norm": 2.359375, + "learning_rate": 1.415290652206105e-06, + "loss": 0.49795201420783997, + "mean_token_accuracy": 0.834921807050705, + "num_tokens": 2887841.0, + "step": 353 + }, + { + "entropy": 0.4867304265499115, + "epoch": 0.15483870967741936, + "grad_norm": 2.203125, + "learning_rate": 1.399522378271527e-06, + "loss": 0.5051119327545166, + "mean_token_accuracy": 0.83321113884449, + "num_tokens": 2896033.0, + "step": 354 + }, + { + "entropy": 0.5506188422441483, + "epoch": 0.15527610716238383, + "grad_norm": 2.421875, + "learning_rate": 1.3838082200210932e-06, + "loss": 0.5850009322166443, + "mean_token_accuracy": 0.8002199530601501, + "num_tokens": 2904225.0, + "step": 355 + }, + { + "entropy": 0.5183043852448463, + "epoch": 0.1557135046473483, + "grad_norm": 2.265625, + "learning_rate": 1.3681489501942077e-06, + "loss": 0.5410799980163574, + "mean_token_accuracy": 0.82233627140522, + "num_tokens": 2912417.0, + "step": 356 + }, + { + "entropy": 0.5144079178571701, + "epoch": 0.15615090213231275, + "grad_norm": 2.234375, + "learning_rate": 1.3525453388311554e-06, + "loss": 0.5360651612281799, + "mean_token_accuracy": 0.8218474984169006, + "num_tokens": 2920609.0, + "step": 357 + }, + { + "entropy": 0.5170230269432068, + "epoch": 0.1565882996172772, + "grad_norm": 2.453125, + "learning_rate": 1.3369981532352317e-06, + "loss": 0.5590443015098572, + "mean_token_accuracy": 0.8179374486207962, + "num_tokens": 2928801.0, + "step": 358 + }, + { + "entropy": 0.4852587506175041, + "epoch": 0.15702569710224165, + "grad_norm": 2.265625, + "learning_rate": 1.3215081579350058e-06, + "loss": 0.5135608315467834, + "mean_token_accuracy": 0.8256353884935379, + "num_tokens": 2936993.0, + "step": 359 + }, + { + "entropy": 0.5194334834814072, + "epoch": 0.15746309458720611, + "grad_norm": 2.390625, + "learning_rate": 1.306076114646735e-06, + "loss": 0.5491058826446533, + "mean_token_accuracy": 0.8167155385017395, + "num_tokens": 2945185.0, + "step": 360 + }, + { + "entropy": 0.5145406797528267, + "epoch": 0.15790049207217058, + "grad_norm": 2.421875, + "learning_rate": 1.2907027822369006e-06, + "loss": 0.5343478322029114, + "mean_token_accuracy": 0.8262463212013245, + "num_tokens": 2953377.0, + "step": 361 + }, + { + "entropy": 0.43967072665691376, + "epoch": 0.15833788955713504, + "grad_norm": 2.15625, + "learning_rate": 1.2753889166848909e-06, + "loss": 0.4568479061126709, + "mean_token_accuracy": 0.8449413329362869, + "num_tokens": 2961569.0, + "step": 362 + }, + { + "entropy": 0.462729349732399, + "epoch": 0.1587752870420995, + "grad_norm": 2.1875, + "learning_rate": 1.2601352710458314e-06, + "loss": 0.4606771469116211, + "mean_token_accuracy": 0.8440860211849213, + "num_tokens": 2969761.0, + "step": 363 + }, + { + "entropy": 0.46076279133558273, + "epoch": 0.15921268452706397, + "grad_norm": 2.296875, + "learning_rate": 1.2449425954135452e-06, + "loss": 0.5077764987945557, + "mean_token_accuracy": 0.8333333283662796, + "num_tokens": 2977953.0, + "step": 364 + }, + { + "entropy": 0.4890613555908203, + "epoch": 0.15965008201202843, + "grad_norm": 2.453125, + "learning_rate": 1.2298116368836772e-06, + "loss": 0.5199705958366394, + "mean_token_accuracy": 0.8293010741472244, + "num_tokens": 2986145.0, + "step": 365 + }, + { + "entropy": 0.5147231072187424, + "epoch": 0.1600874794969929, + "grad_norm": 2.328125, + "learning_rate": 1.214743139516946e-06, + "loss": 0.547106921672821, + "mean_token_accuracy": 0.8238025456666946, + "num_tokens": 2994337.0, + "step": 366 + }, + { + "entropy": 0.4986709952354431, + "epoch": 0.16052487698195736, + "grad_norm": 2.53125, + "learning_rate": 1.1997378443025633e-06, + "loss": 0.5374823212623596, + "mean_token_accuracy": 0.8257575631141663, + "num_tokens": 3002529.0, + "step": 367 + }, + { + "entropy": 0.5491506159305573, + "epoch": 0.16096227446692182, + "grad_norm": 2.5625, + "learning_rate": 1.1847964891217923e-06, + "loss": 0.5859638452529907, + "mean_token_accuracy": 0.8053518980741501, + "num_tokens": 3010721.0, + "step": 368 + }, + { + "entropy": 0.5448054671287537, + "epoch": 0.1613996719518863, + "grad_norm": 2.390625, + "learning_rate": 1.169919808711659e-06, + "loss": 0.5831042528152466, + "mean_token_accuracy": 0.8087732344865799, + "num_tokens": 3018913.0, + "step": 369 + }, + { + "entropy": 0.44894183427095413, + "epoch": 0.16183706943685075, + "grad_norm": 2.0625, + "learning_rate": 1.1551085346288296e-06, + "loss": 0.47109442949295044, + "mean_token_accuracy": 0.8432306945323944, + "num_tokens": 3027105.0, + "step": 370 + }, + { + "entropy": 0.4341334030032158, + "epoch": 0.16227446692181519, + "grad_norm": 2.09375, + "learning_rate": 1.140363395213629e-06, + "loss": 0.45891037583351135, + "mean_token_accuracy": 0.8500733077526093, + "num_tokens": 3035297.0, + "step": 371 + }, + { + "entropy": 0.46006736159324646, + "epoch": 0.16271186440677965, + "grad_norm": 2.25, + "learning_rate": 1.1256851155542297e-06, + "loss": 0.4995250403881073, + "mean_token_accuracy": 0.8370302766561508, + "num_tokens": 3043386.0, + "step": 372 + }, + { + "entropy": 0.5512775182723999, + "epoch": 0.1631492618917441, + "grad_norm": 2.34375, + "learning_rate": 1.1110744174509952e-06, + "loss": 0.5817916989326477, + "mean_token_accuracy": 0.8120723366737366, + "num_tokens": 3051578.0, + "step": 373 + }, + { + "entropy": 0.5470651388168335, + "epoch": 0.16358665937670858, + "grad_norm": 2.703125, + "learning_rate": 1.0965320193809808e-06, + "loss": 0.5833251476287842, + "mean_token_accuracy": 0.8063294142484665, + "num_tokens": 3059770.0, + "step": 374 + }, + { + "entropy": 0.49073220789432526, + "epoch": 0.16402405686167304, + "grad_norm": 2.453125, + "learning_rate": 1.0820586364626103e-06, + "loss": 0.5221870541572571, + "mean_token_accuracy": 0.8249022364616394, + "num_tokens": 3067962.0, + "step": 375 + }, + { + "entropy": 0.4238805025815964, + "epoch": 0.1644614543466375, + "grad_norm": 2.15625, + "learning_rate": 1.0676549804205048e-06, + "loss": 0.45591771602630615, + "mean_token_accuracy": 0.8479961007833481, + "num_tokens": 3076154.0, + "step": 376 + }, + { + "entropy": 0.549527570605278, + "epoch": 0.16489885183160197, + "grad_norm": 2.46875, + "learning_rate": 1.0533217595504859e-06, + "loss": 0.5825474858283997, + "mean_token_accuracy": 0.8071847558021545, + "num_tokens": 3084346.0, + "step": 377 + }, + { + "entropy": 0.4517410472035408, + "epoch": 0.16533624931656643, + "grad_norm": 2.328125, + "learning_rate": 1.0390596786847435e-06, + "loss": 0.48170745372772217, + "mean_token_accuracy": 0.8393206298351288, + "num_tokens": 3092538.0, + "step": 378 + }, + { + "entropy": 0.4634730815887451, + "epoch": 0.1657736468015309, + "grad_norm": 2.21875, + "learning_rate": 1.02486943915718e-06, + "loss": 0.5015561580657959, + "mean_token_accuracy": 0.8304007798433304, + "num_tokens": 3100730.0, + "step": 379 + }, + { + "entropy": 0.5204268097877502, + "epoch": 0.16621104428649536, + "grad_norm": 2.3125, + "learning_rate": 1.0107517387689168e-06, + "loss": 0.5475565791130066, + "mean_token_accuracy": 0.8212365508079529, + "num_tokens": 3108922.0, + "step": 380 + }, + { + "entropy": 0.4305594936013222, + "epoch": 0.16664844177145982, + "grad_norm": 2.421875, + "learning_rate": 9.967072717539852e-07, + "loss": 0.46948927640914917, + "mean_token_accuracy": 0.8459188789129257, + "num_tokens": 3117114.0, + "step": 381 + }, + { + "entropy": 0.4303949624300003, + "epoch": 0.16708583925642428, + "grad_norm": 2.109375, + "learning_rate": 9.82736728745186e-07, + "loss": 0.4573507308959961, + "mean_token_accuracy": 0.8488514274358749, + "num_tokens": 3125306.0, + "step": 382 + }, + { + "entropy": 0.5324750170111656, + "epoch": 0.16752323674138875, + "grad_norm": 2.53125, + "learning_rate": 9.688407967401248e-07, + "loss": 0.570069432258606, + "mean_token_accuracy": 0.8161045908927917, + "num_tokens": 3133498.0, + "step": 383 + }, + { + "entropy": 0.4431682378053665, + "epoch": 0.1679606342263532, + "grad_norm": 2.40625, + "learning_rate": 9.550201590674343e-07, + "loss": 0.46429842710494995, + "mean_token_accuracy": 0.8443304151296616, + "num_tokens": 3141690.0, + "step": 384 + }, + { + "entropy": 0.3733269274234772, + "epoch": 0.16839803171131765, + "grad_norm": 2.203125, + "learning_rate": 9.412754953531664e-07, + "loss": 0.4124542474746704, + "mean_token_accuracy": 0.8643695116043091, + "num_tokens": 3149882.0, + "step": 385 + }, + { + "entropy": 0.42850935459136963, + "epoch": 0.1688354291962821, + "grad_norm": 2.453125, + "learning_rate": 9.276074814873778e-07, + "loss": 0.47506606578826904, + "mean_token_accuracy": 0.8418866097927094, + "num_tokens": 3158074.0, + "step": 386 + }, + { + "entropy": 0.5115486085414886, + "epoch": 0.16927282668124657, + "grad_norm": 2.46875, + "learning_rate": 9.140167895908867e-07, + "loss": 0.5542680025100708, + "mean_token_accuracy": 0.8277126103639603, + "num_tokens": 3166266.0, + "step": 387 + }, + { + "entropy": 0.5119223147630692, + "epoch": 0.16971022416621104, + "grad_norm": 2.390625, + "learning_rate": 9.005040879822269e-07, + "loss": 0.5519320368766785, + "mean_token_accuracy": 0.8231915831565857, + "num_tokens": 3174458.0, + "step": 388 + }, + { + "entropy": 0.5366015583276749, + "epoch": 0.1701476216511755, + "grad_norm": 2.5, + "learning_rate": 8.870700411447817e-07, + "loss": 0.6139038801193237, + "mean_token_accuracy": 0.8113391995429993, + "num_tokens": 3182650.0, + "step": 389 + }, + { + "entropy": 0.45184676349163055, + "epoch": 0.17058501913613996, + "grad_norm": 2.140625, + "learning_rate": 8.737153096941045e-07, + "loss": 0.46673744916915894, + "mean_token_accuracy": 0.8447262048721313, + "num_tokens": 3190815.0, + "step": 390 + }, + { + "entropy": 0.42296013981103897, + "epoch": 0.17102241662110443, + "grad_norm": 2.203125, + "learning_rate": 8.604405503454399e-07, + "loss": 0.4525824189186096, + "mean_token_accuracy": 0.8526447266340256, + "num_tokens": 3198964.0, + "step": 391 + }, + { + "entropy": 0.5525547116994858, + "epoch": 0.1714598141060689, + "grad_norm": 2.625, + "learning_rate": 8.472464158814256e-07, + "loss": 0.5755280256271362, + "mean_token_accuracy": 0.8087732195854187, + "num_tokens": 3207156.0, + "step": 392 + }, + { + "entropy": 0.4999287575483322, + "epoch": 0.17189721159103336, + "grad_norm": 2.171875, + "learning_rate": 8.341335551199903e-07, + "loss": 0.518805980682373, + "mean_token_accuracy": 0.8279569894075394, + "num_tokens": 3215348.0, + "step": 393 + }, + { + "entropy": 0.5184216350317001, + "epoch": 0.17233460907599782, + "grad_norm": 2.421875, + "learning_rate": 8.21102612882454e-07, + "loss": 0.562769889831543, + "mean_token_accuracy": 0.8147605061531067, + "num_tokens": 3223540.0, + "step": 394 + }, + { + "entropy": 0.4349280297756195, + "epoch": 0.17277200656096228, + "grad_norm": 2.203125, + "learning_rate": 8.081542299618139e-07, + "loss": 0.45633161067962646, + "mean_token_accuracy": 0.8467741906642914, + "num_tokens": 3231732.0, + "step": 395 + }, + { + "entropy": 0.5333330482244492, + "epoch": 0.17320940404592675, + "grad_norm": 2.46875, + "learning_rate": 7.952890430912374e-07, + "loss": 0.5744578838348389, + "mean_token_accuracy": 0.8159824013710022, + "num_tokens": 3239924.0, + "step": 396 + }, + { + "entropy": 0.5034205839037895, + "epoch": 0.1736468015308912, + "grad_norm": 2.34375, + "learning_rate": 7.825076849127458e-07, + "loss": 0.5307492017745972, + "mean_token_accuracy": 0.8242913037538528, + "num_tokens": 3248116.0, + "step": 397 + }, + { + "entropy": 0.471238374710083, + "epoch": 0.17408419901585565, + "grad_norm": 2.34375, + "learning_rate": 7.698107839461111e-07, + "loss": 0.5049214363098145, + "mean_token_accuracy": 0.8361437022686005, + "num_tokens": 3256308.0, + "step": 398 + }, + { + "entropy": 0.4965353161096573, + "epoch": 0.1745215965008201, + "grad_norm": 2.296875, + "learning_rate": 7.57198964557942e-07, + "loss": 0.49766212701797485, + "mean_token_accuracy": 0.8317448496818542, + "num_tokens": 3264500.0, + "step": 399 + }, + { + "entropy": 0.5184244439005852, + "epoch": 0.17495899398578457, + "grad_norm": 2.5, + "learning_rate": 7.446728469309872e-07, + "loss": 0.5625096559524536, + "mean_token_accuracy": 0.815004900097847, + "num_tokens": 3272692.0, + "step": 400 + }, + { + "entropy": 0.4760790169239044, + "epoch": 0.17539639147074904, + "grad_norm": 2.328125, + "learning_rate": 7.322330470336314e-07, + "loss": 0.5208472013473511, + "mean_token_accuracy": 0.8274995982646942, + "num_tokens": 3280738.0, + "step": 401 + }, + { + "entropy": 0.5227344185113907, + "epoch": 0.1758337889557135, + "grad_norm": 2.546875, + "learning_rate": 7.198801765896115e-07, + "loss": 0.5777583122253418, + "mean_token_accuracy": 0.8146696388721466, + "num_tokens": 3288788.0, + "step": 402 + }, + { + "entropy": 0.5049602687358856, + "epoch": 0.17627118644067796, + "grad_norm": 2.390625, + "learning_rate": 7.076148430479321e-07, + "loss": 0.5320097804069519, + "mean_token_accuracy": 0.8231915980577469, + "num_tokens": 3296980.0, + "step": 403 + }, + { + "entropy": 0.4577829986810684, + "epoch": 0.17670858392564243, + "grad_norm": 2.1875, + "learning_rate": 6.954376495529927e-07, + "loss": 0.49313080310821533, + "mean_token_accuracy": 0.8317448645830154, + "num_tokens": 3305172.0, + "step": 404 + }, + { + "entropy": 0.4651181995868683, + "epoch": 0.1771459814106069, + "grad_norm": 2.1875, + "learning_rate": 6.833491949149329e-07, + "loss": 0.49456295371055603, + "mean_token_accuracy": 0.8361437022686005, + "num_tokens": 3313364.0, + "step": 405 + }, + { + "entropy": 0.541761614382267, + "epoch": 0.17758337889557135, + "grad_norm": 2.515625, + "learning_rate": 6.713500735801811e-07, + "loss": 0.5926611423492432, + "mean_token_accuracy": 0.80840665102005, + "num_tokens": 3321556.0, + "step": 406 + }, + { + "entropy": 0.5580184161663055, + "epoch": 0.17802077638053582, + "grad_norm": 2.46875, + "learning_rate": 6.594408756022272e-07, + "loss": 0.584408164024353, + "mean_token_accuracy": 0.8082844614982605, + "num_tokens": 3329748.0, + "step": 407 + }, + { + "entropy": 0.5339596942067146, + "epoch": 0.17845817386550028, + "grad_norm": 2.390625, + "learning_rate": 6.476221866126029e-07, + "loss": 0.5697808265686035, + "mean_token_accuracy": 0.8195258975028992, + "num_tokens": 3337940.0, + "step": 408 + }, + { + "entropy": 0.43793395161628723, + "epoch": 0.17889557135046474, + "grad_norm": 2.28125, + "learning_rate": 6.358945877920861e-07, + "loss": 0.46238237619400024, + "mean_token_accuracy": 0.8409090787172318, + "num_tokens": 3346132.0, + "step": 409 + }, + { + "entropy": 0.4602765664458275, + "epoch": 0.1793329688354292, + "grad_norm": 2.34375, + "learning_rate": 6.242586558421216e-07, + "loss": 0.4967961013317108, + "mean_token_accuracy": 0.8357963114976883, + "num_tokens": 3354067.0, + "step": 410 + }, + { + "entropy": 0.47321972250938416, + "epoch": 0.17977036632039367, + "grad_norm": 2.328125, + "learning_rate": 6.127149629564605e-07, + "loss": 0.5182679295539856, + "mean_token_accuracy": 0.8352883756160736, + "num_tokens": 3362259.0, + "step": 411 + }, + { + "entropy": 0.4488251060247421, + "epoch": 0.1802077638053581, + "grad_norm": 2.4375, + "learning_rate": 6.012640767930217e-07, + "loss": 0.5010267496109009, + "mean_token_accuracy": 0.8410312682390213, + "num_tokens": 3370451.0, + "step": 412 + }, + { + "entropy": 0.5726240426301956, + "epoch": 0.18064516129032257, + "grad_norm": 2.6875, + "learning_rate": 5.899065604459814e-07, + "loss": 0.630211353302002, + "mean_token_accuracy": 0.8007087111473083, + "num_tokens": 3378643.0, + "step": 413 + }, + { + "entropy": 0.475062720477581, + "epoch": 0.18108255877528703, + "grad_norm": 2.21875, + "learning_rate": 5.786429724180801e-07, + "loss": 0.48651978373527527, + "mean_token_accuracy": 0.8344330340623856, + "num_tokens": 3386835.0, + "step": 414 + }, + { + "entropy": 0.5285491198301315, + "epoch": 0.1815199562602515, + "grad_norm": 2.46875, + "learning_rate": 5.674738665931575e-07, + "loss": 0.5599380731582642, + "mean_token_accuracy": 0.8178152590990067, + "num_tokens": 3395027.0, + "step": 415 + }, + { + "entropy": 0.4876674935221672, + "epoch": 0.18195735374521596, + "grad_norm": 2.234375, + "learning_rate": 5.563997922089193e-07, + "loss": 0.515315055847168, + "mean_token_accuracy": 0.8255131989717484, + "num_tokens": 3403219.0, + "step": 416 + }, + { + "entropy": 0.4463843032717705, + "epoch": 0.18239475123018042, + "grad_norm": 2.28125, + "learning_rate": 5.454212938299256e-07, + "loss": 0.4906759262084961, + "mean_token_accuracy": 0.8361436873674393, + "num_tokens": 3411411.0, + "step": 417 + }, + { + "entropy": 0.4502919465303421, + "epoch": 0.1828321487151449, + "grad_norm": 2.203125, + "learning_rate": 5.34538911320811e-07, + "loss": 0.4786447584629059, + "mean_token_accuracy": 0.8421309888362885, + "num_tokens": 3419603.0, + "step": 418 + }, + { + "entropy": 0.5915882289409637, + "epoch": 0.18326954620010935, + "grad_norm": 2.578125, + "learning_rate": 5.237531798197415e-07, + "loss": 0.6288676857948303, + "mean_token_accuracy": 0.8032746911048889, + "num_tokens": 3427795.0, + "step": 419 + }, + { + "entropy": 0.5324423611164093, + "epoch": 0.18370694368507381, + "grad_norm": 2.6875, + "learning_rate": 5.130646297120936e-07, + "loss": 0.5872654318809509, + "mean_token_accuracy": 0.8080400824546814, + "num_tokens": 3435987.0, + "step": 420 + }, + { + "entropy": 0.5749758183956146, + "epoch": 0.18414434117003828, + "grad_norm": 2.59375, + "learning_rate": 5.02473786604378e-07, + "loss": 0.6213216185569763, + "mean_token_accuracy": 0.7925220131874084, + "num_tokens": 3444179.0, + "step": 421 + }, + { + "entropy": 0.41174645721912384, + "epoch": 0.18458173865500274, + "grad_norm": 2.09375, + "learning_rate": 4.91981171298388e-07, + "loss": 0.4298594892024994, + "mean_token_accuracy": 0.8552329391241074, + "num_tokens": 3452181.0, + "step": 422 + }, + { + "entropy": 0.47275856137275696, + "epoch": 0.1850191361399672, + "grad_norm": 2.328125, + "learning_rate": 4.815872997655941e-07, + "loss": 0.5014662742614746, + "mean_token_accuracy": 0.836754634976387, + "num_tokens": 3460373.0, + "step": 423 + }, + { + "entropy": 0.4867643192410469, + "epoch": 0.18545653362493167, + "grad_norm": 2.53125, + "learning_rate": 4.712926831217685e-07, + "loss": 0.5217025279998779, + "mean_token_accuracy": 0.8223951309919357, + "num_tokens": 3468250.0, + "step": 424 + }, + { + "entropy": 0.3914794325828552, + "epoch": 0.1858939311098961, + "grad_norm": 2.171875, + "learning_rate": 4.610978276018496e-07, + "loss": 0.4261934161186218, + "mean_token_accuracy": 0.8550831079483032, + "num_tokens": 3476442.0, + "step": 425 + }, + { + "entropy": 0.4488445296883583, + "epoch": 0.18633132859486057, + "grad_norm": 2.25, + "learning_rate": 4.5100323453505344e-07, + "loss": 0.4937145709991455, + "mean_token_accuracy": 0.8369990140199661, + "num_tokens": 3484634.0, + "step": 426 + }, + { + "entropy": 0.5070808678865433, + "epoch": 0.18676872607982503, + "grad_norm": 2.25, + "learning_rate": 4.410094003202134e-07, + "loss": 0.5438359379768372, + "mean_token_accuracy": 0.8164711743593216, + "num_tokens": 3492826.0, + "step": 427 + }, + { + "entropy": 0.5789096802473068, + "epoch": 0.1872061235647895, + "grad_norm": 2.375, + "learning_rate": 4.3111681640137763e-07, + "loss": 0.610559344291687, + "mean_token_accuracy": 0.8032746911048889, + "num_tokens": 3501018.0, + "step": 428 + }, + { + "entropy": 0.48394788056612015, + "epoch": 0.18764352104975396, + "grad_norm": 2.375, + "learning_rate": 4.2132596924363666e-07, + "loss": 0.49888381361961365, + "mean_token_accuracy": 0.8296676576137543, + "num_tokens": 3509210.0, + "step": 429 + }, + { + "entropy": 0.5190452337265015, + "epoch": 0.18808091853471842, + "grad_norm": 2.328125, + "learning_rate": 4.116373403092058e-07, + "loss": 0.5448641777038574, + "mean_token_accuracy": 0.8240469247102737, + "num_tokens": 3517402.0, + "step": 430 + }, + { + "entropy": 0.4755612760782242, + "epoch": 0.18851831601968289, + "grad_norm": 2.3125, + "learning_rate": 4.020514060337447e-07, + "loss": 0.498245507478714, + "mean_token_accuracy": 0.8312561064958572, + "num_tokens": 3525594.0, + "step": 431 + }, + { + "entropy": 0.45102784037590027, + "epoch": 0.18895571350464735, + "grad_norm": 2.25, + "learning_rate": 3.925686378029331e-07, + "loss": 0.4863170087337494, + "mean_token_accuracy": 0.8376099616289139, + "num_tokens": 3533786.0, + "step": 432 + }, + { + "entropy": 0.4501868635416031, + "epoch": 0.1893931109896118, + "grad_norm": 2.125, + "learning_rate": 3.831895019292897e-07, + "loss": 0.47062358260154724, + "mean_token_accuracy": 0.8438416421413422, + "num_tokens": 3541978.0, + "step": 433 + }, + { + "entropy": 0.45614657551050186, + "epoch": 0.18983050847457628, + "grad_norm": 2.1875, + "learning_rate": 3.7391445962923824e-07, + "loss": 0.48604702949523926, + "mean_token_accuracy": 0.8367546498775482, + "num_tokens": 3550170.0, + "step": 434 + }, + { + "entropy": 0.4634532630443573, + "epoch": 0.19026790595954074, + "grad_norm": 2.203125, + "learning_rate": 3.647439670004316e-07, + "loss": 0.4933796525001526, + "mean_token_accuracy": 0.8395650088787079, + "num_tokens": 3558362.0, + "step": 435 + }, + { + "entropy": 0.4207415208220482, + "epoch": 0.1907053034445052, + "grad_norm": 2.15625, + "learning_rate": 3.5567847499932004e-07, + "loss": 0.4387015104293823, + "mean_token_accuracy": 0.8525171130895615, + "num_tokens": 3566554.0, + "step": 436 + }, + { + "entropy": 0.44867319613695145, + "epoch": 0.19114270092946967, + "grad_norm": 2.28125, + "learning_rate": 3.4671842941897764e-07, + "loss": 0.4772151708602905, + "mean_token_accuracy": 0.8452994525432587, + "num_tokens": 3574373.0, + "step": 437 + }, + { + "entropy": 0.4577394872903824, + "epoch": 0.19158009841443413, + "grad_norm": 2.3125, + "learning_rate": 3.3786427086718006e-07, + "loss": 0.4948572516441345, + "mean_token_accuracy": 0.8396871834993362, + "num_tokens": 3582565.0, + "step": 438 + }, + { + "entropy": 0.5138989388942719, + "epoch": 0.19201749589939857, + "grad_norm": 2.34375, + "learning_rate": 3.2911643474473647e-07, + "loss": 0.5472657680511475, + "mean_token_accuracy": 0.8216031342744827, + "num_tokens": 3590757.0, + "step": 439 + }, + { + "entropy": 0.5489735826849937, + "epoch": 0.19245489338436303, + "grad_norm": 2.328125, + "learning_rate": 3.2047535122408076e-07, + "loss": 0.5895254611968994, + "mean_token_accuracy": 0.8035190552473068, + "num_tokens": 3598949.0, + "step": 440 + }, + { + "entropy": 0.5162252485752106, + "epoch": 0.1928922908693275, + "grad_norm": 2.453125, + "learning_rate": 3.119414452281158e-07, + "loss": 0.5371385216712952, + "mean_token_accuracy": 0.8258797526359558, + "num_tokens": 3607141.0, + "step": 441 + }, + { + "entropy": 0.5523306131362915, + "epoch": 0.19332968835429196, + "grad_norm": 2.65625, + "learning_rate": 3.035151364093214e-07, + "loss": 0.5896844267845154, + "mean_token_accuracy": 0.807062566280365, + "num_tokens": 3615333.0, + "step": 442 + }, + { + "entropy": 0.4346807897090912, + "epoch": 0.19376708583925642, + "grad_norm": 2.140625, + "learning_rate": 2.9519683912911267e-07, + "loss": 0.47453874349594116, + "mean_token_accuracy": 0.8470185697078705, + "num_tokens": 3623525.0, + "step": 443 + }, + { + "entropy": 0.4658875912427902, + "epoch": 0.19420448332422088, + "grad_norm": 2.28125, + "learning_rate": 2.869869624374699e-07, + "loss": 0.4884585440158844, + "mean_token_accuracy": 0.8411309272050858, + "num_tokens": 3631330.0, + "step": 444 + }, + { + "entropy": 0.45280956476926804, + "epoch": 0.19464188080918535, + "grad_norm": 2.15625, + "learning_rate": 2.788859100528196e-07, + "loss": 0.4797264635562897, + "mean_token_accuracy": 0.8361436873674393, + "num_tokens": 3639522.0, + "step": 445 + }, + { + "entropy": 0.4983842372894287, + "epoch": 0.1950792782941498, + "grad_norm": 2.296875, + "learning_rate": 2.708940803421825e-07, + "loss": 0.512587308883667, + "mean_token_accuracy": 0.8296676427125931, + "num_tokens": 3647714.0, + "step": 446 + }, + { + "entropy": 0.4361315816640854, + "epoch": 0.19551667577911427, + "grad_norm": 2.265625, + "learning_rate": 2.630118663015849e-07, + "loss": 0.4708927273750305, + "mean_token_accuracy": 0.8421309888362885, + "num_tokens": 3655906.0, + "step": 447 + }, + { + "entropy": 0.49473853409290314, + "epoch": 0.19595407326407874, + "grad_norm": 2.265625, + "learning_rate": 2.552396555367334e-07, + "loss": 0.5099279880523682, + "mean_token_accuracy": 0.8274682313203812, + "num_tokens": 3664098.0, + "step": 448 + }, + { + "entropy": 0.4408658742904663, + "epoch": 0.1963914707490432, + "grad_norm": 2.15625, + "learning_rate": 2.4757783024395244e-07, + "loss": 0.46858102083206177, + "mean_token_accuracy": 0.8464076220989227, + "num_tokens": 3672290.0, + "step": 449 + }, + { + "entropy": 0.48043954372406006, + "epoch": 0.19682886823400766, + "grad_norm": 2.359375, + "learning_rate": 2.400267671913917e-07, + "loss": 0.5275200605392456, + "mean_token_accuracy": 0.8262463361024857, + "num_tokens": 3680482.0, + "step": 450 + }, + { + "entropy": 0.5617353320121765, + "epoch": 0.19726626571897213, + "grad_norm": 2.421875, + "learning_rate": 2.325868377004986e-07, + "loss": 0.5783557891845703, + "mean_token_accuracy": 0.8124389052391052, + "num_tokens": 3688674.0, + "step": 451 + }, + { + "entropy": 0.4827708601951599, + "epoch": 0.19770366320393656, + "grad_norm": 2.515625, + "learning_rate": 2.2525840762775863e-07, + "loss": 0.5348269939422607, + "mean_token_accuracy": 0.8273460566997528, + "num_tokens": 3696866.0, + "step": 452 + }, + { + "entropy": 0.5675117522478104, + "epoch": 0.19814106068890103, + "grad_norm": 2.46875, + "learning_rate": 2.1804183734670277e-07, + "loss": 0.593768835067749, + "mean_token_accuracy": 0.8073069453239441, + "num_tokens": 3705058.0, + "step": 453 + }, + { + "entropy": 0.5070274025201797, + "epoch": 0.1985784581738655, + "grad_norm": 2.609375, + "learning_rate": 2.1093748173018846e-07, + "loss": 0.5375203490257263, + "mean_token_accuracy": 0.8241690993309021, + "num_tokens": 3713250.0, + "step": 454 + }, + { + "entropy": 0.457052618265152, + "epoch": 0.19901585565882995, + "grad_norm": 2.328125, + "learning_rate": 2.039456901329473e-07, + "loss": 0.5010364651679993, + "mean_token_accuracy": 0.8365102708339691, + "num_tokens": 3721442.0, + "step": 455 + }, + { + "entropy": 0.5044142678380013, + "epoch": 0.19945325314379442, + "grad_norm": 2.203125, + "learning_rate": 1.9706680637440745e-07, + "loss": 0.511577844619751, + "mean_token_accuracy": 0.8309101909399033, + "num_tokens": 3729588.0, + "step": 456 + }, + { + "entropy": 0.5089931935071945, + "epoch": 0.19989065062875888, + "grad_norm": 2.34375, + "learning_rate": 1.9030116872178317e-07, + "loss": 0.5382363796234131, + "mean_token_accuracy": 0.8235581666231155, + "num_tokens": 3737780.0, + "step": 457 + }, + { + "entropy": 0.45246878266334534, + "epoch": 0.20032804811372334, + "grad_norm": 2.34375, + "learning_rate": 1.836491098734447e-07, + "loss": 0.4766382873058319, + "mean_token_accuracy": 0.8478040844202042, + "num_tokens": 3745662.0, + "step": 458 + }, + { + "entropy": 0.49909115582704544, + "epoch": 0.2007654455986878, + "grad_norm": 2.3125, + "learning_rate": 1.771109569425547e-07, + "loss": 0.532217800617218, + "mean_token_accuracy": 0.8260019570589066, + "num_tokens": 3753854.0, + "step": 459 + }, + { + "entropy": 0.46753478050231934, + "epoch": 0.20120284308365227, + "grad_norm": 2.1875, + "learning_rate": 1.7068703144098365e-07, + "loss": 0.49318355321884155, + "mean_token_accuracy": 0.8355327546596527, + "num_tokens": 3762046.0, + "step": 460 + }, + { + "entropy": 0.4208934009075165, + "epoch": 0.20164024056861674, + "grad_norm": 2.015625, + "learning_rate": 1.6437764926350074e-07, + "loss": 0.4540482759475708, + "mean_token_accuracy": 0.8530058562755585, + "num_tokens": 3770238.0, + "step": 461 + }, + { + "entropy": 0.4388285279273987, + "epoch": 0.2020776380535812, + "grad_norm": 2.109375, + "learning_rate": 1.5818312067223673e-07, + "loss": 0.45131394267082214, + "mean_token_accuracy": 0.8472629636526108, + "num_tokens": 3778430.0, + "step": 462 + }, + { + "entropy": 0.4319133758544922, + "epoch": 0.20251503553854566, + "grad_norm": 2.15625, + "learning_rate": 1.5210375028143097e-07, + "loss": 0.46854692697525024, + "mean_token_accuracy": 0.8405425101518631, + "num_tokens": 3786622.0, + "step": 463 + }, + { + "entropy": 0.49125488102436066, + "epoch": 0.20295243302351013, + "grad_norm": 2.34375, + "learning_rate": 1.4613983704244826e-07, + "loss": 0.5005199909210205, + "mean_token_accuracy": 0.8341886550188065, + "num_tokens": 3794814.0, + "step": 464 + }, + { + "entropy": 0.42622826248407364, + "epoch": 0.2033898305084746, + "grad_norm": 2.28125, + "learning_rate": 1.4029167422908107e-07, + "loss": 0.4538479447364807, + "mean_token_accuracy": 0.8445759415626526, + "num_tokens": 3802759.0, + "step": 465 + }, + { + "entropy": 0.5118953883647919, + "epoch": 0.20382722799343903, + "grad_norm": 2.359375, + "learning_rate": 1.345595494231261e-07, + "loss": 0.5550968647003174, + "mean_token_accuracy": 0.8173265010118484, + "num_tokens": 3810951.0, + "step": 466 + }, + { + "entropy": 0.4239061772823334, + "epoch": 0.2042646254784035, + "grad_norm": 2.15625, + "learning_rate": 1.2894374450024338e-07, + "loss": 0.43111279606819153, + "mean_token_accuracy": 0.8519061654806137, + "num_tokens": 3819143.0, + "step": 467 + }, + { + "entropy": 0.4749840125441551, + "epoch": 0.20470202296336795, + "grad_norm": 2.328125, + "learning_rate": 1.234445356160932e-07, + "loss": 0.5093938112258911, + "mean_token_accuracy": 0.8295454531908035, + "num_tokens": 3827335.0, + "step": 468 + }, + { + "entropy": 0.48437564074993134, + "epoch": 0.20513942044833242, + "grad_norm": 2.421875, + "learning_rate": 1.1806219319275918e-07, + "loss": 0.524286150932312, + "mean_token_accuracy": 0.8301564007997513, + "num_tokens": 3835527.0, + "step": 469 + }, + { + "entropy": 0.5392681062221527, + "epoch": 0.20557681793329688, + "grad_norm": 2.53125, + "learning_rate": 1.1279698190544918e-07, + "loss": 0.5733947157859802, + "mean_token_accuracy": 0.813049852848053, + "num_tokens": 3843719.0, + "step": 470 + }, + { + "entropy": 0.38556477427482605, + "epoch": 0.20601421541826134, + "grad_norm": 1.9453125, + "learning_rate": 1.0764916066947795e-07, + "loss": 0.41268569231033325, + "mean_token_accuracy": 0.8622882515192032, + "num_tokens": 3851867.0, + "step": 471 + }, + { + "entropy": 0.5322115048766136, + "epoch": 0.2064516129032258, + "grad_norm": 2.390625, + "learning_rate": 1.0261898262753811e-07, + "loss": 0.5719882249832153, + "mean_token_accuracy": 0.8161045908927917, + "num_tokens": 3860059.0, + "step": 472 + }, + { + "entropy": 0.45311254262924194, + "epoch": 0.20688901038819027, + "grad_norm": 2.234375, + "learning_rate": 9.770669513725128e-08, + "loss": 0.4832289218902588, + "mean_token_accuracy": 0.8361436724662781, + "num_tokens": 3868251.0, + "step": 473 + }, + { + "entropy": 0.5146927535533905, + "epoch": 0.20732640787315473, + "grad_norm": 2.328125, + "learning_rate": 9.291253975900138e-08, + "loss": 0.5571263432502747, + "mean_token_accuracy": 0.8212365657091141, + "num_tokens": 3876443.0, + "step": 474 + }, + { + "entropy": 0.47573499381542206, + "epoch": 0.2077638053581192, + "grad_norm": 2.265625, + "learning_rate": 8.823675224406052e-08, + "loss": 0.5182182788848877, + "mean_token_accuracy": 0.8257741034030914, + "num_tokens": 3884497.0, + "step": 475 + }, + { + "entropy": 0.5343288853764534, + "epoch": 0.20820120284308366, + "grad_norm": 2.625, + "learning_rate": 8.367956252299192e-08, + "loss": 0.5817354917526245, + "mean_token_accuracy": 0.8120723366737366, + "num_tokens": 3892689.0, + "step": 476 + }, + { + "entropy": 0.5038627311587334, + "epoch": 0.20863860032804812, + "grad_norm": 2.359375, + "learning_rate": 7.924119469434666e-08, + "loss": 0.5361433029174805, + "mean_token_accuracy": 0.8236803561449051, + "num_tokens": 3900881.0, + "step": 477 + }, + { + "entropy": 0.4371698275208473, + "epoch": 0.2090759978130126, + "grad_norm": 2.21875, + "learning_rate": 7.492186701364007e-08, + "loss": 0.4635161757469177, + "mean_token_accuracy": 0.8470185697078705, + "num_tokens": 3909073.0, + "step": 478 + }, + { + "entropy": 0.5315583720803261, + "epoch": 0.20951339529797702, + "grad_norm": 2.546875, + "learning_rate": 7.072179188262252e-08, + "loss": 0.567477822303772, + "mean_token_accuracy": 0.8213587403297424, + "num_tokens": 3917265.0, + "step": 479 + }, + { + "entropy": 0.4574076384305954, + "epoch": 0.2099507927829415, + "grad_norm": 2.28125, + "learning_rate": 6.664117583883272e-08, + "loss": 0.4899052381515503, + "mean_token_accuracy": 0.8353865444660187, + "num_tokens": 3925304.0, + "step": 480 + }, + { + "entropy": 0.5328256785869598, + "epoch": 0.21038819026790595, + "grad_norm": 2.40625, + "learning_rate": 6.268021954544095e-08, + "loss": 0.5751959085464478, + "mean_token_accuracy": 0.8115835636854172, + "num_tokens": 3933496.0, + "step": 481 + }, + { + "entropy": 0.4958766922354698, + "epoch": 0.2108255877528704, + "grad_norm": 2.25, + "learning_rate": 5.883911778138324e-08, + "loss": 0.5280551910400391, + "mean_token_accuracy": 0.8273460417985916, + "num_tokens": 3941688.0, + "step": 482 + }, + { + "entropy": 0.46615899354219437, + "epoch": 0.21126298523783488, + "grad_norm": 2.328125, + "learning_rate": 5.5118059431781e-08, + "loss": 0.4785313904285431, + "mean_token_accuracy": 0.8441089540719986, + "num_tokens": 3949774.0, + "step": 483 + }, + { + "entropy": 0.5289159864187241, + "epoch": 0.21170038272279934, + "grad_norm": 2.46875, + "learning_rate": 5.151722747865434e-08, + "loss": 0.5834380984306335, + "mean_token_accuracy": 0.8157380223274231, + "num_tokens": 3957966.0, + "step": 484 + }, + { + "entropy": 0.4474088177084923, + "epoch": 0.2121377802077638, + "grad_norm": 2.34375, + "learning_rate": 4.8036798991923925e-08, + "loss": 0.49255403876304626, + "mean_token_accuracy": 0.8377321660518646, + "num_tokens": 3966158.0, + "step": 485 + }, + { + "entropy": 0.4575616270303726, + "epoch": 0.21257517769272827, + "grad_norm": 2.265625, + "learning_rate": 4.467694512070242e-08, + "loss": 0.48688727617263794, + "mean_token_accuracy": 0.8357771188020706, + "num_tokens": 3974350.0, + "step": 486 + }, + { + "entropy": 0.44955013692379, + "epoch": 0.21301257517769273, + "grad_norm": 2.3125, + "learning_rate": 4.1437831084878974e-08, + "loss": 0.47580015659332275, + "mean_token_accuracy": 0.8404203206300735, + "num_tokens": 3982542.0, + "step": 487 + }, + { + "entropy": 0.4375215768814087, + "epoch": 0.2134499726626572, + "grad_norm": 2.078125, + "learning_rate": 3.831961616699464e-08, + "loss": 0.45662811398506165, + "mean_token_accuracy": 0.8451857268810272, + "num_tokens": 3990734.0, + "step": 488 + }, + { + "entropy": 0.4607667848467827, + "epoch": 0.21388737014762166, + "grad_norm": 2.296875, + "learning_rate": 3.5322453704410286e-08, + "loss": 0.501000165939331, + "mean_token_accuracy": 0.8324779868125916, + "num_tokens": 3998926.0, + "step": 489 + }, + { + "entropy": 0.4871944785118103, + "epoch": 0.21432476763258612, + "grad_norm": 2.375, + "learning_rate": 3.2446491081764566e-08, + "loss": 0.5080267786979675, + "mean_token_accuracy": 0.8289345055818558, + "num_tokens": 4007118.0, + "step": 490 + }, + { + "entropy": 0.48445210605859756, + "epoch": 0.21476216511755059, + "grad_norm": 2.265625, + "learning_rate": 2.969186972372806e-08, + "loss": 0.5219994187355042, + "mean_token_accuracy": 0.8257575780153275, + "num_tokens": 4015310.0, + "step": 491 + }, + { + "entropy": 0.5076911747455597, + "epoch": 0.21519956260251505, + "grad_norm": 2.34375, + "learning_rate": 2.7058725088047466e-08, + "loss": 0.5234426856040955, + "mean_token_accuracy": 0.8255131840705872, + "num_tokens": 4023502.0, + "step": 492 + }, + { + "entropy": 0.48257502168416977, + "epoch": 0.21563696008747948, + "grad_norm": 2.4375, + "learning_rate": 2.454718665888589e-08, + "loss": 0.5169442296028137, + "mean_token_accuracy": 0.8269111812114716, + "num_tokens": 4031419.0, + "step": 493 + }, + { + "entropy": 0.48920300602912903, + "epoch": 0.21607435757244395, + "grad_norm": 2.203125, + "learning_rate": 2.215737794045436e-08, + "loss": 0.5300370454788208, + "mean_token_accuracy": 0.8225806355476379, + "num_tokens": 4039611.0, + "step": 494 + }, + { + "entropy": 0.5572062358260155, + "epoch": 0.2165117550574084, + "grad_norm": 2.34375, + "learning_rate": 1.9889416450938335e-08, + "loss": 0.5881485939025879, + "mean_token_accuracy": 0.8054741024971008, + "num_tokens": 4047803.0, + "step": 495 + }, + { + "entropy": 0.42473385483026505, + "epoch": 0.21694915254237288, + "grad_norm": 2.125, + "learning_rate": 1.7743413716720394e-08, + "loss": 0.44505032896995544, + "mean_token_accuracy": 0.8527614921331406, + "num_tokens": 4055995.0, + "step": 496 + }, + { + "entropy": 0.5073227211833, + "epoch": 0.21738655002733734, + "grad_norm": 2.28125, + "learning_rate": 1.571947526689349e-08, + "loss": 0.5384668707847595, + "mean_token_accuracy": 0.8238025307655334, + "num_tokens": 4064187.0, + "step": 497 + }, + { + "entropy": 0.5434798374772072, + "epoch": 0.2178239475123018, + "grad_norm": 2.796875, + "learning_rate": 1.3817700628074582e-08, + "loss": 0.5646654367446899, + "mean_token_accuracy": 0.8227028399705887, + "num_tokens": 4072379.0, + "step": 498 + }, + { + "entropy": 0.40285155177116394, + "epoch": 0.21826134499726627, + "grad_norm": 2.125, + "learning_rate": 1.2038183319507957e-08, + "loss": 0.43041908740997314, + "mean_token_accuracy": 0.85713692009449, + "num_tokens": 4080489.0, + "step": 499 + }, + { + "entropy": 0.5109951198101044, + "epoch": 0.21869874248223073, + "grad_norm": 2.390625, + "learning_rate": 1.0381010848466988e-08, + "loss": 0.5344597101211548, + "mean_token_accuracy": 0.823924720287323, + "num_tokens": 4088681.0, + "step": 500 + }, + { + "entropy": 0.5647617131471634, + "epoch": 0.2191361399671952, + "grad_norm": 2.40625, + "learning_rate": 8.84626470595229e-09, + "loss": 0.5811930894851685, + "mean_token_accuracy": 0.8090175986289978, + "num_tokens": 4096873.0, + "step": 501 + }, + { + "entropy": 0.5019998624920845, + "epoch": 0.21957353745215966, + "grad_norm": 2.265625, + "learning_rate": 7.434020362682703e-09, + "loss": 0.5267874598503113, + "mean_token_accuracy": 0.8250655829906464, + "num_tokens": 4104919.0, + "step": 502 + }, + { + "entropy": 0.5021177157759666, + "epoch": 0.22001093493712412, + "grad_norm": 2.296875, + "learning_rate": 6.144347265384931e-09, + "loss": 0.5334129929542542, + "mean_token_accuracy": 0.8255132138729095, + "num_tokens": 4113111.0, + "step": 503 + }, + { + "entropy": 0.4931086599826813, + "epoch": 0.22044833242208858, + "grad_norm": 2.3125, + "learning_rate": 4.977308833379324e-09, + "loss": 0.5266363024711609, + "mean_token_accuracy": 0.828323557972908, + "num_tokens": 4121303.0, + "step": 504 + }, + { + "entropy": 0.4733823910355568, + "epoch": 0.22088572990705305, + "grad_norm": 2.3125, + "learning_rate": 3.932962455458489e-09, + "loss": 0.5121600031852722, + "mean_token_accuracy": 0.8324780017137527, + "num_tokens": 4129495.0, + "step": 505 + }, + { + "entropy": 0.45189718902111053, + "epoch": 0.22132312739201748, + "grad_norm": 2.265625, + "learning_rate": 3.011359487068988e-09, + "loss": 0.4659193158149719, + "mean_token_accuracy": 0.8465298265218735, + "num_tokens": 4137687.0, + "step": 506 + }, + { + "entropy": 0.4035600870847702, + "epoch": 0.22176052487698195, + "grad_norm": 2.0, + "learning_rate": 2.2125452477828047e-09, + "loss": 0.41255855560302734, + "mean_token_accuracy": 0.8647360801696777, + "num_tokens": 4145879.0, + "step": 507 + }, + { + "entropy": 0.5318427979946136, + "epoch": 0.2221979223619464, + "grad_norm": 2.421875, + "learning_rate": 1.5365590190699608e-09, + "loss": 0.5627883672714233, + "mean_token_accuracy": 0.8196481019258499, + "num_tokens": 4154071.0, + "step": 508 + }, + { + "entropy": 0.4271271377801895, + "epoch": 0.22263531984691087, + "grad_norm": 2.125, + "learning_rate": 9.834340423678368e-10, + "loss": 0.4386692941188812, + "mean_token_accuracy": 0.8549609035253525, + "num_tokens": 4162263.0, + "step": 509 + }, + { + "entropy": 0.5523240566253662, + "epoch": 0.22307271733187534, + "grad_norm": 2.359375, + "learning_rate": 5.531975174444259e-10, + "loss": 0.5719492435455322, + "mean_token_accuracy": 0.8136608004570007, + "num_tokens": 4170455.0, + "step": 510 + }, + { + "entropy": 0.4898983985185623, + "epoch": 0.2235101148168398, + "grad_norm": 2.359375, + "learning_rate": 2.45870601062459e-10, + "loss": 0.5295259952545166, + "mean_token_accuracy": 0.8220918923616409, + "num_tokens": 4178647.0, + "step": 511 + }, + { + "entropy": 0.4591166228055954, + "epoch": 0.22394751230180426, + "grad_norm": 2.453125, + "learning_rate": 6.14684059385695e-11, + "loss": 0.49673986434936523, + "mean_token_accuracy": 0.8360214978456497, + "num_tokens": 4186839.0, + "step": 512 + }, + { + "epoch": 0.22394751230180426, + "eval_entropy": 0.4859546652017546, + "eval_loss": 0.5158528089523315, + "eval_mean_token_accuracy": 0.8297856922977227, + "eval_num_tokens": 4186839.0, + "eval_runtime": 32.867, + "eval_samples_per_second": 29.3, + "eval_steps_per_second": 3.682, + "step": 512 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 128, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.329857209223578e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}