commit 1e95b91bec79493aea12a8f904b5d234e47b5a57 Author: ModelHub XC Date: Fri Nov 13 06:13:16 2026 +0800 初始化项目,由ModelHub XC社区提供模型 Model: amitdutta121/SmolLM3-MathInstruct-SFT Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/41cd73c8acc3c120/lighteval|gsm8k|0/f61acff0a464569c/GENERATIVE.parquet b/41cd73c8acc3c120/lighteval|gsm8k|0/f61acff0a464569c/GENERATIVE.parquet new file mode 100644 index 0000000..d8f4bbd --- /dev/null +++ b/41cd73c8acc3c120/lighteval|gsm8k|0/f61acff0a464569c/GENERATIVE.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04f67e756d46525899429b2b8ec552c9e19cab265eda306b4ad6cb54715137cb +size 6957777 diff --git a/README.md b/README.md new file mode 100644 index 0000000..42781db --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +base_model: HuggingFaceTB/SmolLM3-3B +library_name: transformers +model_name: SmolLM3-MathInstruct-SFT +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for SmolLM3-MathInstruct-SFT + +This model is a fine-tuned version of [HuggingFaceTB/SmolLM3-3B](https://huggingface.co/HuggingFaceTB/SmolLM3-3B). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device_map="auto") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 1.14.1 +- Transformers: 5.17.0 +- Pytorch: 2.11.0+cu128 +- Datasets: 5.0.1 +- Tokenizers: 0.23.1 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..e01e3a1 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,94 @@ +{# ───── defaults ───── #} +{%- if enable_thinking is not defined -%} +{%- set enable_thinking = true -%} +{%- endif -%} + +{# ───── reasoning mode ───── #} +{%- if enable_thinking -%} + {%- set reasoning_mode = "/think" -%} +{%- else -%} + {%- set reasoning_mode = "/no_think" -%} +{%- endif -%} + +{# ───── header (system message) ───── #} +{{- "<|im_start|>system\n" -}} + +{%- if messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} + {%- if "/no_think" in system_message -%} + {%- set reasoning_mode = "/no_think" -%} + {%- elif "/think" in system_message -%} + {%- set reasoning_mode = "/think" -%} + {%- endif -%} + {%- set custom_instructions = system_message.replace("/no_think", "").replace("/think", "").rstrip() -%} +{%- endif -%} + +{%- if "/system_override" in system_message -%} + {{- custom_instructions.replace("/system_override", "").rstrip() -}} + {{- "<|im_end|>\n" -}} +{%- else -%} + {{- "## Metadata\n\n" -}} + {{- "Knowledge Cutoff Date: June 2025\n" -}} + {%- set today = strftime_now("%d %B %Y") -%} + {{- "Today Date: " ~ today ~ "\n" -}} + {{- "Reasoning Mode: " + reasoning_mode + "\n\n" -}} + + {{- "## Custom Instructions\n\n" -}} + {%- if custom_instructions -%} + {{- custom_instructions + "\n\n" -}} + {%- elif reasoning_mode == "/think" -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracking, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: Thought section Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion.\n\n" -}} + {%- else -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face.\n\n" -}} + {%- endif -%} + + {%- if xml_tools or python_tools or tools -%} + {{- "### Tools\n\n" -}} + {%- if xml_tools or tools -%} + {%- if tools -%} + {%- set xml_tools = tools -%} + {%- endif -%} + {%- set ns = namespace(xml_tool_string="You may call one or more functions to assist with the user query.\nYou are provided with function signatures within XML tags:\n\n\n") -%} + {%- for tool in xml_tools[:] -%} {# The slicing makes sure that xml_tools is a list #} + {%- set ns.xml_tool_string = ns.xml_tool_string ~ (tool | string) ~ "\n" -%} + {%- endfor -%} + {%- set xml_tool_string = ns.xml_tool_string + "\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n" -%} + {{- xml_tool_string -}} + {%- endif -%} + {%- if python_tools -%} + {%- set ns = namespace(python_tool_string="When you send a message containing Python code between '' and '' tags, it will be executed in a stateful Jupyter notebook environment, and you will then be given the output to continued reasoning in an agentic loop.\n\nYou can use the following tools in your python code like regular functions:\n\n") -%} + {%- for tool in python_tools[:] -%} {# The slicing makes sure that python_tools is a list #} + {%- set ns.python_tool_string = ns.python_tool_string ~ (tool | string) ~ "\n" -%} + {%- endfor -%} + {%- set python_tool_string = ns.python_tool_string + "\n\nThe state persists between code executions: so variables that you define in one step are still available thereafter." -%} + {{- python_tool_string -}} + {%- endif -%} + {{- "\n\n" -}} + {{- "<|im_end|>\n" -}} + {%- endif -%} +{%- endif -%} +{# ───── main loop ───── #} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{ "<|im_start|>" + message.role + "\n" + content + "<|im_end|>\n" }} + {%- elif message.role == "assistant" -%} + {% generation %} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- endif -%} + {% endgeneration %} + {%- elif message.role == "tool" -%} + {{ "<|im_start|>" + "user\n" + content + "<|im_end|>\n" }} + {%- endif -%} +{%- endfor -%} +{# ───── generation prompt ───── #} +{%- if add_generation_prompt -%} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" }} + {%- endif -%} +{%- endif -%} \ No newline at end of file diff --git a/checkpoint-7250/README.md b/checkpoint-7250/README.md new file mode 100644 index 0000000..63ca3ea --- /dev/null +++ b/checkpoint-7250/README.md @@ -0,0 +1,209 @@ +--- +base_model: HuggingFaceTB/SmolLM3-3B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:HuggingFaceTB/SmolLM3-3B +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.21.1 \ No newline at end of file diff --git a/checkpoint-7250/adapter_config.json b/checkpoint-7250/adapter_config.json new file mode 100644 index 0000000..284088d --- /dev/null +++ b/checkpoint-7250/adapter_config.json @@ -0,0 +1,51 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "HuggingFaceTB/SmolLM3-3B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "kasa_config": null, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.21.1", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/checkpoint-7250/adapter_model.safetensors b/checkpoint-7250/adapter_model.safetensors new file mode 100644 index 0000000..39b2249 --- /dev/null +++ b/checkpoint-7250/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:231f32b73d8cf0af305df2dac72bae75b730d559ee97d4e06101c2435cd60c1a +size 60523464 diff --git a/checkpoint-7250/chat_template.jinja b/checkpoint-7250/chat_template.jinja new file mode 100644 index 0000000..e01e3a1 --- /dev/null +++ b/checkpoint-7250/chat_template.jinja @@ -0,0 +1,94 @@ +{# ───── defaults ───── #} +{%- if enable_thinking is not defined -%} +{%- set enable_thinking = true -%} +{%- endif -%} + +{# ───── reasoning mode ───── #} +{%- if enable_thinking -%} + {%- set reasoning_mode = "/think" -%} +{%- else -%} + {%- set reasoning_mode = "/no_think" -%} +{%- endif -%} + +{# ───── header (system message) ───── #} +{{- "<|im_start|>system\n" -}} + +{%- if messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} + {%- if "/no_think" in system_message -%} + {%- set reasoning_mode = "/no_think" -%} + {%- elif "/think" in system_message -%} + {%- set reasoning_mode = "/think" -%} + {%- endif -%} + {%- set custom_instructions = system_message.replace("/no_think", "").replace("/think", "").rstrip() -%} +{%- endif -%} + +{%- if "/system_override" in system_message -%} + {{- custom_instructions.replace("/system_override", "").rstrip() -}} + {{- "<|im_end|>\n" -}} +{%- else -%} + {{- "## Metadata\n\n" -}} + {{- "Knowledge Cutoff Date: June 2025\n" -}} + {%- set today = strftime_now("%d %B %Y") -%} + {{- "Today Date: " ~ today ~ "\n" -}} + {{- "Reasoning Mode: " + reasoning_mode + "\n\n" -}} + + {{- "## Custom Instructions\n\n" -}} + {%- if custom_instructions -%} + {{- custom_instructions + "\n\n" -}} + {%- elif reasoning_mode == "/think" -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracking, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: Thought section Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion.\n\n" -}} + {%- else -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face.\n\n" -}} + {%- endif -%} + + {%- if xml_tools or python_tools or tools -%} + {{- "### Tools\n\n" -}} + {%- if xml_tools or tools -%} + {%- if tools -%} + {%- set xml_tools = tools -%} + {%- endif -%} + {%- set ns = namespace(xml_tool_string="You may call one or more functions to assist with the user query.\nYou are provided with function signatures within XML tags:\n\n\n") -%} + {%- for tool in xml_tools[:] -%} {# The slicing makes sure that xml_tools is a list #} + {%- set ns.xml_tool_string = ns.xml_tool_string ~ (tool | string) ~ "\n" -%} + {%- endfor -%} + {%- set xml_tool_string = ns.xml_tool_string + "\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n" -%} + {{- xml_tool_string -}} + {%- endif -%} + {%- if python_tools -%} + {%- set ns = namespace(python_tool_string="When you send a message containing Python code between '' and '' tags, it will be executed in a stateful Jupyter notebook environment, and you will then be given the output to continued reasoning in an agentic loop.\n\nYou can use the following tools in your python code like regular functions:\n\n") -%} + {%- for tool in python_tools[:] -%} {# The slicing makes sure that python_tools is a list #} + {%- set ns.python_tool_string = ns.python_tool_string ~ (tool | string) ~ "\n" -%} + {%- endfor -%} + {%- set python_tool_string = ns.python_tool_string + "\n\nThe state persists between code executions: so variables that you define in one step are still available thereafter." -%} + {{- python_tool_string -}} + {%- endif -%} + {{- "\n\n" -}} + {{- "<|im_end|>\n" -}} + {%- endif -%} +{%- endif -%} +{# ───── main loop ───── #} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{ "<|im_start|>" + message.role + "\n" + content + "<|im_end|>\n" }} + {%- elif message.role == "assistant" -%} + {% generation %} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- endif -%} + {% endgeneration %} + {%- elif message.role == "tool" -%} + {{ "<|im_start|>" + "user\n" + content + "<|im_end|>\n" }} + {%- endif -%} +{%- endfor -%} +{# ───── generation prompt ───── #} +{%- if add_generation_prompt -%} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" }} + {%- endif -%} +{%- endif -%} \ No newline at end of file diff --git a/checkpoint-7250/optimizer.pt b/checkpoint-7250/optimizer.pt new file mode 100644 index 0000000..79ebcbb --- /dev/null +++ b/checkpoint-7250/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f4f1144d0949a1d42aaa591becda1ceb2e0897cc6ccc3562e995fd345e68258 +size 121338699 diff --git a/checkpoint-7250/rng_state.pth b/checkpoint-7250/rng_state.pth new file mode 100644 index 0000000..fa131e8 --- /dev/null +++ b/checkpoint-7250/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:853d373615aadd8fe71a5f502522bf57ea7f589b5b6d035f282547c750a2dc83 +size 14645 diff --git a/checkpoint-7250/scheduler.pt b/checkpoint-7250/scheduler.pt new file mode 100644 index 0000000..147f84f --- /dev/null +++ b/checkpoint-7250/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e999f57721a4e239460a9350e172faf52ebbf7a53c5d283a58d6dc15419cdc7 +size 1465 diff --git a/checkpoint-7250/tokenizer.json b/checkpoint-7250/tokenizer.json new file mode 100644 index 0000000..074fa66 --- /dev/null +++ b/checkpoint-7250/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b6a500b662a34eb3f0374db856ba4ad7de4c81040571d78dc0d357238930005 +size 17208819 diff --git a/checkpoint-7250/tokenizer_config.json b/checkpoint-7250/tokenizer_config.json new file mode 100644 index 0000000..8804a73 --- /dev/null +++ b/checkpoint-7250/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": true, + "eos_token": "<|im_end|>", + "fast": false, + "is_local": false, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|im_end|>", + "tokenizer_class": "TokenizersBackend" +} diff --git a/checkpoint-7250/trainer_state.json b/checkpoint-7250/trainer_state.json new file mode 100644 index 0000000..58f90bd --- /dev/null +++ b/checkpoint-7250/trainer_state.json @@ -0,0 +1,7603 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9863945578231292, + "eval_steps": 250, + "global_step": 7250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.3966208942234516, + "epoch": 0.0013605442176870747, + "grad_norm": 1.5996246337890625, + "learning_rate": 9e-06, + "loss": 0.8333851814270019, + "mean_token_accuracy": 0.8561436295509338, + "num_tokens": 19866.0, + "step": 10 + }, + { + "entropy": 0.34238167852163315, + "epoch": 0.0027210884353741495, + "grad_norm": 1.296663522720337, + "learning_rate": 1.9e-05, + "loss": 0.7776159763336181, + "mean_token_accuracy": 0.8674830228090287, + "num_tokens": 39758.0, + "step": 20 + }, + { + "entropy": 0.42244544103741644, + "epoch": 0.004081632653061225, + "grad_norm": 0.7037413716316223, + "learning_rate": 2.9e-05, + "loss": 0.7272251605987549, + "mean_token_accuracy": 0.8528120934963226, + "num_tokens": 60338.0, + "step": 30 + }, + { + "entropy": 0.40994860902428626, + "epoch": 0.005442176870748299, + "grad_norm": 0.47693488001823425, + "learning_rate": 3.9000000000000006e-05, + "loss": 0.5927484512329102, + "mean_token_accuracy": 0.865676474571228, + "num_tokens": 79139.0, + "step": 40 + }, + { + "entropy": 0.4373441830277443, + "epoch": 0.006802721088435374, + "grad_norm": 0.4747143089771271, + "learning_rate": 4.9e-05, + "loss": 0.515117883682251, + "mean_token_accuracy": 0.8772250235080719, + "num_tokens": 98338.0, + "step": 50 + }, + { + "entropy": 0.5012069299817086, + "epoch": 0.00816326530612245, + "grad_norm": 0.5569216012954712, + "learning_rate": 4.9938356164383565e-05, + "loss": 0.5295265197753907, + "mean_token_accuracy": 0.8785856813192368, + "num_tokens": 115658.0, + "step": 60 + }, + { + "entropy": 0.4689371645450592, + "epoch": 0.009523809523809525, + "grad_norm": 0.2908697724342346, + "learning_rate": 4.986986301369863e-05, + "loss": 0.42938008308410647, + "mean_token_accuracy": 0.8960865825414658, + "num_tokens": 135179.0, + "step": 70 + }, + { + "entropy": 0.4583378002047539, + "epoch": 0.010884353741496598, + "grad_norm": 0.402289479970932, + "learning_rate": 4.9801369863013704e-05, + "loss": 0.4500479221343994, + "mean_token_accuracy": 0.8930670320987701, + "num_tokens": 154492.0, + "step": 80 + }, + { + "entropy": 0.45134490057826043, + "epoch": 0.012244897959183673, + "grad_norm": 0.4460085928440094, + "learning_rate": 4.973287671232877e-05, + "loss": 0.46530923843383787, + "mean_token_accuracy": 0.8883364647626877, + "num_tokens": 172732.0, + "step": 90 + }, + { + "entropy": 0.421974216401577, + "epoch": 0.013605442176870748, + "grad_norm": 0.4011005759239197, + "learning_rate": 4.966438356164384e-05, + "loss": 0.4124259948730469, + "mean_token_accuracy": 0.8977983146905899, + "num_tokens": 192922.0, + "step": 100 + }, + { + "entropy": 0.38947183191776275, + "epoch": 0.014965986394557823, + "grad_norm": 0.38355737924575806, + "learning_rate": 4.959589041095891e-05, + "loss": 0.3860264778137207, + "mean_token_accuracy": 0.9041510850191117, + "num_tokens": 210310.0, + "step": 110 + }, + { + "entropy": 0.474651126563549, + "epoch": 0.0163265306122449, + "grad_norm": 0.45214158296585083, + "learning_rate": 4.9527397260273975e-05, + "loss": 0.48367671966552733, + "mean_token_accuracy": 0.8782862812280655, + "num_tokens": 229495.0, + "step": 120 + }, + { + "entropy": 0.34797751232981683, + "epoch": 0.017687074829931974, + "grad_norm": 0.40691450238227844, + "learning_rate": 4.945890410958904e-05, + "loss": 0.32767112255096437, + "mean_token_accuracy": 0.9148984670639038, + "num_tokens": 247661.0, + "step": 130 + }, + { + "entropy": 0.40742513462901114, + "epoch": 0.01904761904761905, + "grad_norm": 0.45683032274246216, + "learning_rate": 4.939041095890411e-05, + "loss": 0.40177297592163086, + "mean_token_accuracy": 0.8949861347675323, + "num_tokens": 268110.0, + "step": 140 + }, + { + "entropy": 0.4079647034406662, + "epoch": 0.02040816326530612, + "grad_norm": 0.32131096720695496, + "learning_rate": 4.932191780821918e-05, + "loss": 0.4140042781829834, + "mean_token_accuracy": 0.8966580182313919, + "num_tokens": 286814.0, + "step": 150 + }, + { + "entropy": 0.38506424576044085, + "epoch": 0.021768707482993196, + "grad_norm": 0.4252924919128418, + "learning_rate": 4.9253424657534246e-05, + "loss": 0.3659359931945801, + "mean_token_accuracy": 0.9046422451734543, + "num_tokens": 306080.0, + "step": 160 + }, + { + "entropy": 0.42693412154912946, + "epoch": 0.02312925170068027, + "grad_norm": 0.49110063910484314, + "learning_rate": 4.918493150684932e-05, + "loss": 0.4523786067962646, + "mean_token_accuracy": 0.8923947751522064, + "num_tokens": 324055.0, + "step": 170 + }, + { + "entropy": 0.43308429792523384, + "epoch": 0.024489795918367346, + "grad_norm": 0.39848849177360535, + "learning_rate": 4.9116438356164385e-05, + "loss": 0.44591054916381834, + "mean_token_accuracy": 0.8914232641458512, + "num_tokens": 342105.0, + "step": 180 + }, + { + "entropy": 0.3832189217209816, + "epoch": 0.02585034013605442, + "grad_norm": 0.4442504346370697, + "learning_rate": 4.904794520547946e-05, + "loss": 0.38175168037414553, + "mean_token_accuracy": 0.9029964536428452, + "num_tokens": 360113.0, + "step": 190 + }, + { + "entropy": 0.35205388739705085, + "epoch": 0.027210884353741496, + "grad_norm": 0.3656632602214813, + "learning_rate": 4.8979452054794524e-05, + "loss": 0.3521461009979248, + "mean_token_accuracy": 0.9092369765043259, + "num_tokens": 377600.0, + "step": 200 + }, + { + "entropy": 0.37196114510297773, + "epoch": 0.02857142857142857, + "grad_norm": 0.44447311758995056, + "learning_rate": 4.8910958904109597e-05, + "loss": 0.3701155185699463, + "mean_token_accuracy": 0.9007940888404846, + "num_tokens": 398788.0, + "step": 210 + }, + { + "entropy": 0.38314035460352897, + "epoch": 0.029931972789115645, + "grad_norm": 0.3941226303577423, + "learning_rate": 4.884246575342466e-05, + "loss": 0.3909285068511963, + "mean_token_accuracy": 0.8981142967939377, + "num_tokens": 419141.0, + "step": 220 + }, + { + "entropy": 0.3607197545468807, + "epoch": 0.031292517006802724, + "grad_norm": 0.3219183683395386, + "learning_rate": 4.877397260273973e-05, + "loss": 0.37545225620269773, + "mean_token_accuracy": 0.9047404289245605, + "num_tokens": 437786.0, + "step": 230 + }, + { + "entropy": 0.3778640143573284, + "epoch": 0.0326530612244898, + "grad_norm": 0.49071386456489563, + "learning_rate": 4.8705479452054795e-05, + "loss": 0.3791273832321167, + "mean_token_accuracy": 0.9014792263507843, + "num_tokens": 456747.0, + "step": 240 + }, + { + "entropy": 0.4029653251171112, + "epoch": 0.034013605442176874, + "grad_norm": 0.5018599629402161, + "learning_rate": 4.863698630136986e-05, + "loss": 0.4035783290863037, + "mean_token_accuracy": 0.8966485053300858, + "num_tokens": 475585.0, + "step": 250 + }, + { + "epoch": 0.034013605442176874, + "eval_entropy": 0.3818362482388814, + "eval_loss": 0.37777262926101685, + "eval_mean_token_accuracy": 0.9024270363648732, + "eval_num_tokens": 475585.0, + "eval_runtime": 39.4306, + "eval_samples_per_second": 15.217, + "eval_steps_per_second": 7.608, + "step": 250 + }, + { + "entropy": 0.348661782592535, + "epoch": 0.03537414965986395, + "grad_norm": 0.45828038454055786, + "learning_rate": 4.8568493150684934e-05, + "loss": 0.3377650737762451, + "mean_token_accuracy": 0.9096396565437317, + "num_tokens": 494035.0, + "step": 260 + }, + { + "entropy": 0.3596964508295059, + "epoch": 0.036734693877551024, + "grad_norm": 0.4269905984401703, + "learning_rate": 4.85e-05, + "loss": 0.3601596593856812, + "mean_token_accuracy": 0.9069953888654709, + "num_tokens": 512948.0, + "step": 270 + }, + { + "entropy": 0.3371978670358658, + "epoch": 0.0380952380952381, + "grad_norm": 0.3644143342971802, + "learning_rate": 4.843150684931507e-05, + "loss": 0.33878345489501954, + "mean_token_accuracy": 0.9105350911617279, + "num_tokens": 532063.0, + "step": 280 + }, + { + "entropy": 0.35116567388176917, + "epoch": 0.03945578231292517, + "grad_norm": 0.3923342823982239, + "learning_rate": 4.836301369863014e-05, + "loss": 0.3478651523590088, + "mean_token_accuracy": 0.9095128774642944, + "num_tokens": 549043.0, + "step": 290 + }, + { + "entropy": 0.34563929960131645, + "epoch": 0.04081632653061224, + "grad_norm": 0.3414367735385895, + "learning_rate": 4.829452054794521e-05, + "loss": 0.33946025371551514, + "mean_token_accuracy": 0.9125081688165665, + "num_tokens": 567832.0, + "step": 300 + }, + { + "entropy": 0.37908021062612535, + "epoch": 0.04217687074829932, + "grad_norm": 0.4030672311782837, + "learning_rate": 4.822602739726028e-05, + "loss": 0.38597617149353025, + "mean_token_accuracy": 0.900176802277565, + "num_tokens": 588457.0, + "step": 310 + }, + { + "entropy": 0.38542170226573946, + "epoch": 0.04353741496598639, + "grad_norm": 0.40530654788017273, + "learning_rate": 4.8157534246575344e-05, + "loss": 0.39215831756591796, + "mean_token_accuracy": 0.8950019955635071, + "num_tokens": 608720.0, + "step": 320 + }, + { + "entropy": 0.40009846463799476, + "epoch": 0.044897959183673466, + "grad_norm": 0.5024587512016296, + "learning_rate": 4.808904109589042e-05, + "loss": 0.41683034896850585, + "mean_token_accuracy": 0.893940594792366, + "num_tokens": 628467.0, + "step": 330 + }, + { + "entropy": 0.3844646617770195, + "epoch": 0.04625850340136054, + "grad_norm": 0.37910065054893494, + "learning_rate": 4.802054794520548e-05, + "loss": 0.3854831695556641, + "mean_token_accuracy": 0.901634418964386, + "num_tokens": 648387.0, + "step": 340 + }, + { + "entropy": 0.3825081571936607, + "epoch": 0.047619047619047616, + "grad_norm": 0.455219030380249, + "learning_rate": 4.795205479452055e-05, + "loss": 0.38410975933074953, + "mean_token_accuracy": 0.9046315401792526, + "num_tokens": 665488.0, + "step": 350 + }, + { + "entropy": 0.3589709296822548, + "epoch": 0.04897959183673469, + "grad_norm": 0.3238779902458191, + "learning_rate": 4.7883561643835615e-05, + "loss": 0.35581753253936765, + "mean_token_accuracy": 0.9074301540851593, + "num_tokens": 684284.0, + "step": 360 + }, + { + "entropy": 0.3434913203120232, + "epoch": 0.050340136054421766, + "grad_norm": 0.349220871925354, + "learning_rate": 4.781506849315069e-05, + "loss": 0.33537404537200927, + "mean_token_accuracy": 0.9093591779470444, + "num_tokens": 702876.0, + "step": 370 + }, + { + "entropy": 0.3294819757342339, + "epoch": 0.05170068027210884, + "grad_norm": 0.35057687759399414, + "learning_rate": 4.7746575342465754e-05, + "loss": 0.33455510139465333, + "mean_token_accuracy": 0.9156662911176682, + "num_tokens": 720840.0, + "step": 380 + }, + { + "entropy": 0.3731862798333168, + "epoch": 0.053061224489795916, + "grad_norm": 0.4629971981048584, + "learning_rate": 4.767808219178083e-05, + "loss": 0.3719052791595459, + "mean_token_accuracy": 0.903752988576889, + "num_tokens": 739745.0, + "step": 390 + }, + { + "entropy": 0.3066830538213253, + "epoch": 0.05442176870748299, + "grad_norm": 0.4535863697528839, + "learning_rate": 4.760958904109589e-05, + "loss": 0.30530486106872556, + "mean_token_accuracy": 0.9210470616817474, + "num_tokens": 757386.0, + "step": 400 + }, + { + "entropy": 0.33964752927422526, + "epoch": 0.055782312925170066, + "grad_norm": 0.326674222946167, + "learning_rate": 4.7541095890410966e-05, + "loss": 0.3501870632171631, + "mean_token_accuracy": 0.9105898469686509, + "num_tokens": 774681.0, + "step": 410 + }, + { + "entropy": 0.34910165071487426, + "epoch": 0.05714285714285714, + "grad_norm": 0.4973655939102173, + "learning_rate": 4.747260273972603e-05, + "loss": 0.35225639343261717, + "mean_token_accuracy": 0.9095459759235383, + "num_tokens": 794694.0, + "step": 420 + }, + { + "entropy": 0.3515663854777813, + "epoch": 0.058503401360544216, + "grad_norm": 0.3093283772468567, + "learning_rate": 4.74041095890411e-05, + "loss": 0.3447835922241211, + "mean_token_accuracy": 0.9086586356163024, + "num_tokens": 813789.0, + "step": 430 + }, + { + "entropy": 0.3582890912890434, + "epoch": 0.05986394557823129, + "grad_norm": 0.29822006821632385, + "learning_rate": 4.733561643835617e-05, + "loss": 0.3475156784057617, + "mean_token_accuracy": 0.9078772723674774, + "num_tokens": 833162.0, + "step": 440 + }, + { + "entropy": 0.3461380168795586, + "epoch": 0.061224489795918366, + "grad_norm": 0.350153386592865, + "learning_rate": 4.726712328767123e-05, + "loss": 0.3462402820587158, + "mean_token_accuracy": 0.9049428641796112, + "num_tokens": 853866.0, + "step": 450 + }, + { + "entropy": 0.30043474584817886, + "epoch": 0.06258503401360545, + "grad_norm": 0.3542056679725647, + "learning_rate": 4.71986301369863e-05, + "loss": 0.3105747222900391, + "mean_token_accuracy": 0.9163908660411835, + "num_tokens": 873028.0, + "step": 460 + }, + { + "entropy": 0.3481592245399952, + "epoch": 0.06394557823129252, + "grad_norm": 0.3953518271446228, + "learning_rate": 4.713013698630137e-05, + "loss": 0.3386308908462524, + "mean_token_accuracy": 0.9098592340946198, + "num_tokens": 891721.0, + "step": 470 + }, + { + "entropy": 0.3410739690065384, + "epoch": 0.0653061224489796, + "grad_norm": 0.4479163885116577, + "learning_rate": 4.706164383561644e-05, + "loss": 0.3482985496520996, + "mean_token_accuracy": 0.9046086817979813, + "num_tokens": 913549.0, + "step": 480 + }, + { + "entropy": 0.36930672004818915, + "epoch": 0.06666666666666667, + "grad_norm": 0.3130194842815399, + "learning_rate": 4.699315068493151e-05, + "loss": 0.36626100540161133, + "mean_token_accuracy": 0.9040644109249115, + "num_tokens": 932950.0, + "step": 490 + }, + { + "entropy": 0.397816364467144, + "epoch": 0.06802721088435375, + "grad_norm": 0.45680442452430725, + "learning_rate": 4.692465753424658e-05, + "loss": 0.40360026359558104, + "mean_token_accuracy": 0.9006230294704437, + "num_tokens": 951066.0, + "step": 500 + }, + { + "epoch": 0.06802721088435375, + "eval_entropy": 0.3723491595685482, + "eval_loss": 0.3646194040775299, + "eval_mean_token_accuracy": 0.9050061551729838, + "eval_num_tokens": 951066.0, + "eval_runtime": 39.4141, + "eval_samples_per_second": 15.223, + "eval_steps_per_second": 7.611, + "step": 500 + }, + { + "entropy": 0.4003289297223091, + "epoch": 0.06938775510204082, + "grad_norm": 0.37886762619018555, + "learning_rate": 4.685616438356165e-05, + "loss": 0.39484944343566897, + "mean_token_accuracy": 0.8952762842178345, + "num_tokens": 971117.0, + "step": 510 + }, + { + "entropy": 0.3746891237795353, + "epoch": 0.0707482993197279, + "grad_norm": 0.39949730038642883, + "learning_rate": 4.678767123287671e-05, + "loss": 0.3683069467544556, + "mean_token_accuracy": 0.903772982954979, + "num_tokens": 990103.0, + "step": 520 + }, + { + "entropy": 0.31653870418667795, + "epoch": 0.07210884353741497, + "grad_norm": 0.33652064204216003, + "learning_rate": 4.6719178082191786e-05, + "loss": 0.3186372995376587, + "mean_token_accuracy": 0.9144774466753006, + "num_tokens": 1010804.0, + "step": 530 + }, + { + "entropy": 0.3711325004696846, + "epoch": 0.07346938775510205, + "grad_norm": 0.3641034662723541, + "learning_rate": 4.665068493150685e-05, + "loss": 0.3789762735366821, + "mean_token_accuracy": 0.9045424968004226, + "num_tokens": 1029560.0, + "step": 540 + }, + { + "entropy": 0.37221564501523974, + "epoch": 0.07482993197278912, + "grad_norm": 0.4707719087600708, + "learning_rate": 4.658219178082192e-05, + "loss": 0.3638460636138916, + "mean_token_accuracy": 0.9029160022735596, + "num_tokens": 1049933.0, + "step": 550 + }, + { + "entropy": 0.3466314420104027, + "epoch": 0.0761904761904762, + "grad_norm": 0.5033849477767944, + "learning_rate": 4.6513698630136984e-05, + "loss": 0.352337908744812, + "mean_token_accuracy": 0.9113892763853073, + "num_tokens": 1067783.0, + "step": 560 + }, + { + "entropy": 0.277361162006855, + "epoch": 0.07755102040816327, + "grad_norm": 0.3210407793521881, + "learning_rate": 4.644520547945206e-05, + "loss": 0.28125414848327634, + "mean_token_accuracy": 0.922634044289589, + "num_tokens": 1088771.0, + "step": 570 + }, + { + "entropy": 0.3498281344771385, + "epoch": 0.07891156462585033, + "grad_norm": 0.43742120265960693, + "learning_rate": 4.637671232876712e-05, + "loss": 0.3471531391143799, + "mean_token_accuracy": 0.9098026067018509, + "num_tokens": 1108964.0, + "step": 580 + }, + { + "entropy": 0.34425888285040857, + "epoch": 0.08027210884353742, + "grad_norm": 0.32993149757385254, + "learning_rate": 4.6308219178082196e-05, + "loss": 0.35619263648986815, + "mean_token_accuracy": 0.9035626143217087, + "num_tokens": 1130205.0, + "step": 590 + }, + { + "entropy": 0.3691210441291332, + "epoch": 0.08163265306122448, + "grad_norm": 0.4786720871925354, + "learning_rate": 4.623972602739726e-05, + "loss": 0.3622753381729126, + "mean_token_accuracy": 0.9080120652914048, + "num_tokens": 1150314.0, + "step": 600 + }, + { + "entropy": 0.389055623114109, + "epoch": 0.08299319727891157, + "grad_norm": 0.3758779764175415, + "learning_rate": 4.6171232876712335e-05, + "loss": 0.3914887189865112, + "mean_token_accuracy": 0.901269006729126, + "num_tokens": 1168468.0, + "step": 610 + }, + { + "entropy": 0.3246720366179943, + "epoch": 0.08435374149659863, + "grad_norm": 0.4306440055370331, + "learning_rate": 4.61027397260274e-05, + "loss": 0.32867746353149413, + "mean_token_accuracy": 0.9110509842634201, + "num_tokens": 1188023.0, + "step": 620 + }, + { + "entropy": 0.32952589765191076, + "epoch": 0.08571428571428572, + "grad_norm": 0.3729093670845032, + "learning_rate": 4.603424657534247e-05, + "loss": 0.3338437795639038, + "mean_token_accuracy": 0.9136817961931228, + "num_tokens": 1205997.0, + "step": 630 + }, + { + "entropy": 0.34219127148389816, + "epoch": 0.08707482993197278, + "grad_norm": 0.33127057552337646, + "learning_rate": 4.596575342465754e-05, + "loss": 0.3417861223220825, + "mean_token_accuracy": 0.9103910386562347, + "num_tokens": 1225286.0, + "step": 640 + }, + { + "entropy": 0.396539293974638, + "epoch": 0.08843537414965986, + "grad_norm": 0.40578168630599976, + "learning_rate": 4.5897260273972606e-05, + "loss": 0.4101886749267578, + "mean_token_accuracy": 0.8973281621932984, + "num_tokens": 1244108.0, + "step": 650 + }, + { + "entropy": 0.38693875148892404, + "epoch": 0.08979591836734693, + "grad_norm": 0.3798186779022217, + "learning_rate": 4.582876712328767e-05, + "loss": 0.3816530704498291, + "mean_token_accuracy": 0.9010074496269226, + "num_tokens": 1264868.0, + "step": 660 + }, + { + "entropy": 0.3229499511420727, + "epoch": 0.09115646258503401, + "grad_norm": 0.3040688931941986, + "learning_rate": 4.576027397260274e-05, + "loss": 0.3126213073730469, + "mean_token_accuracy": 0.9153517127037049, + "num_tokens": 1285486.0, + "step": 670 + }, + { + "entropy": 0.356215351074934, + "epoch": 0.09251700680272108, + "grad_norm": 0.4674758017063141, + "learning_rate": 4.569178082191781e-05, + "loss": 0.3552473306655884, + "mean_token_accuracy": 0.9089611619710922, + "num_tokens": 1303745.0, + "step": 680 + }, + { + "entropy": 0.40253775492310523, + "epoch": 0.09387755102040816, + "grad_norm": 0.42166969180107117, + "learning_rate": 4.562328767123288e-05, + "loss": 0.41041126251220705, + "mean_token_accuracy": 0.8952078282833099, + "num_tokens": 1324171.0, + "step": 690 + }, + { + "entropy": 0.34822252616286276, + "epoch": 0.09523809523809523, + "grad_norm": 0.3444162905216217, + "learning_rate": 4.555479452054795e-05, + "loss": 0.35946462154388426, + "mean_token_accuracy": 0.9083791345357894, + "num_tokens": 1343401.0, + "step": 700 + }, + { + "entropy": 0.32225404009222985, + "epoch": 0.09659863945578231, + "grad_norm": 0.4599993824958801, + "learning_rate": 4.5486301369863016e-05, + "loss": 0.32996740341186526, + "mean_token_accuracy": 0.9141956597566605, + "num_tokens": 1363084.0, + "step": 710 + }, + { + "entropy": 0.3677783004939556, + "epoch": 0.09795918367346938, + "grad_norm": 0.435357928276062, + "learning_rate": 4.541780821917808e-05, + "loss": 0.37146801948547364, + "mean_token_accuracy": 0.9019740343093872, + "num_tokens": 1381194.0, + "step": 720 + }, + { + "entropy": 0.33645370081067083, + "epoch": 0.09931972789115646, + "grad_norm": 0.42202693223953247, + "learning_rate": 4.5349315068493155e-05, + "loss": 0.33685150146484377, + "mean_token_accuracy": 0.9091235578060151, + "num_tokens": 1400075.0, + "step": 730 + }, + { + "entropy": 0.3678745202720165, + "epoch": 0.10068027210884353, + "grad_norm": 0.45290741324424744, + "learning_rate": 4.528082191780822e-05, + "loss": 0.35833916664123533, + "mean_token_accuracy": 0.9041007101535797, + "num_tokens": 1420258.0, + "step": 740 + }, + { + "entropy": 0.36107349693775176, + "epoch": 0.10204081632653061, + "grad_norm": 0.4961569309234619, + "learning_rate": 4.5212328767123294e-05, + "loss": 0.3725413799285889, + "mean_token_accuracy": 0.90306396484375, + "num_tokens": 1439035.0, + "step": 750 + }, + { + "epoch": 0.10204081632653061, + "eval_entropy": 0.34408375874161723, + "eval_loss": 0.3582100570201874, + "eval_mean_token_accuracy": 0.9064499346415201, + "eval_num_tokens": 1439035.0, + "eval_runtime": 39.3818, + "eval_samples_per_second": 15.235, + "eval_steps_per_second": 7.618, + "step": 750 + }, + { + "entropy": 0.3133292868733406, + "epoch": 0.10340136054421768, + "grad_norm": 0.6212469935417175, + "learning_rate": 4.514383561643836e-05, + "loss": 0.32587475776672364, + "mean_token_accuracy": 0.9168930113315582, + "num_tokens": 1455982.0, + "step": 760 + }, + { + "entropy": 0.3137072093784809, + "epoch": 0.10476190476190476, + "grad_norm": 0.3927687406539917, + "learning_rate": 4.5075342465753426e-05, + "loss": 0.3104274034500122, + "mean_token_accuracy": 0.919685822725296, + "num_tokens": 1474473.0, + "step": 770 + }, + { + "entropy": 0.39127727299928666, + "epoch": 0.10612244897959183, + "grad_norm": 0.47527554631233215, + "learning_rate": 4.500684931506849e-05, + "loss": 0.3964341640472412, + "mean_token_accuracy": 0.9029917776584625, + "num_tokens": 1492907.0, + "step": 780 + }, + { + "entropy": 0.36204536259174347, + "epoch": 0.10748299319727891, + "grad_norm": 0.38715481758117676, + "learning_rate": 4.4938356164383565e-05, + "loss": 0.3774503469467163, + "mean_token_accuracy": 0.9045975625514984, + "num_tokens": 1511281.0, + "step": 790 + }, + { + "entropy": 0.3681113593280315, + "epoch": 0.10884353741496598, + "grad_norm": 0.3628789782524109, + "learning_rate": 4.486986301369863e-05, + "loss": 0.3720179557800293, + "mean_token_accuracy": 0.903636223077774, + "num_tokens": 1531835.0, + "step": 800 + }, + { + "entropy": 0.3314031042158604, + "epoch": 0.11020408163265306, + "grad_norm": 0.34054824709892273, + "learning_rate": 4.4801369863013704e-05, + "loss": 0.3269455909729004, + "mean_token_accuracy": 0.9134721159934998, + "num_tokens": 1550387.0, + "step": 810 + }, + { + "entropy": 0.2866345688700676, + "epoch": 0.11156462585034013, + "grad_norm": 0.7705225944519043, + "learning_rate": 4.473287671232877e-05, + "loss": 0.2993777275085449, + "mean_token_accuracy": 0.9186601221561432, + "num_tokens": 1571096.0, + "step": 820 + }, + { + "entropy": 0.38486768677830696, + "epoch": 0.11292517006802721, + "grad_norm": 0.45260295271873474, + "learning_rate": 4.4664383561643836e-05, + "loss": 0.3849661350250244, + "mean_token_accuracy": 0.9009180992841721, + "num_tokens": 1590859.0, + "step": 830 + }, + { + "entropy": 0.38537542670965197, + "epoch": 0.11428571428571428, + "grad_norm": 0.3428184688091278, + "learning_rate": 4.459589041095891e-05, + "loss": 0.37544195652008056, + "mean_token_accuracy": 0.8985641360282898, + "num_tokens": 1612476.0, + "step": 840 + }, + { + "entropy": 0.3693395532667637, + "epoch": 0.11564625850340136, + "grad_norm": 0.48987624049186707, + "learning_rate": 4.4527397260273975e-05, + "loss": 0.37159268856048583, + "mean_token_accuracy": 0.9065731137990951, + "num_tokens": 1632486.0, + "step": 850 + }, + { + "entropy": 0.3446200601756573, + "epoch": 0.11700680272108843, + "grad_norm": 0.38551682233810425, + "learning_rate": 4.445890410958905e-05, + "loss": 0.340871262550354, + "mean_token_accuracy": 0.9108866393566132, + "num_tokens": 1651787.0, + "step": 860 + }, + { + "entropy": 0.3901903234422207, + "epoch": 0.11836734693877551, + "grad_norm": 0.44852742552757263, + "learning_rate": 4.439041095890411e-05, + "loss": 0.3871347665786743, + "mean_token_accuracy": 0.8977316886186599, + "num_tokens": 1672747.0, + "step": 870 + }, + { + "entropy": 0.35357545092701914, + "epoch": 0.11972789115646258, + "grad_norm": 0.6645485162734985, + "learning_rate": 4.432191780821918e-05, + "loss": 0.36792519092559817, + "mean_token_accuracy": 0.9066919326782227, + "num_tokens": 1692001.0, + "step": 880 + }, + { + "entropy": 0.4160300984978676, + "epoch": 0.12108843537414966, + "grad_norm": 0.4086649417877197, + "learning_rate": 4.4253424657534246e-05, + "loss": 0.4246100425720215, + "mean_token_accuracy": 0.8960783511400223, + "num_tokens": 1710300.0, + "step": 890 + }, + { + "entropy": 0.349920666962862, + "epoch": 0.12244897959183673, + "grad_norm": 0.5265305638313293, + "learning_rate": 4.418493150684932e-05, + "loss": 0.3457962989807129, + "mean_token_accuracy": 0.9070086061954499, + "num_tokens": 1729591.0, + "step": 900 + }, + { + "entropy": 0.32573084980249406, + "epoch": 0.12380952380952381, + "grad_norm": 0.45363783836364746, + "learning_rate": 4.4116438356164385e-05, + "loss": 0.32244632244110105, + "mean_token_accuracy": 0.9139437705278397, + "num_tokens": 1747204.0, + "step": 910 + }, + { + "entropy": 0.31426838859915734, + "epoch": 0.1251700680272109, + "grad_norm": 0.43578457832336426, + "learning_rate": 4.404794520547945e-05, + "loss": 0.31415719985961915, + "mean_token_accuracy": 0.919468006491661, + "num_tokens": 1765776.0, + "step": 920 + }, + { + "entropy": 0.3555323138833046, + "epoch": 0.12653061224489795, + "grad_norm": 0.41630497574806213, + "learning_rate": 4.3979452054794524e-05, + "loss": 0.36635076999664307, + "mean_token_accuracy": 0.9067310243844986, + "num_tokens": 1783746.0, + "step": 930 + }, + { + "entropy": 0.30401672422885895, + "epoch": 0.12789115646258503, + "grad_norm": 0.2979906499385834, + "learning_rate": 4.391095890410959e-05, + "loss": 0.2973449468612671, + "mean_token_accuracy": 0.91805320084095, + "num_tokens": 1805916.0, + "step": 940 + }, + { + "entropy": 0.32692537233233454, + "epoch": 0.1292517006802721, + "grad_norm": 0.415134459733963, + "learning_rate": 4.384246575342466e-05, + "loss": 0.3329088926315308, + "mean_token_accuracy": 0.9109864830970764, + "num_tokens": 1824061.0, + "step": 950 + }, + { + "entropy": 0.33077087327837945, + "epoch": 0.1306122448979592, + "grad_norm": 0.309216171503067, + "learning_rate": 4.377397260273973e-05, + "loss": 0.3365213871002197, + "mean_token_accuracy": 0.9099713116884232, + "num_tokens": 1845404.0, + "step": 960 + }, + { + "entropy": 0.3487712278962135, + "epoch": 0.13197278911564625, + "grad_norm": 0.46856454014778137, + "learning_rate": 4.3705479452054795e-05, + "loss": 0.3540360689163208, + "mean_token_accuracy": 0.9070927411317825, + "num_tokens": 1866651.0, + "step": 970 + }, + { + "entropy": 0.33622991368174554, + "epoch": 0.13333333333333333, + "grad_norm": 0.26888006925582886, + "learning_rate": 4.363698630136986e-05, + "loss": 0.345446252822876, + "mean_token_accuracy": 0.9077248781919479, + "num_tokens": 1886454.0, + "step": 980 + }, + { + "entropy": 0.3214356690645218, + "epoch": 0.1346938775510204, + "grad_norm": 0.3736676275730133, + "learning_rate": 4.3568493150684934e-05, + "loss": 0.30926587581634524, + "mean_token_accuracy": 0.9138902932405472, + "num_tokens": 1905432.0, + "step": 990 + }, + { + "entropy": 0.3608489066362381, + "epoch": 0.1360544217687075, + "grad_norm": 0.5109230279922485, + "learning_rate": 4.35e-05, + "loss": 0.3655172109603882, + "mean_token_accuracy": 0.906938511133194, + "num_tokens": 1925558.0, + "step": 1000 + }, + { + "epoch": 0.1360544217687075, + "eval_entropy": 0.34845481877525647, + "eval_loss": 0.3528926968574524, + "eval_mean_token_accuracy": 0.9076920866966247, + "eval_num_tokens": 1925558.0, + "eval_runtime": 39.6694, + "eval_samples_per_second": 15.125, + "eval_steps_per_second": 7.563, + "step": 1000 + }, + { + "entropy": 0.41318991109728814, + "epoch": 0.13741496598639455, + "grad_norm": 0.38739997148513794, + "learning_rate": 4.3431506849315066e-05, + "loss": 0.42798943519592286, + "mean_token_accuracy": 0.8936826854944229, + "num_tokens": 1944122.0, + "step": 1010 + }, + { + "entropy": 0.3004152253270149, + "epoch": 0.13877551020408163, + "grad_norm": 0.3833780586719513, + "learning_rate": 4.336301369863014e-05, + "loss": 0.3056038856506348, + "mean_token_accuracy": 0.9163402706384659, + "num_tokens": 1966172.0, + "step": 1020 + }, + { + "entropy": 0.33445904925465586, + "epoch": 0.1401360544217687, + "grad_norm": 0.4406833350658417, + "learning_rate": 4.3294520547945205e-05, + "loss": 0.3189589500427246, + "mean_token_accuracy": 0.9131261318922043, + "num_tokens": 1985410.0, + "step": 1030 + }, + { + "entropy": 0.3856867581605911, + "epoch": 0.1414965986394558, + "grad_norm": 0.32720640301704407, + "learning_rate": 4.322602739726028e-05, + "loss": 0.3763080358505249, + "mean_token_accuracy": 0.9028674513101578, + "num_tokens": 2004750.0, + "step": 1040 + }, + { + "entropy": 0.3344607546925545, + "epoch": 0.14285714285714285, + "grad_norm": 0.5312451720237732, + "learning_rate": 4.3157534246575344e-05, + "loss": 0.3377204656600952, + "mean_token_accuracy": 0.9150244653224945, + "num_tokens": 2022601.0, + "step": 1050 + }, + { + "entropy": 0.3213494800031185, + "epoch": 0.14421768707482993, + "grad_norm": 0.3459687829017639, + "learning_rate": 4.308904109589042e-05, + "loss": 0.3276652812957764, + "mean_token_accuracy": 0.9159560710191726, + "num_tokens": 2041553.0, + "step": 1060 + }, + { + "entropy": 0.4032025709748268, + "epoch": 0.145578231292517, + "grad_norm": 0.42302119731903076, + "learning_rate": 4.302054794520548e-05, + "loss": 0.40665340423583984, + "mean_token_accuracy": 0.8989956825971603, + "num_tokens": 2060111.0, + "step": 1070 + }, + { + "entropy": 0.30957555398344994, + "epoch": 0.1469387755102041, + "grad_norm": 0.540122389793396, + "learning_rate": 4.295205479452055e-05, + "loss": 0.31520886421203614, + "mean_token_accuracy": 0.9152782648801804, + "num_tokens": 2079453.0, + "step": 1080 + }, + { + "entropy": 0.37794227004051206, + "epoch": 0.14829931972789115, + "grad_norm": 0.34265509247779846, + "learning_rate": 4.2883561643835615e-05, + "loss": 0.3954782009124756, + "mean_token_accuracy": 0.9016141533851624, + "num_tokens": 2097734.0, + "step": 1090 + }, + { + "entropy": 0.34080238491296766, + "epoch": 0.14965986394557823, + "grad_norm": 0.40301233530044556, + "learning_rate": 4.281506849315069e-05, + "loss": 0.3306001901626587, + "mean_token_accuracy": 0.9116725891828537, + "num_tokens": 2115880.0, + "step": 1100 + }, + { + "entropy": 0.38523422852158545, + "epoch": 0.1510204081632653, + "grad_norm": 0.4232766330242157, + "learning_rate": 4.2746575342465754e-05, + "loss": 0.3976067304611206, + "mean_token_accuracy": 0.9017782896757126, + "num_tokens": 2134164.0, + "step": 1110 + }, + { + "entropy": 0.35867138430476186, + "epoch": 0.1523809523809524, + "grad_norm": 0.29137903451919556, + "learning_rate": 4.267808219178082e-05, + "loss": 0.34880263805389405, + "mean_token_accuracy": 0.9045450478792191, + "num_tokens": 2155503.0, + "step": 1120 + }, + { + "entropy": 0.4304305985569954, + "epoch": 0.15374149659863945, + "grad_norm": 0.4157155752182007, + "learning_rate": 4.260958904109589e-05, + "loss": 0.4243772029876709, + "mean_token_accuracy": 0.8891431093215942, + "num_tokens": 2174849.0, + "step": 1130 + }, + { + "entropy": 0.3275195978581905, + "epoch": 0.15510204081632653, + "grad_norm": 0.42673543095588684, + "learning_rate": 4.254109589041096e-05, + "loss": 0.32062151432037356, + "mean_token_accuracy": 0.9139250129461288, + "num_tokens": 2195274.0, + "step": 1140 + }, + { + "entropy": 0.31267223581671716, + "epoch": 0.1564625850340136, + "grad_norm": 0.4825468957424164, + "learning_rate": 4.247260273972603e-05, + "loss": 0.3127827882766724, + "mean_token_accuracy": 0.9189456045627594, + "num_tokens": 2213025.0, + "step": 1150 + }, + { + "entropy": 0.34783718213438986, + "epoch": 0.15782312925170067, + "grad_norm": 0.3787980377674103, + "learning_rate": 4.24041095890411e-05, + "loss": 0.33619091510772703, + "mean_token_accuracy": 0.9100428611040116, + "num_tokens": 2233726.0, + "step": 1160 + }, + { + "entropy": 0.25309042185544967, + "epoch": 0.15918367346938775, + "grad_norm": 0.3635607659816742, + "learning_rate": 4.233561643835617e-05, + "loss": 0.25443458557128906, + "mean_token_accuracy": 0.9310213536024093, + "num_tokens": 2252029.0, + "step": 1170 + }, + { + "entropy": 0.3736408993601799, + "epoch": 0.16054421768707483, + "grad_norm": 0.4148007333278656, + "learning_rate": 4.226712328767124e-05, + "loss": 0.38751490116119386, + "mean_token_accuracy": 0.9018953382968903, + "num_tokens": 2269954.0, + "step": 1180 + }, + { + "entropy": 0.31956026926636694, + "epoch": 0.1619047619047619, + "grad_norm": 0.4839099645614624, + "learning_rate": 4.21986301369863e-05, + "loss": 0.32132811546325685, + "mean_token_accuracy": 0.912396851181984, + "num_tokens": 2288506.0, + "step": 1190 + }, + { + "entropy": 0.3568165637552738, + "epoch": 0.16326530612244897, + "grad_norm": 0.24555467069149017, + "learning_rate": 4.213013698630137e-05, + "loss": 0.3542040824890137, + "mean_token_accuracy": 0.9109734386205673, + "num_tokens": 2307894.0, + "step": 1200 + }, + { + "entropy": 0.35158547312021254, + "epoch": 0.16462585034013605, + "grad_norm": 0.412095844745636, + "learning_rate": 4.2061643835616435e-05, + "loss": 0.3496436357498169, + "mean_token_accuracy": 0.9088854372501374, + "num_tokens": 2327240.0, + "step": 1210 + }, + { + "entropy": 0.34600893333554267, + "epoch": 0.16598639455782313, + "grad_norm": 0.3335567116737366, + "learning_rate": 4.199315068493151e-05, + "loss": 0.34185967445373533, + "mean_token_accuracy": 0.905845221877098, + "num_tokens": 2345594.0, + "step": 1220 + }, + { + "entropy": 0.3424117244780064, + "epoch": 0.1673469387755102, + "grad_norm": 0.4858080744743347, + "learning_rate": 4.1924657534246574e-05, + "loss": 0.33852126598358157, + "mean_token_accuracy": 0.913614547252655, + "num_tokens": 2363745.0, + "step": 1230 + }, + { + "entropy": 0.340135945379734, + "epoch": 0.16870748299319727, + "grad_norm": 0.4143927991390228, + "learning_rate": 4.185616438356165e-05, + "loss": 0.3389458417892456, + "mean_token_accuracy": 0.9139795809984207, + "num_tokens": 2382660.0, + "step": 1240 + }, + { + "entropy": 0.35108796805143355, + "epoch": 0.17006802721088435, + "grad_norm": 0.4392998516559601, + "learning_rate": 4.178767123287671e-05, + "loss": 0.356292724609375, + "mean_token_accuracy": 0.9068086326122284, + "num_tokens": 2403842.0, + "step": 1250 + }, + { + "epoch": 0.17006802721088435, + "eval_entropy": 0.34711426744858426, + "eval_loss": 0.349116712808609, + "eval_mean_token_accuracy": 0.9085024491945902, + "eval_num_tokens": 2403842.0, + "eval_runtime": 39.5145, + "eval_samples_per_second": 15.184, + "eval_steps_per_second": 7.592, + "step": 1250 + }, + { + "entropy": 0.3387380100786686, + "epoch": 0.17142857142857143, + "grad_norm": 0.4315812587738037, + "learning_rate": 4.1719178082191786e-05, + "loss": 0.3605659484863281, + "mean_token_accuracy": 0.9103223711252213, + "num_tokens": 2422375.0, + "step": 1260 + }, + { + "entropy": 0.3285294592380524, + "epoch": 0.1727891156462585, + "grad_norm": 0.28438887000083923, + "learning_rate": 4.165068493150685e-05, + "loss": 0.32664496898651124, + "mean_token_accuracy": 0.9147592782974243, + "num_tokens": 2441057.0, + "step": 1270 + }, + { + "entropy": 0.33626926839351656, + "epoch": 0.17414965986394557, + "grad_norm": 0.3710070848464966, + "learning_rate": 4.1582191780821925e-05, + "loss": 0.3354215621948242, + "mean_token_accuracy": 0.9121009469032287, + "num_tokens": 2459887.0, + "step": 1280 + }, + { + "entropy": 0.32256170734763145, + "epoch": 0.17551020408163265, + "grad_norm": 0.3689749240875244, + "learning_rate": 4.151369863013699e-05, + "loss": 0.334096360206604, + "mean_token_accuracy": 0.9107798606157302, + "num_tokens": 2481764.0, + "step": 1290 + }, + { + "entropy": 0.3187907926738262, + "epoch": 0.17687074829931973, + "grad_norm": 0.45171478390693665, + "learning_rate": 4.144520547945206e-05, + "loss": 0.31167116165161135, + "mean_token_accuracy": 0.916716742515564, + "num_tokens": 2501140.0, + "step": 1300 + }, + { + "entropy": 0.3347592830657959, + "epoch": 0.1782312925170068, + "grad_norm": 0.3578483760356903, + "learning_rate": 4.1376712328767123e-05, + "loss": 0.3300051689147949, + "mean_token_accuracy": 0.9117922872304917, + "num_tokens": 2520373.0, + "step": 1310 + }, + { + "entropy": 0.34503707885742185, + "epoch": 0.17959183673469387, + "grad_norm": 0.48708587884902954, + "learning_rate": 4.130821917808219e-05, + "loss": 0.35149393081665037, + "mean_token_accuracy": 0.9106566429138183, + "num_tokens": 2539252.0, + "step": 1320 + }, + { + "entropy": 0.2878501363098621, + "epoch": 0.18095238095238095, + "grad_norm": 0.295443594455719, + "learning_rate": 4.123972602739726e-05, + "loss": 0.28683664798736574, + "mean_token_accuracy": 0.9232668906450272, + "num_tokens": 2559632.0, + "step": 1330 + }, + { + "entropy": 0.35331556648015977, + "epoch": 0.18231292517006803, + "grad_norm": 0.39578038454055786, + "learning_rate": 4.117123287671233e-05, + "loss": 0.3638563871383667, + "mean_token_accuracy": 0.9076030403375626, + "num_tokens": 2578101.0, + "step": 1340 + }, + { + "entropy": 0.2722251106053591, + "epoch": 0.1836734693877551, + "grad_norm": 0.36816802620887756, + "learning_rate": 4.11027397260274e-05, + "loss": 0.2725792169570923, + "mean_token_accuracy": 0.9225607991218567, + "num_tokens": 2596872.0, + "step": 1350 + }, + { + "entropy": 0.31183348074555395, + "epoch": 0.18503401360544217, + "grad_norm": 0.31272897124290466, + "learning_rate": 4.103424657534247e-05, + "loss": 0.3018923044204712, + "mean_token_accuracy": 0.918904721736908, + "num_tokens": 2616948.0, + "step": 1360 + }, + { + "entropy": 0.3306739188730717, + "epoch": 0.18639455782312925, + "grad_norm": 0.49856531620025635, + "learning_rate": 4.096575342465754e-05, + "loss": 0.326325798034668, + "mean_token_accuracy": 0.9133409053087235, + "num_tokens": 2635194.0, + "step": 1370 + }, + { + "entropy": 0.3449342973530293, + "epoch": 0.18775510204081633, + "grad_norm": 0.3672386705875397, + "learning_rate": 4.0897260273972606e-05, + "loss": 0.34992690086364747, + "mean_token_accuracy": 0.910362508893013, + "num_tokens": 2653415.0, + "step": 1380 + }, + { + "entropy": 0.34170059114694595, + "epoch": 0.1891156462585034, + "grad_norm": 0.438903272151947, + "learning_rate": 4.082876712328767e-05, + "loss": 0.362008261680603, + "mean_token_accuracy": 0.9066114306449891, + "num_tokens": 2672556.0, + "step": 1390 + }, + { + "entropy": 0.36418116837739944, + "epoch": 0.19047619047619047, + "grad_norm": 0.3735516369342804, + "learning_rate": 4.076027397260274e-05, + "loss": 0.36933226585388185, + "mean_token_accuracy": 0.903956550359726, + "num_tokens": 2691019.0, + "step": 1400 + }, + { + "entropy": 0.33693169951438906, + "epoch": 0.19183673469387755, + "grad_norm": 0.3407398760318756, + "learning_rate": 4.0691780821917805e-05, + "loss": 0.3302572965621948, + "mean_token_accuracy": 0.9152946561574936, + "num_tokens": 2710242.0, + "step": 1410 + }, + { + "entropy": 0.3172802679240704, + "epoch": 0.19319727891156463, + "grad_norm": 0.3666602373123169, + "learning_rate": 4.062328767123288e-05, + "loss": 0.3129624366760254, + "mean_token_accuracy": 0.9118194162845612, + "num_tokens": 2731126.0, + "step": 1420 + }, + { + "entropy": 0.39142856150865557, + "epoch": 0.1945578231292517, + "grad_norm": 0.3450668752193451, + "learning_rate": 4.0554794520547944e-05, + "loss": 0.3807058572769165, + "mean_token_accuracy": 0.899824932217598, + "num_tokens": 2752136.0, + "step": 1430 + }, + { + "entropy": 0.354360631108284, + "epoch": 0.19591836734693877, + "grad_norm": 0.41342902183532715, + "learning_rate": 4.0486301369863016e-05, + "loss": 0.3667241096496582, + "mean_token_accuracy": 0.9088259905576705, + "num_tokens": 2769136.0, + "step": 1440 + }, + { + "entropy": 0.38877039551734927, + "epoch": 0.19727891156462585, + "grad_norm": 0.49252015352249146, + "learning_rate": 4.041780821917808e-05, + "loss": 0.3996612548828125, + "mean_token_accuracy": 0.898360651731491, + "num_tokens": 2788101.0, + "step": 1450 + }, + { + "entropy": 0.3267239861190319, + "epoch": 0.19863945578231293, + "grad_norm": 0.3556021451950073, + "learning_rate": 4.0349315068493155e-05, + "loss": 0.31743640899658204, + "mean_token_accuracy": 0.9156175047159195, + "num_tokens": 2809422.0, + "step": 1460 + }, + { + "entropy": 0.3570724129676819, + "epoch": 0.2, + "grad_norm": 0.41443583369255066, + "learning_rate": 4.028082191780822e-05, + "loss": 0.34733829498291013, + "mean_token_accuracy": 0.9090855300426484, + "num_tokens": 2828897.0, + "step": 1470 + }, + { + "entropy": 0.36629826575517654, + "epoch": 0.20136054421768707, + "grad_norm": 0.43215256929397583, + "learning_rate": 4.0212328767123294e-05, + "loss": 0.37642669677734375, + "mean_token_accuracy": 0.9042314618825913, + "num_tokens": 2846647.0, + "step": 1480 + }, + { + "entropy": 0.3530550479888916, + "epoch": 0.20272108843537415, + "grad_norm": 0.3693477213382721, + "learning_rate": 4.014383561643836e-05, + "loss": 0.35866584777832033, + "mean_token_accuracy": 0.9062743157148361, + "num_tokens": 2866552.0, + "step": 1490 + }, + { + "entropy": 0.3125508435070515, + "epoch": 0.20408163265306123, + "grad_norm": 0.48748907446861267, + "learning_rate": 4.0075342465753427e-05, + "loss": 0.3137276887893677, + "mean_token_accuracy": 0.9152947783470153, + "num_tokens": 2887091.0, + "step": 1500 + }, + { + "epoch": 0.20408163265306123, + "eval_entropy": 0.3448227975765864, + "eval_loss": 0.34629908204078674, + "eval_mean_token_accuracy": 0.9089537409941355, + "eval_num_tokens": 2887091.0, + "eval_runtime": 39.562, + "eval_samples_per_second": 15.166, + "eval_steps_per_second": 7.583, + "step": 1500 + }, + { + "entropy": 0.3338837943971157, + "epoch": 0.2054421768707483, + "grad_norm": 0.453998327255249, + "learning_rate": 4.000684931506849e-05, + "loss": 0.33314809799194334, + "mean_token_accuracy": 0.9141378700733185, + "num_tokens": 2906558.0, + "step": 1510 + }, + { + "entropy": 0.35243152529001237, + "epoch": 0.20680272108843537, + "grad_norm": 0.514372706413269, + "learning_rate": 3.993835616438356e-05, + "loss": 0.36109921932220457, + "mean_token_accuracy": 0.909517452120781, + "num_tokens": 2924701.0, + "step": 1520 + }, + { + "entropy": 0.35977045372128486, + "epoch": 0.20816326530612245, + "grad_norm": 0.47331997752189636, + "learning_rate": 3.986986301369863e-05, + "loss": 0.3542989730834961, + "mean_token_accuracy": 0.9023140043020248, + "num_tokens": 2944323.0, + "step": 1530 + }, + { + "entropy": 0.3476133652031422, + "epoch": 0.20952380952380953, + "grad_norm": 0.39329883456230164, + "learning_rate": 3.98013698630137e-05, + "loss": 0.35582053661346436, + "mean_token_accuracy": 0.9074772745370865, + "num_tokens": 2962847.0, + "step": 1540 + }, + { + "entropy": 0.36178177297115327, + "epoch": 0.2108843537414966, + "grad_norm": 0.42516037821769714, + "learning_rate": 3.973287671232877e-05, + "loss": 0.3781341552734375, + "mean_token_accuracy": 0.9034180849790573, + "num_tokens": 2982036.0, + "step": 1550 + }, + { + "entropy": 0.3484080970287323, + "epoch": 0.21224489795918366, + "grad_norm": 0.4276966452598572, + "learning_rate": 3.9664383561643837e-05, + "loss": 0.34070544242858886, + "mean_token_accuracy": 0.9100886523723603, + "num_tokens": 3001246.0, + "step": 1560 + }, + { + "entropy": 0.296015814691782, + "epoch": 0.21360544217687075, + "grad_norm": 0.3481068015098572, + "learning_rate": 3.959589041095891e-05, + "loss": 0.3041130542755127, + "mean_token_accuracy": 0.9180979192256927, + "num_tokens": 3021457.0, + "step": 1570 + }, + { + "entropy": 0.33566633649170396, + "epoch": 0.21496598639455783, + "grad_norm": 0.46466073393821716, + "learning_rate": 3.9527397260273976e-05, + "loss": 0.3371039152145386, + "mean_token_accuracy": 0.9121709406375885, + "num_tokens": 3040277.0, + "step": 1580 + }, + { + "entropy": 0.37958977892994883, + "epoch": 0.2163265306122449, + "grad_norm": 0.38823264837265015, + "learning_rate": 3.945890410958905e-05, + "loss": 0.36861262321472166, + "mean_token_accuracy": 0.9052564293146134, + "num_tokens": 3060403.0, + "step": 1590 + }, + { + "entropy": 0.35140516981482506, + "epoch": 0.21768707482993196, + "grad_norm": 0.3407391607761383, + "learning_rate": 3.9390410958904114e-05, + "loss": 0.3561867237091064, + "mean_token_accuracy": 0.9101497054100036, + "num_tokens": 3079560.0, + "step": 1600 + }, + { + "entropy": 0.32827898040413855, + "epoch": 0.21904761904761905, + "grad_norm": 0.3683736324310303, + "learning_rate": 3.932191780821918e-05, + "loss": 0.3281137704849243, + "mean_token_accuracy": 0.9150744706392289, + "num_tokens": 3096762.0, + "step": 1610 + }, + { + "entropy": 0.38666338548064233, + "epoch": 0.22040816326530613, + "grad_norm": 0.4814898669719696, + "learning_rate": 3.925342465753425e-05, + "loss": 0.3958434104919434, + "mean_token_accuracy": 0.9005985498428345, + "num_tokens": 3116551.0, + "step": 1620 + }, + { + "entropy": 0.3289907656610012, + "epoch": 0.2217687074829932, + "grad_norm": 0.34105587005615234, + "learning_rate": 3.918493150684931e-05, + "loss": 0.33880622386932374, + "mean_token_accuracy": 0.9120993465185165, + "num_tokens": 3136954.0, + "step": 1630 + }, + { + "entropy": 0.4120957344770432, + "epoch": 0.22312925170068026, + "grad_norm": 0.2596409022808075, + "learning_rate": 3.9116438356164386e-05, + "loss": 0.411406946182251, + "mean_token_accuracy": 0.8971390217542649, + "num_tokens": 3157285.0, + "step": 1640 + }, + { + "entropy": 0.32644312381744384, + "epoch": 0.22448979591836735, + "grad_norm": 0.3950192332267761, + "learning_rate": 3.904794520547945e-05, + "loss": 0.31403555870056155, + "mean_token_accuracy": 0.9128539711236954, + "num_tokens": 3176288.0, + "step": 1650 + }, + { + "entropy": 0.35888044387102125, + "epoch": 0.22585034013605443, + "grad_norm": 0.35740935802459717, + "learning_rate": 3.8979452054794525e-05, + "loss": 0.3686177968978882, + "mean_token_accuracy": 0.9048507601022721, + "num_tokens": 3196362.0, + "step": 1660 + }, + { + "entropy": 0.3604126326739788, + "epoch": 0.2272108843537415, + "grad_norm": 0.4237874746322632, + "learning_rate": 3.891095890410959e-05, + "loss": 0.3526318073272705, + "mean_token_accuracy": 0.9068311870098114, + "num_tokens": 3214053.0, + "step": 1670 + }, + { + "entropy": 0.34602590948343276, + "epoch": 0.22857142857142856, + "grad_norm": 0.36912399530410767, + "learning_rate": 3.8842465753424663e-05, + "loss": 0.3596346378326416, + "mean_token_accuracy": 0.905837458372116, + "num_tokens": 3233180.0, + "step": 1680 + }, + { + "entropy": 0.3587561517953873, + "epoch": 0.22993197278911565, + "grad_norm": 0.5846858024597168, + "learning_rate": 3.877397260273973e-05, + "loss": 0.3572895288467407, + "mean_token_accuracy": 0.9061489969491958, + "num_tokens": 3253148.0, + "step": 1690 + }, + { + "entropy": 0.31483854800462724, + "epoch": 0.23129251700680273, + "grad_norm": 0.4492280185222626, + "learning_rate": 3.8705479452054796e-05, + "loss": 0.3197118520736694, + "mean_token_accuracy": 0.91756871342659, + "num_tokens": 3272318.0, + "step": 1700 + }, + { + "entropy": 0.399021328240633, + "epoch": 0.23265306122448978, + "grad_norm": 0.44325029850006104, + "learning_rate": 3.863698630136987e-05, + "loss": 0.40597949028015134, + "mean_token_accuracy": 0.895935806632042, + "num_tokens": 3291607.0, + "step": 1710 + }, + { + "entropy": 0.3489011608064175, + "epoch": 0.23401360544217686, + "grad_norm": 0.4789123237133026, + "learning_rate": 3.856849315068493e-05, + "loss": 0.3425743579864502, + "mean_token_accuracy": 0.9078541338443756, + "num_tokens": 3311423.0, + "step": 1720 + }, + { + "entropy": 0.3557139202952385, + "epoch": 0.23537414965986395, + "grad_norm": 0.3092873990535736, + "learning_rate": 3.85e-05, + "loss": 0.36077446937561036, + "mean_token_accuracy": 0.9053440719842911, + "num_tokens": 3333426.0, + "step": 1730 + }, + { + "entropy": 0.3405990473926067, + "epoch": 0.23673469387755103, + "grad_norm": 0.4567224383354187, + "learning_rate": 3.843150684931507e-05, + "loss": 0.3459942817687988, + "mean_token_accuracy": 0.9079800844192505, + "num_tokens": 3352658.0, + "step": 1740 + }, + { + "entropy": 0.3876554772257805, + "epoch": 0.23809523809523808, + "grad_norm": 0.5339434146881104, + "learning_rate": 3.836301369863014e-05, + "loss": 0.39633448123931886, + "mean_token_accuracy": 0.902654567360878, + "num_tokens": 3371460.0, + "step": 1750 + }, + { + "epoch": 0.23809523809523808, + "eval_entropy": 0.3414275799691677, + "eval_loss": 0.34352293610572815, + "eval_mean_token_accuracy": 0.9096791291236878, + "eval_num_tokens": 3371460.0, + "eval_runtime": 39.7497, + "eval_samples_per_second": 15.094, + "eval_steps_per_second": 7.547, + "step": 1750 + }, + { + "entropy": 0.3382601335644722, + "epoch": 0.23945578231292516, + "grad_norm": 0.4466438591480255, + "learning_rate": 3.8294520547945206e-05, + "loss": 0.3433582305908203, + "mean_token_accuracy": 0.911861115694046, + "num_tokens": 3389275.0, + "step": 1760 + }, + { + "entropy": 0.42951491847634315, + "epoch": 0.24081632653061225, + "grad_norm": 0.37805283069610596, + "learning_rate": 3.822602739726028e-05, + "loss": 0.42836475372314453, + "mean_token_accuracy": 0.8932142198085785, + "num_tokens": 3408444.0, + "step": 1770 + }, + { + "entropy": 0.35968755930662155, + "epoch": 0.24217687074829933, + "grad_norm": 0.40297001600265503, + "learning_rate": 3.8157534246575345e-05, + "loss": 0.358404278755188, + "mean_token_accuracy": 0.9076766610145569, + "num_tokens": 3427335.0, + "step": 1780 + }, + { + "entropy": 0.3270475797355175, + "epoch": 0.24353741496598638, + "grad_norm": 0.3639969825744629, + "learning_rate": 3.808904109589042e-05, + "loss": 0.31723456382751464, + "mean_token_accuracy": 0.9145333647727967, + "num_tokens": 3446864.0, + "step": 1790 + }, + { + "entropy": 0.30234274081885815, + "epoch": 0.24489795918367346, + "grad_norm": 0.443294882774353, + "learning_rate": 3.8020547945205484e-05, + "loss": 0.3082306385040283, + "mean_token_accuracy": 0.9216614037752151, + "num_tokens": 3464556.0, + "step": 1800 + }, + { + "entropy": 0.29795855954289435, + "epoch": 0.24625850340136055, + "grad_norm": 0.39209097623825073, + "learning_rate": 3.795205479452055e-05, + "loss": 0.3015660285949707, + "mean_token_accuracy": 0.9175312757492066, + "num_tokens": 3482284.0, + "step": 1810 + }, + { + "entropy": 0.31364143155515195, + "epoch": 0.24761904761904763, + "grad_norm": 0.522806704044342, + "learning_rate": 3.7883561643835616e-05, + "loss": 0.3150556802749634, + "mean_token_accuracy": 0.9156683564186097, + "num_tokens": 3500454.0, + "step": 1820 + }, + { + "entropy": 0.31209259331226347, + "epoch": 0.24897959183673468, + "grad_norm": 0.36000123620033264, + "learning_rate": 3.781506849315068e-05, + "loss": 0.3044718027114868, + "mean_token_accuracy": 0.9180886089801789, + "num_tokens": 3519086.0, + "step": 1830 + }, + { + "entropy": 0.3723416179418564, + "epoch": 0.2503401360544218, + "grad_norm": 0.4354872703552246, + "learning_rate": 3.7746575342465755e-05, + "loss": 0.3991520404815674, + "mean_token_accuracy": 0.9001334726810455, + "num_tokens": 3537252.0, + "step": 1840 + }, + { + "entropy": 0.375165493786335, + "epoch": 0.25170068027210885, + "grad_norm": 0.46084484457969666, + "learning_rate": 3.767808219178082e-05, + "loss": 0.3683051586151123, + "mean_token_accuracy": 0.902921998500824, + "num_tokens": 3555749.0, + "step": 1850 + }, + { + "entropy": 0.3341012619435787, + "epoch": 0.2530612244897959, + "grad_norm": 0.4521940052509308, + "learning_rate": 3.7609589041095894e-05, + "loss": 0.3300362825393677, + "mean_token_accuracy": 0.9145215421915054, + "num_tokens": 3573805.0, + "step": 1860 + }, + { + "entropy": 0.3390877529978752, + "epoch": 0.254421768707483, + "grad_norm": 0.392527312040329, + "learning_rate": 3.754109589041096e-05, + "loss": 0.3547995090484619, + "mean_token_accuracy": 0.9103965133428573, + "num_tokens": 3592844.0, + "step": 1870 + }, + { + "entropy": 0.27256989255547526, + "epoch": 0.25578231292517006, + "grad_norm": 0.4466555118560791, + "learning_rate": 3.747260273972603e-05, + "loss": 0.27302353382110595, + "mean_token_accuracy": 0.9277669697999954, + "num_tokens": 3612185.0, + "step": 1880 + }, + { + "entropy": 0.24652124755084515, + "epoch": 0.2571428571428571, + "grad_norm": 0.4279734194278717, + "learning_rate": 3.74041095890411e-05, + "loss": 0.23633365631103515, + "mean_token_accuracy": 0.9341515839099884, + "num_tokens": 3630518.0, + "step": 1890 + }, + { + "entropy": 0.35038353204727174, + "epoch": 0.2585034013605442, + "grad_norm": 0.5714797973632812, + "learning_rate": 3.7335616438356165e-05, + "loss": 0.344893741607666, + "mean_token_accuracy": 0.907752126455307, + "num_tokens": 3650072.0, + "step": 1900 + }, + { + "entropy": 0.3278027452528477, + "epoch": 0.2598639455782313, + "grad_norm": 0.37882331013679504, + "learning_rate": 3.726712328767124e-05, + "loss": 0.330623722076416, + "mean_token_accuracy": 0.9118026465177536, + "num_tokens": 3670750.0, + "step": 1910 + }, + { + "entropy": 0.3359388172626495, + "epoch": 0.2612244897959184, + "grad_norm": 0.5028720498085022, + "learning_rate": 3.7198630136986304e-05, + "loss": 0.3352250576019287, + "mean_token_accuracy": 0.9122653841972351, + "num_tokens": 3690610.0, + "step": 1920 + }, + { + "entropy": 0.3143607035279274, + "epoch": 0.26258503401360545, + "grad_norm": 0.46486079692840576, + "learning_rate": 3.713013698630137e-05, + "loss": 0.30714452266693115, + "mean_token_accuracy": 0.9196935653686523, + "num_tokens": 3708272.0, + "step": 1930 + }, + { + "entropy": 0.35282155945897103, + "epoch": 0.2639455782312925, + "grad_norm": 0.45841529965400696, + "learning_rate": 3.7061643835616436e-05, + "loss": 0.3545929670333862, + "mean_token_accuracy": 0.9059949398040772, + "num_tokens": 3726869.0, + "step": 1940 + }, + { + "entropy": 0.2984108135104179, + "epoch": 0.2653061224489796, + "grad_norm": 0.42509031295776367, + "learning_rate": 3.699315068493151e-05, + "loss": 0.2950887203216553, + "mean_token_accuracy": 0.9201341837644577, + "num_tokens": 3746172.0, + "step": 1950 + }, + { + "entropy": 0.31338903233408927, + "epoch": 0.26666666666666666, + "grad_norm": 0.37572163343429565, + "learning_rate": 3.6924657534246575e-05, + "loss": 0.3111075162887573, + "mean_token_accuracy": 0.916234827041626, + "num_tokens": 3765690.0, + "step": 1960 + }, + { + "entropy": 0.31969371140003205, + "epoch": 0.2680272108843537, + "grad_norm": 0.4484844207763672, + "learning_rate": 3.685616438356165e-05, + "loss": 0.32388155460357665, + "mean_token_accuracy": 0.9160292863845825, + "num_tokens": 3783682.0, + "step": 1970 + }, + { + "entropy": 0.34558562338352206, + "epoch": 0.2693877551020408, + "grad_norm": 0.6314212679862976, + "learning_rate": 3.6787671232876714e-05, + "loss": 0.3459887742996216, + "mean_token_accuracy": 0.9086391866207123, + "num_tokens": 3801830.0, + "step": 1980 + }, + { + "entropy": 0.315334652364254, + "epoch": 0.2707482993197279, + "grad_norm": 0.9491491317749023, + "learning_rate": 3.671917808219179e-05, + "loss": 0.3217782974243164, + "mean_token_accuracy": 0.914330929517746, + "num_tokens": 3821971.0, + "step": 1990 + }, + { + "entropy": 0.3376677267253399, + "epoch": 0.272108843537415, + "grad_norm": 0.4743102192878723, + "learning_rate": 3.665068493150685e-05, + "loss": 0.3347081184387207, + "mean_token_accuracy": 0.9096920937299728, + "num_tokens": 3841489.0, + "step": 2000 + }, + { + "epoch": 0.272108843537415, + "eval_entropy": 0.3519680585463842, + "eval_loss": 0.342134028673172, + "eval_mean_token_accuracy": 0.9097507733106613, + "eval_num_tokens": 3841489.0, + "eval_runtime": 39.6137, + "eval_samples_per_second": 15.146, + "eval_steps_per_second": 7.573, + "step": 2000 + }, + { + "entropy": 0.35166340619325637, + "epoch": 0.27346938775510204, + "grad_norm": 0.2940121591091156, + "learning_rate": 3.658219178082192e-05, + "loss": 0.3434895515441895, + "mean_token_accuracy": 0.9088268965482712, + "num_tokens": 3861515.0, + "step": 2010 + }, + { + "entropy": 0.3252212144434452, + "epoch": 0.2748299319727891, + "grad_norm": 0.33630767464637756, + "learning_rate": 3.651369863013699e-05, + "loss": 0.3320222616195679, + "mean_token_accuracy": 0.9136297762393951, + "num_tokens": 3880492.0, + "step": 2020 + }, + { + "entropy": 0.30923423618078233, + "epoch": 0.2761904761904762, + "grad_norm": 0.3880252242088318, + "learning_rate": 3.644520547945206e-05, + "loss": 0.30057878494262696, + "mean_token_accuracy": 0.921365550160408, + "num_tokens": 3900283.0, + "step": 2030 + }, + { + "entropy": 0.3484115995466709, + "epoch": 0.27755102040816326, + "grad_norm": 0.4368172883987427, + "learning_rate": 3.6376712328767124e-05, + "loss": 0.33826982975006104, + "mean_token_accuracy": 0.9120531916618347, + "num_tokens": 3918622.0, + "step": 2040 + }, + { + "entropy": 0.301137863099575, + "epoch": 0.2789115646258503, + "grad_norm": 0.44099488854408264, + "learning_rate": 3.630821917808219e-05, + "loss": 0.3043475866317749, + "mean_token_accuracy": 0.9190519124269485, + "num_tokens": 3936916.0, + "step": 2050 + }, + { + "entropy": 0.32400831282138826, + "epoch": 0.2802721088435374, + "grad_norm": 0.3683919310569763, + "learning_rate": 3.623972602739726e-05, + "loss": 0.32993350028991697, + "mean_token_accuracy": 0.9162216484546661, + "num_tokens": 3954118.0, + "step": 2060 + }, + { + "entropy": 0.36117612645030023, + "epoch": 0.2816326530612245, + "grad_norm": 0.36590471863746643, + "learning_rate": 3.617123287671233e-05, + "loss": 0.366199254989624, + "mean_token_accuracy": 0.9073205977678299, + "num_tokens": 3972244.0, + "step": 2070 + }, + { + "entropy": 0.30845518559217455, + "epoch": 0.2829931972789116, + "grad_norm": 0.4904245436191559, + "learning_rate": 3.61027397260274e-05, + "loss": 0.3000732660293579, + "mean_token_accuracy": 0.9213456630706787, + "num_tokens": 3991933.0, + "step": 2080 + }, + { + "entropy": 0.33342998176813127, + "epoch": 0.28435374149659864, + "grad_norm": 0.3451215326786041, + "learning_rate": 3.603424657534247e-05, + "loss": 0.3519883155822754, + "mean_token_accuracy": 0.9118590801954269, + "num_tokens": 4009549.0, + "step": 2090 + }, + { + "entropy": 0.3225520886480808, + "epoch": 0.2857142857142857, + "grad_norm": 0.41656896471977234, + "learning_rate": 3.5965753424657534e-05, + "loss": 0.33789401054382323, + "mean_token_accuracy": 0.9126103073358536, + "num_tokens": 4028579.0, + "step": 2100 + }, + { + "entropy": 0.33171583339571953, + "epoch": 0.2870748299319728, + "grad_norm": 0.44855958223342896, + "learning_rate": 3.589726027397261e-05, + "loss": 0.33625202178955077, + "mean_token_accuracy": 0.9129469394683838, + "num_tokens": 4047155.0, + "step": 2110 + }, + { + "entropy": 0.3181050315499306, + "epoch": 0.28843537414965986, + "grad_norm": 0.45970842242240906, + "learning_rate": 3.582876712328767e-05, + "loss": 0.3234997749328613, + "mean_token_accuracy": 0.9154122680425644, + "num_tokens": 4065666.0, + "step": 2120 + }, + { + "entropy": 0.3713026508688927, + "epoch": 0.2897959183673469, + "grad_norm": 0.4234248399734497, + "learning_rate": 3.5760273972602746e-05, + "loss": 0.36728248596191404, + "mean_token_accuracy": 0.9066497653722763, + "num_tokens": 4084684.0, + "step": 2130 + }, + { + "entropy": 0.36940672621130943, + "epoch": 0.291156462585034, + "grad_norm": 0.490075021982193, + "learning_rate": 3.569178082191781e-05, + "loss": 0.3767258644104004, + "mean_token_accuracy": 0.9038274347782135, + "num_tokens": 4104584.0, + "step": 2140 + }, + { + "entropy": 0.36020471081137656, + "epoch": 0.2925170068027211, + "grad_norm": 0.45468005537986755, + "learning_rate": 3.562328767123288e-05, + "loss": 0.36363468170166013, + "mean_token_accuracy": 0.907564926147461, + "num_tokens": 4122918.0, + "step": 2150 + }, + { + "entropy": 0.320191890001297, + "epoch": 0.2938775510204082, + "grad_norm": 0.4919210374355316, + "learning_rate": 3.5554794520547944e-05, + "loss": 0.3147299766540527, + "mean_token_accuracy": 0.919364845752716, + "num_tokens": 4140528.0, + "step": 2160 + }, + { + "entropy": 0.39545504078269006, + "epoch": 0.29523809523809524, + "grad_norm": 0.4305582344532013, + "learning_rate": 3.548630136986302e-05, + "loss": 0.4035975456237793, + "mean_token_accuracy": 0.8999768882989884, + "num_tokens": 4159136.0, + "step": 2170 + }, + { + "entropy": 0.32939055263996125, + "epoch": 0.2965986394557823, + "grad_norm": 0.5001799464225769, + "learning_rate": 3.541780821917808e-05, + "loss": 0.32146081924438474, + "mean_token_accuracy": 0.9139656722545624, + "num_tokens": 4178728.0, + "step": 2180 + }, + { + "entropy": 0.3295467995107174, + "epoch": 0.2979591836734694, + "grad_norm": 0.5351600050926208, + "learning_rate": 3.5349315068493156e-05, + "loss": 0.32642207145690916, + "mean_token_accuracy": 0.9078761160373687, + "num_tokens": 4199352.0, + "step": 2190 + }, + { + "entropy": 0.3639287866652012, + "epoch": 0.29931972789115646, + "grad_norm": 0.5141214728355408, + "learning_rate": 3.528082191780822e-05, + "loss": 0.3758009195327759, + "mean_token_accuracy": 0.9029638439416885, + "num_tokens": 4220071.0, + "step": 2200 + }, + { + "entropy": 0.31801966652274133, + "epoch": 0.3006802721088435, + "grad_norm": 0.4239181876182556, + "learning_rate": 3.521232876712329e-05, + "loss": 0.3060357093811035, + "mean_token_accuracy": 0.9188521355390549, + "num_tokens": 4238856.0, + "step": 2210 + }, + { + "entropy": 0.3254566751420498, + "epoch": 0.3020408163265306, + "grad_norm": 0.5890150666236877, + "learning_rate": 3.514383561643836e-05, + "loss": 0.33809313774108884, + "mean_token_accuracy": 0.9106950730085372, + "num_tokens": 4258568.0, + "step": 2220 + }, + { + "entropy": 0.3605205237865448, + "epoch": 0.3034013605442177, + "grad_norm": 0.4173797070980072, + "learning_rate": 3.507534246575343e-05, + "loss": 0.3492074251174927, + "mean_token_accuracy": 0.9083913922309875, + "num_tokens": 4277649.0, + "step": 2230 + }, + { + "entropy": 0.3326388791203499, + "epoch": 0.3047619047619048, + "grad_norm": 0.510657012462616, + "learning_rate": 3.500684931506849e-05, + "loss": 0.3288934946060181, + "mean_token_accuracy": 0.9091042637825012, + "num_tokens": 4297822.0, + "step": 2240 + }, + { + "entropy": 0.32781085297465323, + "epoch": 0.30612244897959184, + "grad_norm": 0.41473498940467834, + "learning_rate": 3.493835616438356e-05, + "loss": 0.320549750328064, + "mean_token_accuracy": 0.9140338152647018, + "num_tokens": 4316581.0, + "step": 2250 + }, + { + "epoch": 0.30612244897959184, + "eval_entropy": 0.3347464001675447, + "eval_loss": 0.3400094211101532, + "eval_mean_token_accuracy": 0.9105568333466848, + "eval_num_tokens": 4316581.0, + "eval_runtime": 39.5997, + "eval_samples_per_second": 15.152, + "eval_steps_per_second": 7.576, + "step": 2250 + }, + { + "entropy": 0.29784226417541504, + "epoch": 0.3074829931972789, + "grad_norm": 0.3205791711807251, + "learning_rate": 3.486986301369863e-05, + "loss": 0.2996628761291504, + "mean_token_accuracy": 0.918051677942276, + "num_tokens": 4335064.0, + "step": 2260 + }, + { + "entropy": 0.31014653742313386, + "epoch": 0.308843537414966, + "grad_norm": 0.4868067502975464, + "learning_rate": 3.48013698630137e-05, + "loss": 0.31834020614624026, + "mean_token_accuracy": 0.9162485599517822, + "num_tokens": 4355734.0, + "step": 2270 + }, + { + "entropy": 0.32516508251428605, + "epoch": 0.31020408163265306, + "grad_norm": 0.4637530446052551, + "learning_rate": 3.473287671232877e-05, + "loss": 0.33171901702880857, + "mean_token_accuracy": 0.9147695600986481, + "num_tokens": 4373625.0, + "step": 2280 + }, + { + "entropy": 0.3301460199058056, + "epoch": 0.3115646258503401, + "grad_norm": 0.39910921454429626, + "learning_rate": 3.466438356164384e-05, + "loss": 0.33153936862945554, + "mean_token_accuracy": 0.9145438969135284, + "num_tokens": 4391557.0, + "step": 2290 + }, + { + "entropy": 0.35131814479827883, + "epoch": 0.3129251700680272, + "grad_norm": 0.4119342267513275, + "learning_rate": 3.45958904109589e-05, + "loss": 0.36605679988861084, + "mean_token_accuracy": 0.9072954088449479, + "num_tokens": 4409285.0, + "step": 2300 + }, + { + "entropy": 0.38690097257494926, + "epoch": 0.3142857142857143, + "grad_norm": 0.4198577404022217, + "learning_rate": 3.4527397260273976e-05, + "loss": 0.39443681240081785, + "mean_token_accuracy": 0.9032483011484146, + "num_tokens": 4427549.0, + "step": 2310 + }, + { + "entropy": 0.3198526822030544, + "epoch": 0.31564625850340133, + "grad_norm": 0.5783278942108154, + "learning_rate": 3.445890410958904e-05, + "loss": 0.31236040592193604, + "mean_token_accuracy": 0.916850745677948, + "num_tokens": 4445150.0, + "step": 2320 + }, + { + "entropy": 0.37613857463002204, + "epoch": 0.31700680272108844, + "grad_norm": 0.4824889302253723, + "learning_rate": 3.4390410958904115e-05, + "loss": 0.38405096530914307, + "mean_token_accuracy": 0.9016139656305313, + "num_tokens": 4464890.0, + "step": 2330 + }, + { + "entropy": 0.3105509489774704, + "epoch": 0.3183673469387755, + "grad_norm": 0.40339168906211853, + "learning_rate": 3.432191780821918e-05, + "loss": 0.30131852626800537, + "mean_token_accuracy": 0.916943484544754, + "num_tokens": 4484914.0, + "step": 2340 + }, + { + "entropy": 0.34861349165439603, + "epoch": 0.3197278911564626, + "grad_norm": 0.35453712940216064, + "learning_rate": 3.425342465753425e-05, + "loss": 0.3576103687286377, + "mean_token_accuracy": 0.9065870314836502, + "num_tokens": 4503897.0, + "step": 2350 + }, + { + "entropy": 0.3504960760474205, + "epoch": 0.32108843537414966, + "grad_norm": 0.36962467432022095, + "learning_rate": 3.418493150684931e-05, + "loss": 0.3403719186782837, + "mean_token_accuracy": 0.9101202934980392, + "num_tokens": 4523958.0, + "step": 2360 + }, + { + "entropy": 0.3424271956086159, + "epoch": 0.3224489795918367, + "grad_norm": 0.4860788881778717, + "learning_rate": 3.4116438356164386e-05, + "loss": 0.3569652080535889, + "mean_token_accuracy": 0.9091333240270615, + "num_tokens": 4540850.0, + "step": 2370 + }, + { + "entropy": 0.3189090140163898, + "epoch": 0.3238095238095238, + "grad_norm": 0.4140557050704956, + "learning_rate": 3.404794520547945e-05, + "loss": 0.315256404876709, + "mean_token_accuracy": 0.9175257176160813, + "num_tokens": 4558898.0, + "step": 2380 + }, + { + "entropy": 0.33776298984885217, + "epoch": 0.3251700680272109, + "grad_norm": 0.36254385113716125, + "learning_rate": 3.397945205479452e-05, + "loss": 0.33310599327087403, + "mean_token_accuracy": 0.9075568825006485, + "num_tokens": 4578067.0, + "step": 2390 + }, + { + "entropy": 0.35714403837919234, + "epoch": 0.32653061224489793, + "grad_norm": 0.36844414472579956, + "learning_rate": 3.391095890410959e-05, + "loss": 0.36895968914031985, + "mean_token_accuracy": 0.9088596940040589, + "num_tokens": 4596553.0, + "step": 2400 + }, + { + "entropy": 0.34672648683190344, + "epoch": 0.32789115646258504, + "grad_norm": 0.5768862962722778, + "learning_rate": 3.384246575342466e-05, + "loss": 0.3418919801712036, + "mean_token_accuracy": 0.9105334728956223, + "num_tokens": 4617409.0, + "step": 2410 + }, + { + "entropy": 0.35704168528318403, + "epoch": 0.3292517006802721, + "grad_norm": 0.3797224760055542, + "learning_rate": 3.377397260273973e-05, + "loss": 0.35338263511657714, + "mean_token_accuracy": 0.907687571644783, + "num_tokens": 4636036.0, + "step": 2420 + }, + { + "entropy": 0.37071596682071684, + "epoch": 0.3306122448979592, + "grad_norm": 0.41465795040130615, + "learning_rate": 3.3705479452054796e-05, + "loss": 0.375014066696167, + "mean_token_accuracy": 0.9049693375825882, + "num_tokens": 4654270.0, + "step": 2430 + }, + { + "entropy": 0.3085879869759083, + "epoch": 0.33197278911564626, + "grad_norm": 0.5098187327384949, + "learning_rate": 3.363698630136987e-05, + "loss": 0.30675497055053713, + "mean_token_accuracy": 0.9197484374046325, + "num_tokens": 4672398.0, + "step": 2440 + }, + { + "entropy": 0.3546553373336792, + "epoch": 0.3333333333333333, + "grad_norm": 0.4272635877132416, + "learning_rate": 3.3568493150684935e-05, + "loss": 0.36507577896118165, + "mean_token_accuracy": 0.9074768930673599, + "num_tokens": 4690119.0, + "step": 2450 + }, + { + "entropy": 0.3424846582114697, + "epoch": 0.3346938775510204, + "grad_norm": 0.47268643975257874, + "learning_rate": 3.35e-05, + "loss": 0.33769257068634034, + "mean_token_accuracy": 0.912876409292221, + "num_tokens": 4709956.0, + "step": 2460 + }, + { + "entropy": 0.36993408799171446, + "epoch": 0.3360544217687075, + "grad_norm": 0.4779612421989441, + "learning_rate": 3.343150684931507e-05, + "loss": 0.37760152816772463, + "mean_token_accuracy": 0.9003741055727005, + "num_tokens": 4730284.0, + "step": 2470 + }, + { + "entropy": 0.34832309633493425, + "epoch": 0.33741496598639453, + "grad_norm": 0.6033436059951782, + "learning_rate": 3.336301369863014e-05, + "loss": 0.346789026260376, + "mean_token_accuracy": 0.9062613487243653, + "num_tokens": 4750695.0, + "step": 2480 + }, + { + "entropy": 0.3566281244158745, + "epoch": 0.33877551020408164, + "grad_norm": 0.5511232018470764, + "learning_rate": 3.3294520547945206e-05, + "loss": 0.33579089641571047, + "mean_token_accuracy": 0.9080673784017563, + "num_tokens": 4770493.0, + "step": 2490 + }, + { + "entropy": 0.3206308774650097, + "epoch": 0.3401360544217687, + "grad_norm": 0.3410138189792633, + "learning_rate": 3.322602739726027e-05, + "loss": 0.3231546401977539, + "mean_token_accuracy": 0.9120282381772995, + "num_tokens": 4791470.0, + "step": 2500 + }, + { + "epoch": 0.3401360544217687, + "eval_entropy": 0.3339786528050899, + "eval_loss": 0.3382870852947235, + "eval_mean_token_accuracy": 0.910913230975469, + "eval_num_tokens": 4791470.0, + "eval_runtime": 39.7078, + "eval_samples_per_second": 15.11, + "eval_steps_per_second": 7.555, + "step": 2500 + }, + { + "entropy": 0.3026216134428978, + "epoch": 0.3414965986394558, + "grad_norm": 0.49190741777420044, + "learning_rate": 3.3157534246575345e-05, + "loss": 0.31151907444000243, + "mean_token_accuracy": 0.9197212547063828, + "num_tokens": 4808923.0, + "step": 2510 + }, + { + "entropy": 0.35362531840801237, + "epoch": 0.34285714285714286, + "grad_norm": 0.4947229325771332, + "learning_rate": 3.308904109589041e-05, + "loss": 0.35634171962738037, + "mean_token_accuracy": 0.905270916223526, + "num_tokens": 4828698.0, + "step": 2520 + }, + { + "entropy": 0.2948874182999134, + "epoch": 0.3442176870748299, + "grad_norm": 0.588706910610199, + "learning_rate": 3.3020547945205484e-05, + "loss": 0.3044130802154541, + "mean_token_accuracy": 0.9255930632352829, + "num_tokens": 4846152.0, + "step": 2530 + }, + { + "entropy": 0.30128093957901003, + "epoch": 0.345578231292517, + "grad_norm": 0.38747715950012207, + "learning_rate": 3.295205479452055e-05, + "loss": 0.2940336227416992, + "mean_token_accuracy": 0.9189084142446518, + "num_tokens": 4866211.0, + "step": 2540 + }, + { + "entropy": 0.3247291907668114, + "epoch": 0.3469387755102041, + "grad_norm": 0.4447266161441803, + "learning_rate": 3.288356164383562e-05, + "loss": 0.32945706844329836, + "mean_token_accuracy": 0.912653061747551, + "num_tokens": 4886257.0, + "step": 2550 + }, + { + "entropy": 0.35191837064921855, + "epoch": 0.34829931972789113, + "grad_norm": 0.44626155495643616, + "learning_rate": 3.281506849315069e-05, + "loss": 0.33861584663391114, + "mean_token_accuracy": 0.9103022038936615, + "num_tokens": 4905249.0, + "step": 2560 + }, + { + "entropy": 0.2881168477237225, + "epoch": 0.34965986394557824, + "grad_norm": 0.44346174597740173, + "learning_rate": 3.2746575342465755e-05, + "loss": 0.28231775760650635, + "mean_token_accuracy": 0.9268552869558334, + "num_tokens": 4923178.0, + "step": 2570 + }, + { + "entropy": 0.3358376182615757, + "epoch": 0.3510204081632653, + "grad_norm": 0.32187992334365845, + "learning_rate": 3.267808219178082e-05, + "loss": 0.3331818342208862, + "mean_token_accuracy": 0.9085330307483673, + "num_tokens": 4942861.0, + "step": 2580 + }, + { + "entropy": 0.30966882929205897, + "epoch": 0.3523809523809524, + "grad_norm": 0.5043341517448425, + "learning_rate": 3.260958904109589e-05, + "loss": 0.31597659587860105, + "mean_token_accuracy": 0.9186759322881699, + "num_tokens": 4960880.0, + "step": 2590 + }, + { + "entropy": 0.31486412063241004, + "epoch": 0.35374149659863946, + "grad_norm": 0.3755909204483032, + "learning_rate": 3.254109589041096e-05, + "loss": 0.30836541652679444, + "mean_token_accuracy": 0.9174209177494049, + "num_tokens": 4979663.0, + "step": 2600 + }, + { + "entropy": 0.3569600723683834, + "epoch": 0.3551020408163265, + "grad_norm": 0.5074579119682312, + "learning_rate": 3.2472602739726026e-05, + "loss": 0.3442463159561157, + "mean_token_accuracy": 0.9093485355377198, + "num_tokens": 4999805.0, + "step": 2610 + }, + { + "entropy": 0.31667746156454085, + "epoch": 0.3564625850340136, + "grad_norm": 0.41301679611206055, + "learning_rate": 3.24041095890411e-05, + "loss": 0.3286094903945923, + "mean_token_accuracy": 0.9134628236293793, + "num_tokens": 5019199.0, + "step": 2620 + }, + { + "entropy": 0.321327581256628, + "epoch": 0.3578231292517007, + "grad_norm": 0.6529260277748108, + "learning_rate": 3.2335616438356165e-05, + "loss": 0.3306095838546753, + "mean_token_accuracy": 0.9112082034349441, + "num_tokens": 5038915.0, + "step": 2630 + }, + { + "entropy": 0.36806232705712316, + "epoch": 0.35918367346938773, + "grad_norm": 0.4105146527290344, + "learning_rate": 3.226712328767124e-05, + "loss": 0.37603011131286623, + "mean_token_accuracy": 0.903413000702858, + "num_tokens": 5057723.0, + "step": 2640 + }, + { + "entropy": 0.31727024130523207, + "epoch": 0.36054421768707484, + "grad_norm": 0.4973562955856323, + "learning_rate": 3.2198630136986304e-05, + "loss": 0.3162988662719727, + "mean_token_accuracy": 0.9186434358358383, + "num_tokens": 5076899.0, + "step": 2650 + }, + { + "entropy": 0.3479673244059086, + "epoch": 0.3619047619047619, + "grad_norm": 0.471534788608551, + "learning_rate": 3.213013698630137e-05, + "loss": 0.34523987770080566, + "mean_token_accuracy": 0.9076945811510087, + "num_tokens": 5098260.0, + "step": 2660 + }, + { + "entropy": 0.3287705913186073, + "epoch": 0.363265306122449, + "grad_norm": 0.3614688217639923, + "learning_rate": 3.2061643835616436e-05, + "loss": 0.3317812442779541, + "mean_token_accuracy": 0.9130234062671662, + "num_tokens": 5118009.0, + "step": 2670 + }, + { + "entropy": 0.3424487292766571, + "epoch": 0.36462585034013606, + "grad_norm": 0.473599910736084, + "learning_rate": 3.199315068493151e-05, + "loss": 0.3391443729400635, + "mean_token_accuracy": 0.9129292786121368, + "num_tokens": 5137099.0, + "step": 2680 + }, + { + "entropy": 0.370761601626873, + "epoch": 0.3659863945578231, + "grad_norm": 0.6083559989929199, + "learning_rate": 3.1924657534246575e-05, + "loss": 0.3879230499267578, + "mean_token_accuracy": 0.8995687663555145, + "num_tokens": 5157056.0, + "step": 2690 + }, + { + "entropy": 0.34785791113972664, + "epoch": 0.3673469387755102, + "grad_norm": 0.3559763431549072, + "learning_rate": 3.185616438356164e-05, + "loss": 0.34124653339385985, + "mean_token_accuracy": 0.909286892414093, + "num_tokens": 5177253.0, + "step": 2700 + }, + { + "entropy": 0.3293781645596027, + "epoch": 0.3687074829931973, + "grad_norm": 0.5223445296287537, + "learning_rate": 3.1787671232876714e-05, + "loss": 0.32764997482299807, + "mean_token_accuracy": 0.9157842248678207, + "num_tokens": 5194945.0, + "step": 2710 + }, + { + "entropy": 0.31182133629918096, + "epoch": 0.37006802721088433, + "grad_norm": 0.3634929358959198, + "learning_rate": 3.171917808219178e-05, + "loss": 0.31785967350006106, + "mean_token_accuracy": 0.9155223101377488, + "num_tokens": 5215068.0, + "step": 2720 + }, + { + "entropy": 0.39456270560622214, + "epoch": 0.37142857142857144, + "grad_norm": 1.0898386240005493, + "learning_rate": 3.165068493150685e-05, + "loss": 0.4019328594207764, + "mean_token_accuracy": 0.896870681643486, + "num_tokens": 5236257.0, + "step": 2730 + }, + { + "entropy": 0.3375735577195883, + "epoch": 0.3727891156462585, + "grad_norm": 0.5985054969787598, + "learning_rate": 3.158219178082192e-05, + "loss": 0.34434778690338136, + "mean_token_accuracy": 0.914423531293869, + "num_tokens": 5254512.0, + "step": 2740 + }, + { + "entropy": 0.3534530118107796, + "epoch": 0.3741496598639456, + "grad_norm": 0.6718593835830688, + "learning_rate": 3.151369863013699e-05, + "loss": 0.34789319038391114, + "mean_token_accuracy": 0.9068450212478638, + "num_tokens": 5273306.0, + "step": 2750 + }, + { + "epoch": 0.3741496598639456, + "eval_entropy": 0.34889839991927146, + "eval_loss": 0.33721673488616943, + "eval_mean_token_accuracy": 0.911351277033488, + "eval_num_tokens": 5273306.0, + "eval_runtime": 39.6807, + "eval_samples_per_second": 15.121, + "eval_steps_per_second": 7.56, + "step": 2750 + }, + { + "entropy": 0.34381640776991845, + "epoch": 0.37551020408163266, + "grad_norm": 0.3617115914821625, + "learning_rate": 3.144520547945206e-05, + "loss": 0.34432876110076904, + "mean_token_accuracy": 0.9133523643016815, + "num_tokens": 5293581.0, + "step": 2760 + }, + { + "entropy": 0.3725735809653997, + "epoch": 0.3768707482993197, + "grad_norm": 0.39663493633270264, + "learning_rate": 3.1376712328767124e-05, + "loss": 0.3688955545425415, + "mean_token_accuracy": 0.9088011533021927, + "num_tokens": 5312584.0, + "step": 2770 + }, + { + "entropy": 0.287184102833271, + "epoch": 0.3782312925170068, + "grad_norm": 0.5590073466300964, + "learning_rate": 3.130821917808219e-05, + "loss": 0.27388792037963866, + "mean_token_accuracy": 0.919337859749794, + "num_tokens": 5333734.0, + "step": 2780 + }, + { + "entropy": 0.31367961689829826, + "epoch": 0.3795918367346939, + "grad_norm": 0.4748844504356384, + "learning_rate": 3.1239726027397256e-05, + "loss": 0.3206137418746948, + "mean_token_accuracy": 0.9146426022052765, + "num_tokens": 5353231.0, + "step": 2790 + }, + { + "entropy": 0.33578074127435686, + "epoch": 0.38095238095238093, + "grad_norm": 0.5165243148803711, + "learning_rate": 3.117123287671233e-05, + "loss": 0.3436616897583008, + "mean_token_accuracy": 0.9078721046447754, + "num_tokens": 5373437.0, + "step": 2800 + }, + { + "entropy": 0.30183863267302513, + "epoch": 0.38231292517006804, + "grad_norm": 0.5405089259147644, + "learning_rate": 3.1102739726027395e-05, + "loss": 0.29791650772094724, + "mean_token_accuracy": 0.9211073070764542, + "num_tokens": 5394481.0, + "step": 2810 + }, + { + "entropy": 0.3263554021716118, + "epoch": 0.3836734693877551, + "grad_norm": 0.3398967981338501, + "learning_rate": 3.103424657534247e-05, + "loss": 0.33998684883117675, + "mean_token_accuracy": 0.911750203371048, + "num_tokens": 5414726.0, + "step": 2820 + }, + { + "entropy": 0.3496020220220089, + "epoch": 0.38503401360544215, + "grad_norm": 0.49333450198173523, + "learning_rate": 3.0965753424657534e-05, + "loss": 0.3456233263015747, + "mean_token_accuracy": 0.9098768889904022, + "num_tokens": 5433364.0, + "step": 2830 + }, + { + "entropy": 0.35282182693481445, + "epoch": 0.38639455782312926, + "grad_norm": 0.4709138572216034, + "learning_rate": 3.089726027397261e-05, + "loss": 0.3554483413696289, + "mean_token_accuracy": 0.9042170315980911, + "num_tokens": 5454586.0, + "step": 2840 + }, + { + "entropy": 0.2756012246012688, + "epoch": 0.3877551020408163, + "grad_norm": 0.3347083330154419, + "learning_rate": 3.082876712328767e-05, + "loss": 0.2776206016540527, + "mean_token_accuracy": 0.9246499836444855, + "num_tokens": 5472836.0, + "step": 2850 + }, + { + "entropy": 0.3471778355538845, + "epoch": 0.3891156462585034, + "grad_norm": 0.3866668939590454, + "learning_rate": 3.0760273972602746e-05, + "loss": 0.3517859697341919, + "mean_token_accuracy": 0.9072955340147019, + "num_tokens": 5491741.0, + "step": 2860 + }, + { + "entropy": 0.31448092609643935, + "epoch": 0.3904761904761905, + "grad_norm": 0.5152408480644226, + "learning_rate": 3.069178082191781e-05, + "loss": 0.32549118995666504, + "mean_token_accuracy": 0.9156864136457443, + "num_tokens": 5510575.0, + "step": 2870 + }, + { + "entropy": 0.37327151745557785, + "epoch": 0.39183673469387753, + "grad_norm": 0.4741872549057007, + "learning_rate": 3.062328767123288e-05, + "loss": 0.3845684289932251, + "mean_token_accuracy": 0.8996114999055862, + "num_tokens": 5529986.0, + "step": 2880 + }, + { + "entropy": 0.3675620459020138, + "epoch": 0.39319727891156464, + "grad_norm": 0.43704453110694885, + "learning_rate": 3.0554794520547944e-05, + "loss": 0.3695012092590332, + "mean_token_accuracy": 0.9072604596614837, + "num_tokens": 5549632.0, + "step": 2890 + }, + { + "entropy": 0.3603727616369724, + "epoch": 0.3945578231292517, + "grad_norm": 0.4204140901565552, + "learning_rate": 3.0486301369863014e-05, + "loss": 0.361452054977417, + "mean_token_accuracy": 0.9049343466758728, + "num_tokens": 5569455.0, + "step": 2900 + }, + { + "entropy": 0.31857734769582746, + "epoch": 0.39591836734693875, + "grad_norm": 0.543210506439209, + "learning_rate": 3.0417808219178083e-05, + "loss": 0.3216622114181519, + "mean_token_accuracy": 0.9114610522985458, + "num_tokens": 5590154.0, + "step": 2910 + }, + { + "entropy": 0.32558867931365965, + "epoch": 0.39727891156462586, + "grad_norm": 0.39182421565055847, + "learning_rate": 3.034931506849315e-05, + "loss": 0.3142657518386841, + "mean_token_accuracy": 0.9159083515405655, + "num_tokens": 5608934.0, + "step": 2920 + }, + { + "entropy": 0.35831537619233134, + "epoch": 0.3986394557823129, + "grad_norm": 0.43255022168159485, + "learning_rate": 3.0280821917808222e-05, + "loss": 0.3551504373550415, + "mean_token_accuracy": 0.9060514450073243, + "num_tokens": 5629700.0, + "step": 2930 + }, + { + "entropy": 0.32889301925897596, + "epoch": 0.4, + "grad_norm": 0.4353126883506775, + "learning_rate": 3.021232876712329e-05, + "loss": 0.33272716999053953, + "mean_token_accuracy": 0.9120247811079025, + "num_tokens": 5648205.0, + "step": 2940 + }, + { + "entropy": 0.4077422976493835, + "epoch": 0.4013605442176871, + "grad_norm": 0.5193459391593933, + "learning_rate": 3.0143835616438358e-05, + "loss": 0.42622690200805663, + "mean_token_accuracy": 0.8950145274400712, + "num_tokens": 5666658.0, + "step": 2950 + }, + { + "entropy": 0.2837504968047142, + "epoch": 0.40272108843537413, + "grad_norm": 0.4110262095928192, + "learning_rate": 3.0075342465753424e-05, + "loss": 0.2780358552932739, + "mean_token_accuracy": 0.9248676776885987, + "num_tokens": 5685895.0, + "step": 2960 + }, + { + "entropy": 0.2994197480380535, + "epoch": 0.40408163265306124, + "grad_norm": 0.5185297727584839, + "learning_rate": 3.0006849315068497e-05, + "loss": 0.3070257902145386, + "mean_token_accuracy": 0.9217073917388916, + "num_tokens": 5705410.0, + "step": 2970 + }, + { + "entropy": 0.3405400663614273, + "epoch": 0.4054421768707483, + "grad_norm": 0.446480929851532, + "learning_rate": 2.9938356164383563e-05, + "loss": 0.34464187622070314, + "mean_token_accuracy": 0.91073057949543, + "num_tokens": 5725937.0, + "step": 2980 + }, + { + "entropy": 0.3204948931932449, + "epoch": 0.40680272108843535, + "grad_norm": 0.4274948835372925, + "learning_rate": 2.986986301369863e-05, + "loss": 0.3086549758911133, + "mean_token_accuracy": 0.9216050624847412, + "num_tokens": 5744274.0, + "step": 2990 + }, + { + "entropy": 0.319742513448, + "epoch": 0.40816326530612246, + "grad_norm": 0.43988749384880066, + "learning_rate": 2.9801369863013702e-05, + "loss": 0.3238105535507202, + "mean_token_accuracy": 0.9127194285392761, + "num_tokens": 5764224.0, + "step": 3000 + }, + { + "epoch": 0.40816326530612246, + "eval_entropy": 0.33606284650663537, + "eval_loss": 0.33526092767715454, + "eval_mean_token_accuracy": 0.9117824711402257, + "eval_num_tokens": 5764224.0, + "eval_runtime": 39.6422, + "eval_samples_per_second": 15.135, + "eval_steps_per_second": 7.568, + "step": 3000 + }, + { + "entropy": 0.3738269850611687, + "epoch": 0.4095238095238095, + "grad_norm": 0.5675323605537415, + "learning_rate": 2.9732876712328768e-05, + "loss": 0.3804152488708496, + "mean_token_accuracy": 0.9026560753583908, + "num_tokens": 5781507.0, + "step": 3010 + }, + { + "entropy": 0.35528839454054834, + "epoch": 0.4108843537414966, + "grad_norm": 0.37698662281036377, + "learning_rate": 2.9664383561643837e-05, + "loss": 0.35562241077423096, + "mean_token_accuracy": 0.9055985480546951, + "num_tokens": 5801540.0, + "step": 3020 + }, + { + "entropy": 0.34376412332057954, + "epoch": 0.4122448979591837, + "grad_norm": 0.48629990220069885, + "learning_rate": 2.9595890410958904e-05, + "loss": 0.3413914680480957, + "mean_token_accuracy": 0.9106980264186859, + "num_tokens": 5820095.0, + "step": 3030 + }, + { + "entropy": 0.28705876022577287, + "epoch": 0.41360544217687073, + "grad_norm": 0.3930319845676422, + "learning_rate": 2.9527397260273976e-05, + "loss": 0.2918708801269531, + "mean_token_accuracy": 0.9254306346178055, + "num_tokens": 5836806.0, + "step": 3040 + }, + { + "entropy": 0.28956920132040975, + "epoch": 0.41496598639455784, + "grad_norm": 0.3387986421585083, + "learning_rate": 2.9458904109589042e-05, + "loss": 0.28510375022888185, + "mean_token_accuracy": 0.9249218046665192, + "num_tokens": 5855499.0, + "step": 3050 + }, + { + "entropy": 0.3717873513698578, + "epoch": 0.4163265306122449, + "grad_norm": 0.5153553485870361, + "learning_rate": 2.9390410958904112e-05, + "loss": 0.36185555458068847, + "mean_token_accuracy": 0.9043754696846008, + "num_tokens": 5875651.0, + "step": 3060 + }, + { + "entropy": 0.3651160977780819, + "epoch": 0.41768707482993195, + "grad_norm": 0.622138261795044, + "learning_rate": 2.9321917808219178e-05, + "loss": 0.36736748218536375, + "mean_token_accuracy": 0.9075702458620072, + "num_tokens": 5893916.0, + "step": 3070 + }, + { + "entropy": 0.2804809086024761, + "epoch": 0.41904761904761906, + "grad_norm": 0.4219469726085663, + "learning_rate": 2.9253424657534244e-05, + "loss": 0.2844717025756836, + "mean_token_accuracy": 0.921401646733284, + "num_tokens": 5912149.0, + "step": 3080 + }, + { + "entropy": 0.314235956966877, + "epoch": 0.4204081632653061, + "grad_norm": 0.5418047308921814, + "learning_rate": 2.9184931506849317e-05, + "loss": 0.3110402822494507, + "mean_token_accuracy": 0.9149896830320359, + "num_tokens": 5932858.0, + "step": 3090 + }, + { + "entropy": 0.32378696352243425, + "epoch": 0.4217687074829932, + "grad_norm": 0.5057699680328369, + "learning_rate": 2.9116438356164383e-05, + "loss": 0.31246180534362794, + "mean_token_accuracy": 0.917678028345108, + "num_tokens": 5950456.0, + "step": 3100 + }, + { + "entropy": 0.3295158065855503, + "epoch": 0.4231292517006803, + "grad_norm": 0.32313019037246704, + "learning_rate": 2.9047945205479456e-05, + "loss": 0.32827062606811525, + "mean_token_accuracy": 0.913808137178421, + "num_tokens": 5970315.0, + "step": 3110 + }, + { + "entropy": 0.3258155144751072, + "epoch": 0.42448979591836733, + "grad_norm": 0.6149294972419739, + "learning_rate": 2.897945205479452e-05, + "loss": 0.3333756923675537, + "mean_token_accuracy": 0.9134649366140366, + "num_tokens": 5989083.0, + "step": 3120 + }, + { + "entropy": 0.33037797883152964, + "epoch": 0.42585034013605444, + "grad_norm": 0.6826731562614441, + "learning_rate": 2.891095890410959e-05, + "loss": 0.33439276218414304, + "mean_token_accuracy": 0.9117500305175781, + "num_tokens": 6008071.0, + "step": 3130 + }, + { + "entropy": 0.3115034744143486, + "epoch": 0.4272108843537415, + "grad_norm": 0.44018131494522095, + "learning_rate": 2.8842465753424658e-05, + "loss": 0.3135596990585327, + "mean_token_accuracy": 0.9178120344877243, + "num_tokens": 6028939.0, + "step": 3140 + }, + { + "entropy": 0.3361651122570038, + "epoch": 0.42857142857142855, + "grad_norm": 0.3561447858810425, + "learning_rate": 2.877397260273973e-05, + "loss": 0.3284465312957764, + "mean_token_accuracy": 0.912507513165474, + "num_tokens": 6050975.0, + "step": 3150 + }, + { + "entropy": 0.3167419731616974, + "epoch": 0.42993197278911566, + "grad_norm": 0.4988236725330353, + "learning_rate": 2.8705479452054797e-05, + "loss": 0.31540229320526125, + "mean_token_accuracy": 0.91805699467659, + "num_tokens": 6068811.0, + "step": 3160 + }, + { + "entropy": 0.35025607869029046, + "epoch": 0.4312925170068027, + "grad_norm": 0.3634585440158844, + "learning_rate": 2.8636986301369866e-05, + "loss": 0.34669277667999265, + "mean_token_accuracy": 0.908183416724205, + "num_tokens": 6089617.0, + "step": 3170 + }, + { + "entropy": 0.33410345017910004, + "epoch": 0.4326530612244898, + "grad_norm": 0.380386084318161, + "learning_rate": 2.8568493150684932e-05, + "loss": 0.35002827644348145, + "mean_token_accuracy": 0.9116801351308823, + "num_tokens": 6108293.0, + "step": 3180 + }, + { + "entropy": 0.3474400773644447, + "epoch": 0.4340136054421769, + "grad_norm": 0.41803106665611267, + "learning_rate": 2.8499999999999998e-05, + "loss": 0.35688033103942873, + "mean_token_accuracy": 0.9099474012851715, + "num_tokens": 6127624.0, + "step": 3190 + }, + { + "entropy": 0.3581741526722908, + "epoch": 0.43537414965986393, + "grad_norm": 0.3337022364139557, + "learning_rate": 2.843150684931507e-05, + "loss": 0.3569644451141357, + "mean_token_accuracy": 0.9068025410175323, + "num_tokens": 6147219.0, + "step": 3200 + }, + { + "entropy": 0.366722059994936, + "epoch": 0.43673469387755104, + "grad_norm": 0.47171923518180847, + "learning_rate": 2.8363013698630137e-05, + "loss": 0.3685397386550903, + "mean_token_accuracy": 0.9036656260490418, + "num_tokens": 6167669.0, + "step": 3210 + }, + { + "entropy": 0.32237810268998146, + "epoch": 0.4380952380952381, + "grad_norm": 0.41389840841293335, + "learning_rate": 2.8294520547945207e-05, + "loss": 0.31839354038238527, + "mean_token_accuracy": 0.9167311549186706, + "num_tokens": 6187141.0, + "step": 3220 + }, + { + "entropy": 0.3237307697534561, + "epoch": 0.43945578231292515, + "grad_norm": 0.5178200006484985, + "learning_rate": 2.8226027397260273e-05, + "loss": 0.32291412353515625, + "mean_token_accuracy": 0.9083501189947129, + "num_tokens": 6209252.0, + "step": 3230 + }, + { + "entropy": 0.34426340013742446, + "epoch": 0.44081632653061226, + "grad_norm": 0.4553462862968445, + "learning_rate": 2.8157534246575346e-05, + "loss": 0.3504526615142822, + "mean_token_accuracy": 0.9100097268819809, + "num_tokens": 6228894.0, + "step": 3240 + }, + { + "entropy": 0.28253506049513816, + "epoch": 0.4421768707482993, + "grad_norm": 0.37686920166015625, + "learning_rate": 2.808904109589041e-05, + "loss": 0.2929633617401123, + "mean_token_accuracy": 0.9215697795152664, + "num_tokens": 6249587.0, + "step": 3250 + }, + { + "epoch": 0.4421768707482993, + "eval_entropy": 0.3371030978113413, + "eval_loss": 0.3342117369174957, + "eval_mean_token_accuracy": 0.9119487730662028, + "eval_num_tokens": 6249587.0, + "eval_runtime": 39.7403, + "eval_samples_per_second": 15.098, + "eval_steps_per_second": 7.549, + "step": 3250 + }, + { + "entropy": 0.3359912931919098, + "epoch": 0.4435374149659864, + "grad_norm": 0.5517682433128357, + "learning_rate": 2.8020547945205484e-05, + "loss": 0.3350799322128296, + "mean_token_accuracy": 0.9072889268398285, + "num_tokens": 6270421.0, + "step": 3260 + }, + { + "entropy": 0.38644204661250114, + "epoch": 0.4448979591836735, + "grad_norm": 0.38367074728012085, + "learning_rate": 2.795205479452055e-05, + "loss": 0.40497770309448244, + "mean_token_accuracy": 0.8989514976739883, + "num_tokens": 6289750.0, + "step": 3270 + }, + { + "entropy": 0.36485532075166704, + "epoch": 0.44625850340136053, + "grad_norm": 0.4921395182609558, + "learning_rate": 2.7883561643835617e-05, + "loss": 0.3587671995162964, + "mean_token_accuracy": 0.9088137924671174, + "num_tokens": 6309476.0, + "step": 3280 + }, + { + "entropy": 0.32781576067209245, + "epoch": 0.44761904761904764, + "grad_norm": 0.405824214220047, + "learning_rate": 2.7815068493150686e-05, + "loss": 0.3323342323303223, + "mean_token_accuracy": 0.9141393929719925, + "num_tokens": 6330359.0, + "step": 3290 + }, + { + "entropy": 0.35046872347593305, + "epoch": 0.4489795918367347, + "grad_norm": 0.4356522858142853, + "learning_rate": 2.7746575342465752e-05, + "loss": 0.3580676555633545, + "mean_token_accuracy": 0.9075710386037826, + "num_tokens": 6350395.0, + "step": 3300 + }, + { + "entropy": 0.3306349150836468, + "epoch": 0.45034013605442175, + "grad_norm": 0.3762452304363251, + "learning_rate": 2.7678082191780825e-05, + "loss": 0.3267848968505859, + "mean_token_accuracy": 0.9178031712770462, + "num_tokens": 6368777.0, + "step": 3310 + }, + { + "entropy": 0.29314617067575455, + "epoch": 0.45170068027210886, + "grad_norm": 0.4260740876197815, + "learning_rate": 2.760958904109589e-05, + "loss": 0.2808928728103638, + "mean_token_accuracy": 0.9242286056280136, + "num_tokens": 6386705.0, + "step": 3320 + }, + { + "entropy": 0.33772081658244135, + "epoch": 0.4530612244897959, + "grad_norm": 0.4309479594230652, + "learning_rate": 2.754109589041096e-05, + "loss": 0.3572484731674194, + "mean_token_accuracy": 0.9095708549022674, + "num_tokens": 6405597.0, + "step": 3330 + }, + { + "entropy": 0.3147235192358494, + "epoch": 0.454421768707483, + "grad_norm": 0.6065618395805359, + "learning_rate": 2.7472602739726027e-05, + "loss": 0.3092878580093384, + "mean_token_accuracy": 0.9218635380268096, + "num_tokens": 6425006.0, + "step": 3340 + }, + { + "entropy": 0.35482636019587516, + "epoch": 0.4557823129251701, + "grad_norm": 0.40962573885917664, + "learning_rate": 2.74041095890411e-05, + "loss": 0.3521085023880005, + "mean_token_accuracy": 0.9050173759460449, + "num_tokens": 6444964.0, + "step": 3350 + }, + { + "entropy": 0.3443373300135136, + "epoch": 0.45714285714285713, + "grad_norm": 0.47444677352905273, + "learning_rate": 2.7335616438356166e-05, + "loss": 0.3417380332946777, + "mean_token_accuracy": 0.9109756410121918, + "num_tokens": 6464883.0, + "step": 3360 + }, + { + "entropy": 0.30666022598743437, + "epoch": 0.45850340136054424, + "grad_norm": 0.3451339900493622, + "learning_rate": 2.7267123287671235e-05, + "loss": 0.308093786239624, + "mean_token_accuracy": 0.9187242925167084, + "num_tokens": 6484170.0, + "step": 3370 + }, + { + "entropy": 0.32256651744246484, + "epoch": 0.4598639455782313, + "grad_norm": 0.6191990971565247, + "learning_rate": 2.71986301369863e-05, + "loss": 0.3281312704086304, + "mean_token_accuracy": 0.915521040558815, + "num_tokens": 6502657.0, + "step": 3380 + }, + { + "entropy": 0.3734627470374107, + "epoch": 0.46122448979591835, + "grad_norm": 0.44824740290641785, + "learning_rate": 2.7130136986301367e-05, + "loss": 0.39291067123413087, + "mean_token_accuracy": 0.9002336442470551, + "num_tokens": 6522335.0, + "step": 3390 + }, + { + "entropy": 0.36504360362887384, + "epoch": 0.46258503401360546, + "grad_norm": 0.44734102487564087, + "learning_rate": 2.706164383561644e-05, + "loss": 0.366800332069397, + "mean_token_accuracy": 0.9116403192281723, + "num_tokens": 6541259.0, + "step": 3400 + }, + { + "entropy": 0.33948337733745576, + "epoch": 0.4639455782312925, + "grad_norm": 0.4975481331348419, + "learning_rate": 2.6993150684931506e-05, + "loss": 0.3351884126663208, + "mean_token_accuracy": 0.911971241235733, + "num_tokens": 6562033.0, + "step": 3410 + }, + { + "entropy": 0.36278425380587576, + "epoch": 0.46530612244897956, + "grad_norm": 0.44590020179748535, + "learning_rate": 2.692465753424658e-05, + "loss": 0.3616467475891113, + "mean_token_accuracy": 0.9112220734357834, + "num_tokens": 6579861.0, + "step": 3420 + }, + { + "entropy": 0.296990454941988, + "epoch": 0.4666666666666667, + "grad_norm": 0.35668641328811646, + "learning_rate": 2.6856164383561645e-05, + "loss": 0.28709557056427004, + "mean_token_accuracy": 0.923720845580101, + "num_tokens": 6597963.0, + "step": 3430 + }, + { + "entropy": 0.30575042217969894, + "epoch": 0.46802721088435373, + "grad_norm": 0.39647671580314636, + "learning_rate": 2.6787671232876715e-05, + "loss": 0.3078245401382446, + "mean_token_accuracy": 0.9169597178697586, + "num_tokens": 6616696.0, + "step": 3440 + }, + { + "entropy": 0.3101175561547279, + "epoch": 0.46938775510204084, + "grad_norm": 0.5445443987846375, + "learning_rate": 2.671917808219178e-05, + "loss": 0.3175575494766235, + "mean_token_accuracy": 0.9126063853502273, + "num_tokens": 6637062.0, + "step": 3450 + }, + { + "entropy": 0.34780861288309095, + "epoch": 0.4707482993197279, + "grad_norm": 0.3975132405757904, + "learning_rate": 2.6650684931506854e-05, + "loss": 0.35774109363555906, + "mean_token_accuracy": 0.9050011664628983, + "num_tokens": 6659231.0, + "step": 3460 + }, + { + "entropy": 0.32717221304774285, + "epoch": 0.47210884353741495, + "grad_norm": 0.4195864796638489, + "learning_rate": 2.658219178082192e-05, + "loss": 0.3109439373016357, + "mean_token_accuracy": 0.9161674290895462, + "num_tokens": 6679690.0, + "step": 3470 + }, + { + "entropy": 0.3339711882174015, + "epoch": 0.47346938775510206, + "grad_norm": 0.35985931754112244, + "learning_rate": 2.6513698630136986e-05, + "loss": 0.33592188358306885, + "mean_token_accuracy": 0.9106827944517135, + "num_tokens": 6698470.0, + "step": 3480 + }, + { + "entropy": 0.3498953923583031, + "epoch": 0.4748299319727891, + "grad_norm": 0.3127373158931732, + "learning_rate": 2.6445205479452055e-05, + "loss": 0.36162471771240234, + "mean_token_accuracy": 0.9053647011518479, + "num_tokens": 6719482.0, + "step": 3490 + }, + { + "entropy": 0.28822562620043757, + "epoch": 0.47619047619047616, + "grad_norm": 0.4204389452934265, + "learning_rate": 2.637671232876712e-05, + "loss": 0.29782910346984864, + "mean_token_accuracy": 0.9186246633529663, + "num_tokens": 6740195.0, + "step": 3500 + }, + { + "epoch": 0.47619047619047616, + "eval_entropy": 0.3430745062977076, + "eval_loss": 0.33298492431640625, + "eval_mean_token_accuracy": 0.9121425338586171, + "eval_num_tokens": 6740195.0, + "eval_runtime": 39.8932, + "eval_samples_per_second": 15.04, + "eval_steps_per_second": 7.52, + "step": 3500 + }, + { + "entropy": 0.32787102460861206, + "epoch": 0.4775510204081633, + "grad_norm": 0.5970384478569031, + "learning_rate": 2.6308219178082194e-05, + "loss": 0.31883673667907714, + "mean_token_accuracy": 0.9141757398843765, + "num_tokens": 6758388.0, + "step": 3510 + }, + { + "entropy": 0.36540004089474676, + "epoch": 0.47891156462585033, + "grad_norm": 0.5385341644287109, + "learning_rate": 2.623972602739726e-05, + "loss": 0.3638452053070068, + "mean_token_accuracy": 0.9118665605783463, + "num_tokens": 6776273.0, + "step": 3520 + }, + { + "entropy": 0.3373605191707611, + "epoch": 0.48027210884353744, + "grad_norm": 0.38150331377983093, + "learning_rate": 2.6171232876712333e-05, + "loss": 0.3207834243774414, + "mean_token_accuracy": 0.9155988216400146, + "num_tokens": 6793132.0, + "step": 3530 + }, + { + "entropy": 0.34099703952670096, + "epoch": 0.4816326530612245, + "grad_norm": 0.5244598984718323, + "learning_rate": 2.61027397260274e-05, + "loss": 0.33450777530670167, + "mean_token_accuracy": 0.9131689667701721, + "num_tokens": 6812749.0, + "step": 3540 + }, + { + "entropy": 0.43126574754714964, + "epoch": 0.48299319727891155, + "grad_norm": 0.4791218638420105, + "learning_rate": 2.603424657534247e-05, + "loss": 0.43479280471801757, + "mean_token_accuracy": 0.8856066018342972, + "num_tokens": 6833553.0, + "step": 3550 + }, + { + "entropy": 0.3489451795816422, + "epoch": 0.48435374149659866, + "grad_norm": 0.6042513251304626, + "learning_rate": 2.5965753424657535e-05, + "loss": 0.3617150068283081, + "mean_token_accuracy": 0.904024675488472, + "num_tokens": 6851609.0, + "step": 3560 + }, + { + "entropy": 0.3345202341675758, + "epoch": 0.4857142857142857, + "grad_norm": 0.47959575057029724, + "learning_rate": 2.5897260273972608e-05, + "loss": 0.34854423999786377, + "mean_token_accuracy": 0.9128339350223541, + "num_tokens": 6869710.0, + "step": 3570 + }, + { + "entropy": 0.2840237572789192, + "epoch": 0.48707482993197276, + "grad_norm": 0.542008638381958, + "learning_rate": 2.5828767123287674e-05, + "loss": 0.28817789554595946, + "mean_token_accuracy": 0.9232237786054611, + "num_tokens": 6889265.0, + "step": 3580 + }, + { + "entropy": 0.37114314883947375, + "epoch": 0.4884353741496599, + "grad_norm": 0.42148521542549133, + "learning_rate": 2.576027397260274e-05, + "loss": 0.3705109119415283, + "mean_token_accuracy": 0.9055974930524826, + "num_tokens": 6907724.0, + "step": 3590 + }, + { + "entropy": 0.3097227029502392, + "epoch": 0.4897959183673469, + "grad_norm": 0.5400570631027222, + "learning_rate": 2.569178082191781e-05, + "loss": 0.3039916276931763, + "mean_token_accuracy": 0.915545716881752, + "num_tokens": 6927218.0, + "step": 3600 + }, + { + "entropy": 0.311353924870491, + "epoch": 0.49115646258503404, + "grad_norm": 0.36434206366539, + "learning_rate": 2.5623287671232875e-05, + "loss": 0.3071993589401245, + "mean_token_accuracy": 0.9152117878198623, + "num_tokens": 6948124.0, + "step": 3610 + }, + { + "entropy": 0.34956820532679556, + "epoch": 0.4925170068027211, + "grad_norm": 0.3756413161754608, + "learning_rate": 2.5554794520547948e-05, + "loss": 0.345737624168396, + "mean_token_accuracy": 0.9104509115219116, + "num_tokens": 6967337.0, + "step": 3620 + }, + { + "entropy": 0.28347715772688387, + "epoch": 0.49387755102040815, + "grad_norm": 0.3341163694858551, + "learning_rate": 2.5486301369863014e-05, + "loss": 0.28453564643859863, + "mean_token_accuracy": 0.9223557978868484, + "num_tokens": 6985719.0, + "step": 3630 + }, + { + "entropy": 0.3215967521071434, + "epoch": 0.49523809523809526, + "grad_norm": 0.4879300892353058, + "learning_rate": 2.5417808219178084e-05, + "loss": 0.3172642230987549, + "mean_token_accuracy": 0.9150819301605224, + "num_tokens": 7003145.0, + "step": 3640 + }, + { + "entropy": 0.3079038575291634, + "epoch": 0.4965986394557823, + "grad_norm": 0.3948201537132263, + "learning_rate": 2.534931506849315e-05, + "loss": 0.32240614891052244, + "mean_token_accuracy": 0.9127428829669952, + "num_tokens": 7023038.0, + "step": 3650 + }, + { + "entropy": 0.38890808299183843, + "epoch": 0.49795918367346936, + "grad_norm": 0.40552157163619995, + "learning_rate": 2.5280821917808223e-05, + "loss": 0.39409432411193845, + "mean_token_accuracy": 0.897028648853302, + "num_tokens": 7046752.0, + "step": 3660 + }, + { + "entropy": 0.3068940281867981, + "epoch": 0.4993197278911565, + "grad_norm": 0.5080393552780151, + "learning_rate": 2.521232876712329e-05, + "loss": 0.2945326566696167, + "mean_token_accuracy": 0.9193342208862305, + "num_tokens": 7066803.0, + "step": 3670 + }, + { + "entropy": 0.30003788247704505, + "epoch": 0.5006802721088436, + "grad_norm": 0.4544333517551422, + "learning_rate": 2.5143835616438355e-05, + "loss": 0.30088942050933837, + "mean_token_accuracy": 0.919922587275505, + "num_tokens": 7086548.0, + "step": 3680 + }, + { + "entropy": 0.3212012261152267, + "epoch": 0.5020408163265306, + "grad_norm": 0.5139970183372498, + "learning_rate": 2.5075342465753428e-05, + "loss": 0.3209099054336548, + "mean_token_accuracy": 0.914613538980484, + "num_tokens": 7103932.0, + "step": 3690 + }, + { + "entropy": 0.3605083726346493, + "epoch": 0.5034013605442177, + "grad_norm": 0.5004434585571289, + "learning_rate": 2.5006849315068494e-05, + "loss": 0.3503889799118042, + "mean_token_accuracy": 0.910282626748085, + "num_tokens": 7122604.0, + "step": 3700 + }, + { + "entropy": 0.32034995295107366, + "epoch": 0.5047619047619047, + "grad_norm": 0.4455152451992035, + "learning_rate": 2.4938356164383563e-05, + "loss": 0.3144600629806519, + "mean_token_accuracy": 0.913508927822113, + "num_tokens": 7141520.0, + "step": 3710 + }, + { + "entropy": 0.32677931264042853, + "epoch": 0.5061224489795918, + "grad_norm": 0.5444691181182861, + "learning_rate": 2.4869863013698633e-05, + "loss": 0.33878235816955565, + "mean_token_accuracy": 0.9117316961288452, + "num_tokens": 7162081.0, + "step": 3720 + }, + { + "entropy": 0.35325164198875425, + "epoch": 0.507482993197279, + "grad_norm": 0.504077136516571, + "learning_rate": 2.48013698630137e-05, + "loss": 0.3583226680755615, + "mean_token_accuracy": 0.9071122765541076, + "num_tokens": 7181824.0, + "step": 3730 + }, + { + "entropy": 0.3118510887026787, + "epoch": 0.508843537414966, + "grad_norm": 0.45883047580718994, + "learning_rate": 2.473287671232877e-05, + "loss": 0.30337512493133545, + "mean_token_accuracy": 0.9171320110559463, + "num_tokens": 7200624.0, + "step": 3740 + }, + { + "entropy": 0.329176065325737, + "epoch": 0.5102040816326531, + "grad_norm": 0.48111435770988464, + "learning_rate": 2.4664383561643838e-05, + "loss": 0.33903625011444094, + "mean_token_accuracy": 0.9144390225410461, + "num_tokens": 7217963.0, + "step": 3750 + }, + { + "epoch": 0.5102040816326531, + "eval_entropy": 0.32912359858552614, + "eval_loss": 0.3315099775791168, + "eval_mean_token_accuracy": 0.9126162391901016, + "eval_num_tokens": 7217963.0, + "eval_runtime": 39.8303, + "eval_samples_per_second": 15.064, + "eval_steps_per_second": 7.532, + "step": 3750 + }, + { + "entropy": 0.3421541929244995, + "epoch": 0.5115646258503401, + "grad_norm": 0.4506266415119171, + "learning_rate": 2.4595890410958904e-05, + "loss": 0.34347808361053467, + "mean_token_accuracy": 0.9077278554439545, + "num_tokens": 7237883.0, + "step": 3760 + }, + { + "entropy": 0.3590623676776886, + "epoch": 0.5129251700680272, + "grad_norm": 0.38278287649154663, + "learning_rate": 2.4527397260273973e-05, + "loss": 0.3690760850906372, + "mean_token_accuracy": 0.9089499413967133, + "num_tokens": 7257138.0, + "step": 3770 + }, + { + "entropy": 0.36101208850741384, + "epoch": 0.5142857142857142, + "grad_norm": 0.32584571838378906, + "learning_rate": 2.4458904109589043e-05, + "loss": 0.3672176837921143, + "mean_token_accuracy": 0.9057380646467209, + "num_tokens": 7277589.0, + "step": 3780 + }, + { + "entropy": 0.2965987451374531, + "epoch": 0.5156462585034014, + "grad_norm": 0.37990301847457886, + "learning_rate": 2.4390410958904112e-05, + "loss": 0.294060206413269, + "mean_token_accuracy": 0.920094347000122, + "num_tokens": 7295419.0, + "step": 3790 + }, + { + "entropy": 0.3669845648109913, + "epoch": 0.5170068027210885, + "grad_norm": 0.395302414894104, + "learning_rate": 2.432191780821918e-05, + "loss": 0.3642202138900757, + "mean_token_accuracy": 0.9083656400442124, + "num_tokens": 7313005.0, + "step": 3800 + }, + { + "entropy": 0.3839815385639668, + "epoch": 0.5183673469387755, + "grad_norm": 0.4179222881793976, + "learning_rate": 2.4253424657534248e-05, + "loss": 0.38060669898986815, + "mean_token_accuracy": 0.9010459512472153, + "num_tokens": 7334087.0, + "step": 3810 + }, + { + "entropy": 0.33595051877200605, + "epoch": 0.5197278911564626, + "grad_norm": 0.35726672410964966, + "learning_rate": 2.4184931506849314e-05, + "loss": 0.3160902976989746, + "mean_token_accuracy": 0.9160772681236267, + "num_tokens": 7353243.0, + "step": 3820 + }, + { + "entropy": 0.3190218761563301, + "epoch": 0.5210884353741496, + "grad_norm": 0.39096274971961975, + "learning_rate": 2.4116438356164384e-05, + "loss": 0.3128466844558716, + "mean_token_accuracy": 0.9177798658609391, + "num_tokens": 7372683.0, + "step": 3830 + }, + { + "entropy": 0.28777577802538873, + "epoch": 0.5224489795918368, + "grad_norm": 0.4006626605987549, + "learning_rate": 2.4047945205479453e-05, + "loss": 0.2932948350906372, + "mean_token_accuracy": 0.9220825433731079, + "num_tokens": 7391156.0, + "step": 3840 + }, + { + "entropy": 0.3199619509279728, + "epoch": 0.5238095238095238, + "grad_norm": 0.35508039593696594, + "learning_rate": 2.3979452054794522e-05, + "loss": 0.3138143539428711, + "mean_token_accuracy": 0.9143597453832626, + "num_tokens": 7412725.0, + "step": 3850 + }, + { + "entropy": 0.35134848281741143, + "epoch": 0.5251700680272109, + "grad_norm": 0.406851202249527, + "learning_rate": 2.391095890410959e-05, + "loss": 0.35069732666015624, + "mean_token_accuracy": 0.910768261551857, + "num_tokens": 7431365.0, + "step": 3860 + }, + { + "entropy": 0.3226579576730728, + "epoch": 0.5265306122448979, + "grad_norm": 0.41831570863723755, + "learning_rate": 2.3842465753424658e-05, + "loss": 0.325051474571228, + "mean_token_accuracy": 0.9144408613443374, + "num_tokens": 7451520.0, + "step": 3870 + }, + { + "entropy": 0.27824380323290826, + "epoch": 0.527891156462585, + "grad_norm": 0.45743539929389954, + "learning_rate": 2.3773972602739727e-05, + "loss": 0.27892796993255614, + "mean_token_accuracy": 0.9252461075782776, + "num_tokens": 7470779.0, + "step": 3880 + }, + { + "entropy": 0.31983816549181937, + "epoch": 0.5292517006802722, + "grad_norm": 0.41483598947525024, + "learning_rate": 2.3705479452054797e-05, + "loss": 0.3180886507034302, + "mean_token_accuracy": 0.9165385901927948, + "num_tokens": 7489545.0, + "step": 3890 + }, + { + "entropy": 0.3343293689191341, + "epoch": 0.5306122448979592, + "grad_norm": 0.5109151601791382, + "learning_rate": 2.3636986301369866e-05, + "loss": 0.34193923473358157, + "mean_token_accuracy": 0.9141032576560975, + "num_tokens": 7507380.0, + "step": 3900 + }, + { + "entropy": 0.3208622790873051, + "epoch": 0.5319727891156463, + "grad_norm": 0.4916702210903168, + "learning_rate": 2.3568493150684933e-05, + "loss": 0.3239308834075928, + "mean_token_accuracy": 0.914379420876503, + "num_tokens": 7526466.0, + "step": 3910 + }, + { + "entropy": 0.3093146879225969, + "epoch": 0.5333333333333333, + "grad_norm": 0.32689499855041504, + "learning_rate": 2.35e-05, + "loss": 0.3163978815078735, + "mean_token_accuracy": 0.9175358593463898, + "num_tokens": 7547417.0, + "step": 3920 + }, + { + "entropy": 0.3612152311950922, + "epoch": 0.5346938775510204, + "grad_norm": 0.44085893034935, + "learning_rate": 2.3431506849315068e-05, + "loss": 0.3802115678787231, + "mean_token_accuracy": 0.902223750948906, + "num_tokens": 7568296.0, + "step": 3930 + }, + { + "entropy": 0.3383258245885372, + "epoch": 0.5360544217687074, + "grad_norm": 0.5608168244361877, + "learning_rate": 2.3363013698630138e-05, + "loss": 0.3415839433670044, + "mean_token_accuracy": 0.9112567931413651, + "num_tokens": 7587752.0, + "step": 3940 + }, + { + "entropy": 0.3647989623248577, + "epoch": 0.5374149659863946, + "grad_norm": 0.4722004532814026, + "learning_rate": 2.3294520547945207e-05, + "loss": 0.3584833860397339, + "mean_token_accuracy": 0.9099332094192505, + "num_tokens": 7607739.0, + "step": 3950 + }, + { + "entropy": 0.3759518824517727, + "epoch": 0.5387755102040817, + "grad_norm": 0.6017438173294067, + "learning_rate": 2.3226027397260277e-05, + "loss": 0.3851759433746338, + "mean_token_accuracy": 0.902659210562706, + "num_tokens": 7625816.0, + "step": 3960 + }, + { + "entropy": 0.31676906198263166, + "epoch": 0.5401360544217687, + "grad_norm": 0.3612753450870514, + "learning_rate": 2.3157534246575343e-05, + "loss": 0.31891517639160155, + "mean_token_accuracy": 0.9155329942703248, + "num_tokens": 7646074.0, + "step": 3970 + }, + { + "entropy": 0.3787412360310555, + "epoch": 0.5414965986394558, + "grad_norm": 0.41047224402427673, + "learning_rate": 2.3089041095890412e-05, + "loss": 0.3822017192840576, + "mean_token_accuracy": 0.9003373116254807, + "num_tokens": 7664536.0, + "step": 3980 + }, + { + "entropy": 0.3361421704292297, + "epoch": 0.5428571428571428, + "grad_norm": 0.41327348351478577, + "learning_rate": 2.302054794520548e-05, + "loss": 0.3355587959289551, + "mean_token_accuracy": 0.911680155992508, + "num_tokens": 7682379.0, + "step": 3990 + }, + { + "entropy": 0.3021339297294617, + "epoch": 0.54421768707483, + "grad_norm": 0.36242055892944336, + "learning_rate": 2.295205479452055e-05, + "loss": 0.3072765111923218, + "mean_token_accuracy": 0.9183337926864624, + "num_tokens": 7701447.0, + "step": 4000 + }, + { + "epoch": 0.54421768707483, + "eval_entropy": 0.3300413696716229, + "eval_loss": 0.3298623263835907, + "eval_mean_token_accuracy": 0.9126938621203105, + "eval_num_tokens": 7701447.0, + "eval_runtime": 39.9379, + "eval_samples_per_second": 15.023, + "eval_steps_per_second": 7.512, + "step": 4000 + }, + { + "entropy": 0.30280276834964753, + "epoch": 0.545578231292517, + "grad_norm": 0.5451289415359497, + "learning_rate": 2.2883561643835617e-05, + "loss": 0.30348217487335205, + "mean_token_accuracy": 0.9159641265869141, + "num_tokens": 7723410.0, + "step": 4010 + }, + { + "entropy": 0.26205305345356467, + "epoch": 0.5469387755102041, + "grad_norm": 0.4319462776184082, + "learning_rate": 2.2815068493150683e-05, + "loss": 0.2545130252838135, + "mean_token_accuracy": 0.9308669567108154, + "num_tokens": 7743532.0, + "step": 4020 + }, + { + "entropy": 0.32944763377308844, + "epoch": 0.5482993197278911, + "grad_norm": 0.5175082087516785, + "learning_rate": 2.2746575342465753e-05, + "loss": 0.3209054946899414, + "mean_token_accuracy": 0.9140241533517838, + "num_tokens": 7764584.0, + "step": 4030 + }, + { + "entropy": 0.3470706418156624, + "epoch": 0.5496598639455782, + "grad_norm": 0.4611230790615082, + "learning_rate": 2.2678082191780822e-05, + "loss": 0.3442211151123047, + "mean_token_accuracy": 0.9116791754961013, + "num_tokens": 7783273.0, + "step": 4040 + }, + { + "entropy": 0.3467777229845524, + "epoch": 0.5510204081632653, + "grad_norm": 0.5174922943115234, + "learning_rate": 2.260958904109589e-05, + "loss": 0.3491501331329346, + "mean_token_accuracy": 0.9060128688812256, + "num_tokens": 7803308.0, + "step": 4050 + }, + { + "entropy": 0.3715716302394867, + "epoch": 0.5523809523809524, + "grad_norm": 0.5032371878623962, + "learning_rate": 2.254109589041096e-05, + "loss": 0.36512949466705324, + "mean_token_accuracy": 0.9032170832157135, + "num_tokens": 7823268.0, + "step": 4060 + }, + { + "entropy": 0.355232909321785, + "epoch": 0.5537414965986395, + "grad_norm": 0.3794972896575928, + "learning_rate": 2.2472602739726027e-05, + "loss": 0.3677065849304199, + "mean_token_accuracy": 0.9080590635538102, + "num_tokens": 7843048.0, + "step": 4070 + }, + { + "entropy": 0.3806071810424328, + "epoch": 0.5551020408163265, + "grad_norm": 0.398653507232666, + "learning_rate": 2.2404109589041097e-05, + "loss": 0.37912545204162595, + "mean_token_accuracy": 0.9007632732391357, + "num_tokens": 7862587.0, + "step": 4080 + }, + { + "entropy": 0.29905916675925254, + "epoch": 0.5564625850340136, + "grad_norm": 0.39064309000968933, + "learning_rate": 2.2335616438356166e-05, + "loss": 0.2870203971862793, + "mean_token_accuracy": 0.9261332839727402, + "num_tokens": 7880399.0, + "step": 4090 + }, + { + "entropy": 0.3375513479113579, + "epoch": 0.5578231292517006, + "grad_norm": 0.4097396433353424, + "learning_rate": 2.2267123287671236e-05, + "loss": 0.3322545289993286, + "mean_token_accuracy": 0.9148608386516571, + "num_tokens": 7901445.0, + "step": 4100 + }, + { + "entropy": 0.3285654172301292, + "epoch": 0.5591836734693878, + "grad_norm": 0.47398117184638977, + "learning_rate": 2.2198630136986305e-05, + "loss": 0.32115213871002196, + "mean_token_accuracy": 0.9161695718765259, + "num_tokens": 7919428.0, + "step": 4110 + }, + { + "entropy": 0.32259594202041625, + "epoch": 0.5605442176870749, + "grad_norm": 0.4475609362125397, + "learning_rate": 2.213013698630137e-05, + "loss": 0.32392292022705077, + "mean_token_accuracy": 0.9168900787830353, + "num_tokens": 7938761.0, + "step": 4120 + }, + { + "entropy": 0.32752382531762125, + "epoch": 0.5619047619047619, + "grad_norm": 0.41794857382774353, + "learning_rate": 2.2061643835616437e-05, + "loss": 0.3300853967666626, + "mean_token_accuracy": 0.911037164926529, + "num_tokens": 7957516.0, + "step": 4130 + }, + { + "entropy": 0.3652478508651257, + "epoch": 0.563265306122449, + "grad_norm": 0.4401056468486786, + "learning_rate": 2.1993150684931507e-05, + "loss": 0.3552700519561768, + "mean_token_accuracy": 0.9047274440526962, + "num_tokens": 7977608.0, + "step": 4140 + }, + { + "entropy": 0.343021310120821, + "epoch": 0.564625850340136, + "grad_norm": 0.46899110078811646, + "learning_rate": 2.1924657534246576e-05, + "loss": 0.34839396476745604, + "mean_token_accuracy": 0.9092651695013046, + "num_tokens": 7995943.0, + "step": 4150 + }, + { + "entropy": 0.3456800006330013, + "epoch": 0.5659863945578232, + "grad_norm": 0.5434921383857727, + "learning_rate": 2.1856164383561646e-05, + "loss": 0.3406283140182495, + "mean_token_accuracy": 0.9088764578104019, + "num_tokens": 8016350.0, + "step": 4160 + }, + { + "entropy": 0.3137536272406578, + "epoch": 0.5673469387755102, + "grad_norm": 0.5325484871864319, + "learning_rate": 2.1787671232876715e-05, + "loss": 0.3117046594619751, + "mean_token_accuracy": 0.9154979318380356, + "num_tokens": 8037048.0, + "step": 4170 + }, + { + "entropy": 0.3241760052740574, + "epoch": 0.5687074829931973, + "grad_norm": 0.3219553232192993, + "learning_rate": 2.171917808219178e-05, + "loss": 0.319587254524231, + "mean_token_accuracy": 0.9146761208772659, + "num_tokens": 8056300.0, + "step": 4180 + }, + { + "entropy": 0.2647711656987667, + "epoch": 0.5700680272108843, + "grad_norm": 0.5715179443359375, + "learning_rate": 2.165068493150685e-05, + "loss": 0.2673149585723877, + "mean_token_accuracy": 0.9255240142345429, + "num_tokens": 8074144.0, + "step": 4190 + }, + { + "entropy": 0.3415665104985237, + "epoch": 0.5714285714285714, + "grad_norm": 0.4954402446746826, + "learning_rate": 2.158219178082192e-05, + "loss": 0.3447671175003052, + "mean_token_accuracy": 0.9117257714271545, + "num_tokens": 8091291.0, + "step": 4200 + }, + { + "entropy": 0.30447668135166167, + "epoch": 0.5727891156462585, + "grad_norm": 0.3845031261444092, + "learning_rate": 2.151369863013699e-05, + "loss": 0.29765496253967283, + "mean_token_accuracy": 0.9206657618284225, + "num_tokens": 8109439.0, + "step": 4210 + }, + { + "entropy": 0.3095638811588287, + "epoch": 0.5741496598639456, + "grad_norm": 0.3968900740146637, + "learning_rate": 2.1445205479452056e-05, + "loss": 0.322363543510437, + "mean_token_accuracy": 0.9155390083789825, + "num_tokens": 8131053.0, + "step": 4220 + }, + { + "entropy": 0.33604917004704477, + "epoch": 0.5755102040816327, + "grad_norm": 0.45397162437438965, + "learning_rate": 2.1376712328767122e-05, + "loss": 0.33268921375274657, + "mean_token_accuracy": 0.9129331290721894, + "num_tokens": 8152572.0, + "step": 4230 + }, + { + "entropy": 0.3526121251285076, + "epoch": 0.5768707482993197, + "grad_norm": 0.3945638835430145, + "learning_rate": 2.130821917808219e-05, + "loss": 0.35298843383789064, + "mean_token_accuracy": 0.9088378429412842, + "num_tokens": 8171370.0, + "step": 4240 + }, + { + "entropy": 0.3636915013194084, + "epoch": 0.5782312925170068, + "grad_norm": 0.5967618823051453, + "learning_rate": 2.123972602739726e-05, + "loss": 0.37089223861694337, + "mean_token_accuracy": 0.9077885746955872, + "num_tokens": 8188389.0, + "step": 4250 + }, + { + "epoch": 0.5782312925170068, + "eval_entropy": 0.32578996996084847, + "eval_loss": 0.329336941242218, + "eval_mean_token_accuracy": 0.9131122646729152, + "eval_num_tokens": 8188389.0, + "eval_runtime": 38.822, + "eval_samples_per_second": 15.455, + "eval_steps_per_second": 7.728, + "step": 4250 + }, + { + "entropy": 0.31215301901102066, + "epoch": 0.5795918367346938, + "grad_norm": 0.4824981689453125, + "learning_rate": 2.117123287671233e-05, + "loss": 0.3266132831573486, + "mean_token_accuracy": 0.9147640526294708, + "num_tokens": 8206409.0, + "step": 4260 + }, + { + "entropy": 0.3209221847355366, + "epoch": 0.580952380952381, + "grad_norm": 0.6410835981369019, + "learning_rate": 2.11027397260274e-05, + "loss": 0.31994919776916503, + "mean_token_accuracy": 0.9160535246133804, + "num_tokens": 8226250.0, + "step": 4270 + }, + { + "entropy": 0.3772594600915909, + "epoch": 0.582312925170068, + "grad_norm": 0.4383021891117096, + "learning_rate": 2.1034246575342466e-05, + "loss": 0.3900163650512695, + "mean_token_accuracy": 0.9026447653770446, + "num_tokens": 8245881.0, + "step": 4280 + }, + { + "entropy": 0.30700850337743757, + "epoch": 0.5836734693877551, + "grad_norm": 0.4085005521774292, + "learning_rate": 2.0965753424657535e-05, + "loss": 0.30626041889190675, + "mean_token_accuracy": 0.9162427425384522, + "num_tokens": 8264812.0, + "step": 4290 + }, + { + "entropy": 0.3398926958441734, + "epoch": 0.5850340136054422, + "grad_norm": 0.5330114960670471, + "learning_rate": 2.0897260273972605e-05, + "loss": 0.34441685676574707, + "mean_token_accuracy": 0.9106619924306869, + "num_tokens": 8283010.0, + "step": 4300 + }, + { + "entropy": 0.35671548917889595, + "epoch": 0.5863945578231292, + "grad_norm": 0.45322227478027344, + "learning_rate": 2.0828767123287674e-05, + "loss": 0.3564306259155273, + "mean_token_accuracy": 0.9076927453279495, + "num_tokens": 8300835.0, + "step": 4310 + }, + { + "entropy": 0.37111207768321036, + "epoch": 0.5877551020408164, + "grad_norm": 0.4854358732700348, + "learning_rate": 2.076027397260274e-05, + "loss": 0.3827904939651489, + "mean_token_accuracy": 0.9036923080682755, + "num_tokens": 8320214.0, + "step": 4320 + }, + { + "entropy": 0.36895843744277956, + "epoch": 0.5891156462585034, + "grad_norm": 0.4703625738620758, + "learning_rate": 2.069178082191781e-05, + "loss": 0.3752447128295898, + "mean_token_accuracy": 0.9071332007646561, + "num_tokens": 8339775.0, + "step": 4330 + }, + { + "entropy": 0.39478201493620874, + "epoch": 0.5904761904761905, + "grad_norm": 0.4411693513393402, + "learning_rate": 2.0623287671232876e-05, + "loss": 0.3966845989227295, + "mean_token_accuracy": 0.8968111395835876, + "num_tokens": 8360604.0, + "step": 4340 + }, + { + "entropy": 0.41347152292728423, + "epoch": 0.5918367346938775, + "grad_norm": 0.47045063972473145, + "learning_rate": 2.0554794520547945e-05, + "loss": 0.41939425468444824, + "mean_token_accuracy": 0.8988784611225128, + "num_tokens": 8379152.0, + "step": 4350 + }, + { + "entropy": 0.28764098733663557, + "epoch": 0.5931972789115646, + "grad_norm": 0.4305931329727173, + "learning_rate": 2.0486301369863015e-05, + "loss": 0.2825281858444214, + "mean_token_accuracy": 0.9262870222330093, + "num_tokens": 8397421.0, + "step": 4360 + }, + { + "entropy": 0.3454031616449356, + "epoch": 0.5945578231292517, + "grad_norm": 0.5220937132835388, + "learning_rate": 2.0417808219178084e-05, + "loss": 0.3588573694229126, + "mean_token_accuracy": 0.9089827597141266, + "num_tokens": 8416839.0, + "step": 4370 + }, + { + "entropy": 0.3660218968987465, + "epoch": 0.5959183673469388, + "grad_norm": 0.8208481073379517, + "learning_rate": 2.0349315068493154e-05, + "loss": 0.36882529258728025, + "mean_token_accuracy": 0.9018866419792175, + "num_tokens": 8437932.0, + "step": 4380 + }, + { + "entropy": 0.32640354707837105, + "epoch": 0.5972789115646259, + "grad_norm": 0.7729905843734741, + "learning_rate": 2.028082191780822e-05, + "loss": 0.32541344165802, + "mean_token_accuracy": 0.9163854837417602, + "num_tokens": 8456033.0, + "step": 4390 + }, + { + "entropy": 0.31475903652608395, + "epoch": 0.5986394557823129, + "grad_norm": 0.49552851915359497, + "learning_rate": 2.021232876712329e-05, + "loss": 0.30246944427490235, + "mean_token_accuracy": 0.918095150589943, + "num_tokens": 8474315.0, + "step": 4400 + }, + { + "entropy": 0.3114713206887245, + "epoch": 0.6, + "grad_norm": 0.7524731755256653, + "learning_rate": 2.014383561643836e-05, + "loss": 0.306212329864502, + "mean_token_accuracy": 0.9181839942932128, + "num_tokens": 8493812.0, + "step": 4410 + }, + { + "entropy": 0.3414305582642555, + "epoch": 0.601360544217687, + "grad_norm": 0.4476776421070099, + "learning_rate": 2.0075342465753425e-05, + "loss": 0.3371569156646729, + "mean_token_accuracy": 0.9159269124269486, + "num_tokens": 8512284.0, + "step": 4420 + }, + { + "entropy": 0.2988259807229042, + "epoch": 0.6027210884353742, + "grad_norm": 0.5182522535324097, + "learning_rate": 2.0006849315068494e-05, + "loss": 0.31771996021270754, + "mean_token_accuracy": 0.9163232058286667, + "num_tokens": 8531033.0, + "step": 4430 + }, + { + "entropy": 0.3422327384352684, + "epoch": 0.6040816326530613, + "grad_norm": 0.4643579125404358, + "learning_rate": 1.993835616438356e-05, + "loss": 0.3435725212097168, + "mean_token_accuracy": 0.9077637434005738, + "num_tokens": 8550689.0, + "step": 4440 + }, + { + "entropy": 0.3316516526043415, + "epoch": 0.6054421768707483, + "grad_norm": 0.5027662515640259, + "learning_rate": 1.986986301369863e-05, + "loss": 0.3284109592437744, + "mean_token_accuracy": 0.9112151861190796, + "num_tokens": 8571488.0, + "step": 4450 + }, + { + "entropy": 0.368307001888752, + "epoch": 0.6068027210884354, + "grad_norm": 0.5623902678489685, + "learning_rate": 1.98013698630137e-05, + "loss": 0.36309108734130857, + "mean_token_accuracy": 0.9045984745025635, + "num_tokens": 8591756.0, + "step": 4460 + }, + { + "entropy": 0.31291708797216417, + "epoch": 0.6081632653061224, + "grad_norm": 0.4109684228897095, + "learning_rate": 1.973287671232877e-05, + "loss": 0.32405858039855956, + "mean_token_accuracy": 0.9153440564870834, + "num_tokens": 8610911.0, + "step": 4470 + }, + { + "entropy": 0.3248316764831543, + "epoch": 0.6095238095238096, + "grad_norm": 0.3668299913406372, + "learning_rate": 1.966438356164384e-05, + "loss": 0.3182776212692261, + "mean_token_accuracy": 0.9126608431339264, + "num_tokens": 8629955.0, + "step": 4480 + }, + { + "entropy": 0.3336681894958019, + "epoch": 0.6108843537414966, + "grad_norm": 0.5103767514228821, + "learning_rate": 1.9595890410958904e-05, + "loss": 0.3319364547729492, + "mean_token_accuracy": 0.9127811759710311, + "num_tokens": 8647894.0, + "step": 4490 + }, + { + "entropy": 0.31860669925808904, + "epoch": 0.6122448979591837, + "grad_norm": 0.4623521566390991, + "learning_rate": 1.9527397260273974e-05, + "loss": 0.3230354070663452, + "mean_token_accuracy": 0.9166040539741516, + "num_tokens": 8667198.0, + "step": 4500 + }, + { + "epoch": 0.6122448979591837, + "eval_entropy": 0.3310739654302597, + "eval_loss": 0.3289305567741394, + "eval_mean_token_accuracy": 0.9130673670768737, + "eval_num_tokens": 8667198.0, + "eval_runtime": 38.7549, + "eval_samples_per_second": 15.482, + "eval_steps_per_second": 7.741, + "step": 4500 + }, + { + "entropy": 0.3124507375061512, + "epoch": 0.6136054421768707, + "grad_norm": 0.385375052690506, + "learning_rate": 1.945890410958904e-05, + "loss": 0.3219736337661743, + "mean_token_accuracy": 0.9118276238441467, + "num_tokens": 8690256.0, + "step": 4510 + }, + { + "entropy": 0.3162266179919243, + "epoch": 0.6149659863945578, + "grad_norm": 0.3542030155658722, + "learning_rate": 1.939041095890411e-05, + "loss": 0.3027017116546631, + "mean_token_accuracy": 0.9203230768442154, + "num_tokens": 8708888.0, + "step": 4520 + }, + { + "entropy": 0.3298695728182793, + "epoch": 0.6163265306122448, + "grad_norm": 0.47942066192626953, + "learning_rate": 1.932191780821918e-05, + "loss": 0.33888137340545654, + "mean_token_accuracy": 0.9136987268924713, + "num_tokens": 8727333.0, + "step": 4530 + }, + { + "entropy": 0.2821711003780365, + "epoch": 0.617687074829932, + "grad_norm": 0.48402971029281616, + "learning_rate": 1.925342465753425e-05, + "loss": 0.2859959125518799, + "mean_token_accuracy": 0.9232243895530701, + "num_tokens": 8747157.0, + "step": 4540 + }, + { + "entropy": 0.2933730438351631, + "epoch": 0.6190476190476191, + "grad_norm": 0.5369724035263062, + "learning_rate": 1.9184931506849314e-05, + "loss": 0.28847010135650636, + "mean_token_accuracy": 0.9202695101499557, + "num_tokens": 8765416.0, + "step": 4550 + }, + { + "entropy": 0.30423177406191826, + "epoch": 0.6204081632653061, + "grad_norm": 0.3822837173938751, + "learning_rate": 1.9116438356164384e-05, + "loss": 0.3026056528091431, + "mean_token_accuracy": 0.9191194862127304, + "num_tokens": 8782981.0, + "step": 4560 + }, + { + "entropy": 0.35051786080002784, + "epoch": 0.6217687074829932, + "grad_norm": 0.403596431016922, + "learning_rate": 1.9047945205479453e-05, + "loss": 0.34621405601501465, + "mean_token_accuracy": 0.911699402332306, + "num_tokens": 8802808.0, + "step": 4570 + }, + { + "entropy": 0.30863510593771937, + "epoch": 0.6231292517006802, + "grad_norm": 0.38001856207847595, + "learning_rate": 1.8979452054794523e-05, + "loss": 0.3103233575820923, + "mean_token_accuracy": 0.9180112898349762, + "num_tokens": 8822863.0, + "step": 4580 + }, + { + "entropy": 0.2945192024111748, + "epoch": 0.6244897959183674, + "grad_norm": 0.5920354127883911, + "learning_rate": 1.8910958904109592e-05, + "loss": 0.29104783535003664, + "mean_token_accuracy": 0.9209440588951111, + "num_tokens": 8843360.0, + "step": 4590 + }, + { + "entropy": 0.3656765803694725, + "epoch": 0.6258503401360545, + "grad_norm": 0.4458155035972595, + "learning_rate": 1.884246575342466e-05, + "loss": 0.37093372344970704, + "mean_token_accuracy": 0.904807910323143, + "num_tokens": 8863752.0, + "step": 4600 + }, + { + "entropy": 0.3120325908064842, + "epoch": 0.6272108843537415, + "grad_norm": 0.45716285705566406, + "learning_rate": 1.8773972602739725e-05, + "loss": 0.3167175054550171, + "mean_token_accuracy": 0.9172279864549637, + "num_tokens": 8883642.0, + "step": 4610 + }, + { + "entropy": 0.32725323736667633, + "epoch": 0.6285714285714286, + "grad_norm": 0.5018922686576843, + "learning_rate": 1.8705479452054794e-05, + "loss": 0.3386115074157715, + "mean_token_accuracy": 0.9148703008890152, + "num_tokens": 8903406.0, + "step": 4620 + }, + { + "entropy": 0.3152094043791294, + "epoch": 0.6299319727891156, + "grad_norm": 0.49038684368133545, + "learning_rate": 1.8636986301369863e-05, + "loss": 0.31165380477905275, + "mean_token_accuracy": 0.9160630106925964, + "num_tokens": 8922781.0, + "step": 4630 + }, + { + "entropy": 0.2654181182384491, + "epoch": 0.6312925170068027, + "grad_norm": 0.499210000038147, + "learning_rate": 1.8568493150684933e-05, + "loss": 0.25337116718292235, + "mean_token_accuracy": 0.9268150001764297, + "num_tokens": 8940811.0, + "step": 4640 + }, + { + "entropy": 0.33546371310949324, + "epoch": 0.6326530612244898, + "grad_norm": 0.5591289401054382, + "learning_rate": 1.85e-05, + "loss": 0.3524590015411377, + "mean_token_accuracy": 0.9123670727014541, + "num_tokens": 8959421.0, + "step": 4650 + }, + { + "entropy": 0.30375867560505865, + "epoch": 0.6340136054421769, + "grad_norm": 0.657088041305542, + "learning_rate": 1.843150684931507e-05, + "loss": 0.2986541509628296, + "mean_token_accuracy": 0.9188253223896027, + "num_tokens": 8978608.0, + "step": 4660 + }, + { + "entropy": 0.29843793213367464, + "epoch": 0.6353741496598639, + "grad_norm": 0.5112680196762085, + "learning_rate": 1.8363013698630138e-05, + "loss": 0.2931118726730347, + "mean_token_accuracy": 0.9223059266805649, + "num_tokens": 8996614.0, + "step": 4670 + }, + { + "entropy": 0.33378486707806587, + "epoch": 0.636734693877551, + "grad_norm": 0.437621533870697, + "learning_rate": 1.8294520547945207e-05, + "loss": 0.32436633110046387, + "mean_token_accuracy": 0.9149109095335006, + "num_tokens": 9017250.0, + "step": 4680 + }, + { + "entropy": 0.2954431213438511, + "epoch": 0.638095238095238, + "grad_norm": 0.626248300075531, + "learning_rate": 1.8226027397260277e-05, + "loss": 0.3086385250091553, + "mean_token_accuracy": 0.9184481203556061, + "num_tokens": 9038262.0, + "step": 4690 + }, + { + "entropy": 0.29330349788069726, + "epoch": 0.6394557823129252, + "grad_norm": 0.5271269679069519, + "learning_rate": 1.8157534246575343e-05, + "loss": 0.2987428426742554, + "mean_token_accuracy": 0.9220951855182647, + "num_tokens": 9059039.0, + "step": 4700 + }, + { + "entropy": 0.3254406459629536, + "epoch": 0.6408163265306123, + "grad_norm": 0.38454508781433105, + "learning_rate": 1.808904109589041e-05, + "loss": 0.3387522459030151, + "mean_token_accuracy": 0.9124308109283448, + "num_tokens": 9077519.0, + "step": 4710 + }, + { + "entropy": 0.3163925364613533, + "epoch": 0.6421768707482993, + "grad_norm": 0.5404890775680542, + "learning_rate": 1.802054794520548e-05, + "loss": 0.3156402587890625, + "mean_token_accuracy": 0.914623373746872, + "num_tokens": 9097280.0, + "step": 4720 + }, + { + "entropy": 0.3105604760348797, + "epoch": 0.6435374149659864, + "grad_norm": 0.38867318630218506, + "learning_rate": 1.7952054794520548e-05, + "loss": 0.3073798656463623, + "mean_token_accuracy": 0.9193964332342148, + "num_tokens": 9116868.0, + "step": 4730 + }, + { + "entropy": 0.34601864591240883, + "epoch": 0.6448979591836734, + "grad_norm": 0.33544185757637024, + "learning_rate": 1.7883561643835618e-05, + "loss": 0.3442887783050537, + "mean_token_accuracy": 0.9118550598621369, + "num_tokens": 9135871.0, + "step": 4740 + }, + { + "entropy": 0.3289314046502113, + "epoch": 0.6462585034013606, + "grad_norm": 0.7844065427780151, + "learning_rate": 1.7815068493150687e-05, + "loss": 0.3229420900344849, + "mean_token_accuracy": 0.9158719092607498, + "num_tokens": 9155805.0, + "step": 4750 + }, + { + "epoch": 0.6462585034013606, + "eval_entropy": 0.33255102773507433, + "eval_loss": 0.32763543725013733, + "eval_mean_token_accuracy": 0.9132404702901841, + "eval_num_tokens": 9155805.0, + "eval_runtime": 39.9686, + "eval_samples_per_second": 15.012, + "eval_steps_per_second": 7.506, + "step": 4750 + }, + { + "entropy": 0.32009952403604985, + "epoch": 0.6476190476190476, + "grad_norm": 0.4704689085483551, + "learning_rate": 1.7746575342465753e-05, + "loss": 0.31637964248657224, + "mean_token_accuracy": 0.9144832044839859, + "num_tokens": 9177273.0, + "step": 4760 + }, + { + "entropy": 0.28996511846780776, + "epoch": 0.6489795918367347, + "grad_norm": 0.41101402044296265, + "learning_rate": 1.7678082191780823e-05, + "loss": 0.285528564453125, + "mean_token_accuracy": 0.92576684653759, + "num_tokens": 9197118.0, + "step": 4770 + }, + { + "entropy": 0.3418573744595051, + "epoch": 0.6503401360544218, + "grad_norm": 0.5408941507339478, + "learning_rate": 1.7609589041095892e-05, + "loss": 0.3393819808959961, + "mean_token_accuracy": 0.9118351727724076, + "num_tokens": 9216516.0, + "step": 4780 + }, + { + "entropy": 0.28226210810244085, + "epoch": 0.6517006802721088, + "grad_norm": 0.3848341703414917, + "learning_rate": 1.754109589041096e-05, + "loss": 0.2931527614593506, + "mean_token_accuracy": 0.9224736660718917, + "num_tokens": 9234639.0, + "step": 4790 + }, + { + "entropy": 0.2693475380539894, + "epoch": 0.6530612244897959, + "grad_norm": 0.6227667331695557, + "learning_rate": 1.747260273972603e-05, + "loss": 0.2764613389968872, + "mean_token_accuracy": 0.9238964438438415, + "num_tokens": 9254926.0, + "step": 4800 + }, + { + "entropy": 0.328694748878479, + "epoch": 0.654421768707483, + "grad_norm": 0.4423985183238983, + "learning_rate": 1.7404109589041097e-05, + "loss": 0.3174917459487915, + "mean_token_accuracy": 0.9097775369882584, + "num_tokens": 9276122.0, + "step": 4810 + }, + { + "entropy": 0.37669333517551423, + "epoch": 0.6557823129251701, + "grad_norm": 0.5443205237388611, + "learning_rate": 1.7335616438356163e-05, + "loss": 0.3813244581222534, + "mean_token_accuracy": 0.9008943438529968, + "num_tokens": 9295112.0, + "step": 4820 + }, + { + "entropy": 0.3570864848792553, + "epoch": 0.6571428571428571, + "grad_norm": 0.539834201335907, + "learning_rate": 1.7267123287671233e-05, + "loss": 0.36410791873931886, + "mean_token_accuracy": 0.9081277132034302, + "num_tokens": 9314161.0, + "step": 4830 + }, + { + "entropy": 0.31993206292390824, + "epoch": 0.6585034013605442, + "grad_norm": 0.3892541825771332, + "learning_rate": 1.7198630136986302e-05, + "loss": 0.29779579639434817, + "mean_token_accuracy": 0.9220435619354248, + "num_tokens": 9332900.0, + "step": 4840 + }, + { + "entropy": 0.2923553496599197, + "epoch": 0.6598639455782312, + "grad_norm": 0.4788413643836975, + "learning_rate": 1.713013698630137e-05, + "loss": 0.29410948753356936, + "mean_token_accuracy": 0.9204231530427933, + "num_tokens": 9351463.0, + "step": 4850 + }, + { + "entropy": 0.28082970082759856, + "epoch": 0.6612244897959184, + "grad_norm": 0.4797994792461395, + "learning_rate": 1.7061643835616438e-05, + "loss": 0.291374945640564, + "mean_token_accuracy": 0.9210595309734344, + "num_tokens": 9370433.0, + "step": 4860 + }, + { + "entropy": 0.3138731438666582, + "epoch": 0.6625850340136055, + "grad_norm": 0.4365018606185913, + "learning_rate": 1.6993150684931507e-05, + "loss": 0.32789273262023927, + "mean_token_accuracy": 0.9142170727252961, + "num_tokens": 9390262.0, + "step": 4870 + }, + { + "entropy": 0.29280120581388475, + "epoch": 0.6639455782312925, + "grad_norm": 0.5428627729415894, + "learning_rate": 1.6924657534246577e-05, + "loss": 0.27776288986206055, + "mean_token_accuracy": 0.9247405260801316, + "num_tokens": 9408584.0, + "step": 4880 + }, + { + "entropy": 0.32643493488430975, + "epoch": 0.6653061224489796, + "grad_norm": 0.5670334100723267, + "learning_rate": 1.6856164383561646e-05, + "loss": 0.3300167560577393, + "mean_token_accuracy": 0.9129970610141754, + "num_tokens": 9427236.0, + "step": 4890 + }, + { + "entropy": 0.3827863521873951, + "epoch": 0.6666666666666666, + "grad_norm": 0.41381803154945374, + "learning_rate": 1.6787671232876716e-05, + "loss": 0.40310182571411135, + "mean_token_accuracy": 0.899763336777687, + "num_tokens": 9444771.0, + "step": 4900 + }, + { + "entropy": 0.32538103610277175, + "epoch": 0.6680272108843538, + "grad_norm": 0.43814322352409363, + "learning_rate": 1.671917808219178e-05, + "loss": 0.33401734828948976, + "mean_token_accuracy": 0.9110633939504623, + "num_tokens": 9463864.0, + "step": 4910 + }, + { + "entropy": 0.24819178245961665, + "epoch": 0.6693877551020408, + "grad_norm": 0.484448105096817, + "learning_rate": 1.6650684931506848e-05, + "loss": 0.23955152034759522, + "mean_token_accuracy": 0.933735242486, + "num_tokens": 9481457.0, + "step": 4920 + }, + { + "entropy": 0.33870874494314196, + "epoch": 0.6707482993197279, + "grad_norm": 0.7475461363792419, + "learning_rate": 1.6582191780821917e-05, + "loss": 0.3345521926879883, + "mean_token_accuracy": 0.9089796751737594, + "num_tokens": 9499700.0, + "step": 4930 + }, + { + "entropy": 0.28814957924187184, + "epoch": 0.672108843537415, + "grad_norm": 0.6497180461883545, + "learning_rate": 1.6513698630136987e-05, + "loss": 0.29244041442871094, + "mean_token_accuracy": 0.9212163001298904, + "num_tokens": 9518473.0, + "step": 4940 + }, + { + "entropy": 0.3113638870418072, + "epoch": 0.673469387755102, + "grad_norm": 0.6817547082901001, + "learning_rate": 1.6445205479452056e-05, + "loss": 0.3035978555679321, + "mean_token_accuracy": 0.9179104954004288, + "num_tokens": 9537747.0, + "step": 4950 + }, + { + "entropy": 0.3260213859379292, + "epoch": 0.6748299319727891, + "grad_norm": 0.5817997455596924, + "learning_rate": 1.6376712328767126e-05, + "loss": 0.3352161407470703, + "mean_token_accuracy": 0.9155428946018219, + "num_tokens": 9556544.0, + "step": 4960 + }, + { + "entropy": 0.32320692837238313, + "epoch": 0.6761904761904762, + "grad_norm": 0.40952223539352417, + "learning_rate": 1.6308219178082192e-05, + "loss": 0.3208148717880249, + "mean_token_accuracy": 0.9158533960580826, + "num_tokens": 9574599.0, + "step": 4970 + }, + { + "entropy": 0.32987525388598443, + "epoch": 0.6775510204081633, + "grad_norm": 0.38160601258277893, + "learning_rate": 1.623972602739726e-05, + "loss": 0.3314782381057739, + "mean_token_accuracy": 0.9137853533029556, + "num_tokens": 9594134.0, + "step": 4980 + }, + { + "entropy": 0.3361847847700119, + "epoch": 0.6789115646258503, + "grad_norm": 0.5430313348770142, + "learning_rate": 1.617123287671233e-05, + "loss": 0.3365447998046875, + "mean_token_accuracy": 0.9137306600809098, + "num_tokens": 9612548.0, + "step": 4990 + }, + { + "entropy": 0.30371019840240476, + "epoch": 0.6802721088435374, + "grad_norm": 0.5549236536026001, + "learning_rate": 1.61027397260274e-05, + "loss": 0.30310888290405275, + "mean_token_accuracy": 0.9160631775856019, + "num_tokens": 9631640.0, + "step": 5000 + }, + { + "epoch": 0.6802721088435374, + "eval_entropy": 0.3331662664314111, + "eval_loss": 0.32677820324897766, + "eval_mean_token_accuracy": 0.9136480834086737, + "eval_num_tokens": 9631640.0, + "eval_runtime": 39.708, + "eval_samples_per_second": 15.11, + "eval_steps_per_second": 7.555, + "step": 5000 + }, + { + "entropy": 0.3210461221635342, + "epoch": 0.6816326530612244, + "grad_norm": 0.6082227230072021, + "learning_rate": 1.6034246575342466e-05, + "loss": 0.3225013256072998, + "mean_token_accuracy": 0.9164930671453476, + "num_tokens": 9650167.0, + "step": 5010 + }, + { + "entropy": 0.33073775470256805, + "epoch": 0.6829931972789116, + "grad_norm": 0.4650852084159851, + "learning_rate": 1.5965753424657536e-05, + "loss": 0.32491614818573, + "mean_token_accuracy": 0.9105731517076492, + "num_tokens": 9669888.0, + "step": 5020 + }, + { + "entropy": 0.2889856085181236, + "epoch": 0.6843537414965987, + "grad_norm": 0.4409579038619995, + "learning_rate": 1.5897260273972602e-05, + "loss": 0.2879915237426758, + "mean_token_accuracy": 0.9247685641050338, + "num_tokens": 9686802.0, + "step": 5030 + }, + { + "entropy": 0.30220348536968233, + "epoch": 0.6857142857142857, + "grad_norm": 0.4094906747341156, + "learning_rate": 1.582876712328767e-05, + "loss": 0.2927564144134521, + "mean_token_accuracy": 0.921038344502449, + "num_tokens": 9705587.0, + "step": 5040 + }, + { + "entropy": 0.30619596429169177, + "epoch": 0.6870748299319728, + "grad_norm": 0.44534412026405334, + "learning_rate": 1.576027397260274e-05, + "loss": 0.30527758598327637, + "mean_token_accuracy": 0.9183918595314026, + "num_tokens": 9723365.0, + "step": 5050 + }, + { + "entropy": 0.27247815728187563, + "epoch": 0.6884353741496598, + "grad_norm": 0.38370785117149353, + "learning_rate": 1.569178082191781e-05, + "loss": 0.28004071712493894, + "mean_token_accuracy": 0.9289819926023484, + "num_tokens": 9741522.0, + "step": 5060 + }, + { + "entropy": 0.3562318213284016, + "epoch": 0.689795918367347, + "grad_norm": 0.42891645431518555, + "learning_rate": 1.5623287671232876e-05, + "loss": 0.3679760456085205, + "mean_token_accuracy": 0.906402376294136, + "num_tokens": 9761886.0, + "step": 5070 + }, + { + "entropy": 0.30469153448939323, + "epoch": 0.691156462585034, + "grad_norm": 0.4258774518966675, + "learning_rate": 1.5554794520547946e-05, + "loss": 0.2953490734100342, + "mean_token_accuracy": 0.9204110503196716, + "num_tokens": 9782113.0, + "step": 5080 + }, + { + "entropy": 0.3751088730990887, + "epoch": 0.6925170068027211, + "grad_norm": 0.43709659576416016, + "learning_rate": 1.5486301369863015e-05, + "loss": 0.3805356740951538, + "mean_token_accuracy": 0.9013684421777726, + "num_tokens": 9801414.0, + "step": 5090 + }, + { + "entropy": 0.3188751734793186, + "epoch": 0.6938775510204082, + "grad_norm": 0.4579872488975525, + "learning_rate": 1.5417808219178085e-05, + "loss": 0.3172019958496094, + "mean_token_accuracy": 0.9144055992364883, + "num_tokens": 9821329.0, + "step": 5100 + }, + { + "entropy": 0.3659577779471874, + "epoch": 0.6952380952380952, + "grad_norm": 0.5475864410400391, + "learning_rate": 1.534931506849315e-05, + "loss": 0.35874128341674805, + "mean_token_accuracy": 0.9058806359767914, + "num_tokens": 9840983.0, + "step": 5110 + }, + { + "entropy": 0.3272984899580479, + "epoch": 0.6965986394557823, + "grad_norm": 0.5433106422424316, + "learning_rate": 1.528082191780822e-05, + "loss": 0.320189380645752, + "mean_token_accuracy": 0.9151940882205963, + "num_tokens": 9861435.0, + "step": 5120 + }, + { + "entropy": 0.3580063499510288, + "epoch": 0.6979591836734694, + "grad_norm": 0.37401801347732544, + "learning_rate": 1.5212328767123288e-05, + "loss": 0.37555441856384275, + "mean_token_accuracy": 0.904027909040451, + "num_tokens": 9881061.0, + "step": 5130 + }, + { + "entropy": 0.3050012022256851, + "epoch": 0.6993197278911565, + "grad_norm": 0.5558372139930725, + "learning_rate": 1.5143835616438356e-05, + "loss": 0.3010509252548218, + "mean_token_accuracy": 0.9206721186637878, + "num_tokens": 9898420.0, + "step": 5140 + }, + { + "entropy": 0.3440397623926401, + "epoch": 0.7006802721088435, + "grad_norm": 0.5558260083198547, + "learning_rate": 1.5075342465753425e-05, + "loss": 0.32512967586517333, + "mean_token_accuracy": 0.9161503165960312, + "num_tokens": 9916386.0, + "step": 5150 + }, + { + "entropy": 0.32052273452281954, + "epoch": 0.7020408163265306, + "grad_norm": 0.42511069774627686, + "learning_rate": 1.5006849315068495e-05, + "loss": 0.31507837772369385, + "mean_token_accuracy": 0.9184840559959412, + "num_tokens": 9935862.0, + "step": 5160 + }, + { + "entropy": 0.3130710139870644, + "epoch": 0.7034013605442176, + "grad_norm": 0.48433229327201843, + "learning_rate": 1.4938356164383563e-05, + "loss": 0.3007185459136963, + "mean_token_accuracy": 0.9166130065917969, + "num_tokens": 9956146.0, + "step": 5170 + }, + { + "entropy": 0.34180518612265587, + "epoch": 0.7047619047619048, + "grad_norm": 0.5500788688659668, + "learning_rate": 1.4869863013698632e-05, + "loss": 0.3480189323425293, + "mean_token_accuracy": 0.9084232956171036, + "num_tokens": 9975283.0, + "step": 5180 + }, + { + "entropy": 0.26845929883420466, + "epoch": 0.7061224489795919, + "grad_norm": 0.4044052064418793, + "learning_rate": 1.48013698630137e-05, + "loss": 0.2679526090621948, + "mean_token_accuracy": 0.9261024564504623, + "num_tokens": 9993710.0, + "step": 5190 + }, + { + "entropy": 0.3100937880575657, + "epoch": 0.7074829931972789, + "grad_norm": 0.3828263282775879, + "learning_rate": 1.473287671232877e-05, + "loss": 0.3154742956161499, + "mean_token_accuracy": 0.9174116760492325, + "num_tokens": 10013864.0, + "step": 5200 + }, + { + "entropy": 0.3618170410394669, + "epoch": 0.708843537414966, + "grad_norm": 0.40374597907066345, + "learning_rate": 1.4664383561643835e-05, + "loss": 0.36072003841400146, + "mean_token_accuracy": 0.9016565412282944, + "num_tokens": 10034656.0, + "step": 5210 + }, + { + "entropy": 0.3923501305282116, + "epoch": 0.710204081632653, + "grad_norm": 0.5628812313079834, + "learning_rate": 1.4595890410958903e-05, + "loss": 0.4183984279632568, + "mean_token_accuracy": 0.900890639424324, + "num_tokens": 10053481.0, + "step": 5220 + }, + { + "entropy": 0.3111284948885441, + "epoch": 0.7115646258503401, + "grad_norm": 0.5592334270477295, + "learning_rate": 1.4527397260273973e-05, + "loss": 0.3107800006866455, + "mean_token_accuracy": 0.9183153718709945, + "num_tokens": 10070985.0, + "step": 5230 + }, + { + "entropy": 0.264003212377429, + "epoch": 0.7129251700680272, + "grad_norm": 0.3325921595096588, + "learning_rate": 1.4458904109589042e-05, + "loss": 0.2693720817565918, + "mean_token_accuracy": 0.9262395352125168, + "num_tokens": 10092029.0, + "step": 5240 + }, + { + "entropy": 0.30503557771444323, + "epoch": 0.7142857142857143, + "grad_norm": 0.6577172875404358, + "learning_rate": 1.439041095890411e-05, + "loss": 0.31351213455200194, + "mean_token_accuracy": 0.9185102015733719, + "num_tokens": 10111599.0, + "step": 5250 + }, + { + "epoch": 0.7142857142857143, + "eval_entropy": 0.3265386664867401, + "eval_loss": 0.32589972019195557, + "eval_mean_token_accuracy": 0.913812648455302, + "eval_num_tokens": 10111599.0, + "eval_runtime": 40.0077, + "eval_samples_per_second": 14.997, + "eval_steps_per_second": 7.499, + "step": 5250 + }, + { + "entropy": 0.32656988948583604, + "epoch": 0.7156462585034014, + "grad_norm": 0.45740580558776855, + "learning_rate": 1.432191780821918e-05, + "loss": 0.3337229251861572, + "mean_token_accuracy": 0.913157707452774, + "num_tokens": 10130030.0, + "step": 5260 + }, + { + "entropy": 0.332012340426445, + "epoch": 0.7170068027210884, + "grad_norm": 0.4293174147605896, + "learning_rate": 1.4253424657534247e-05, + "loss": 0.33874466419219973, + "mean_token_accuracy": 0.9121835470199585, + "num_tokens": 10150708.0, + "step": 5270 + }, + { + "entropy": 0.3409136638045311, + "epoch": 0.7183673469387755, + "grad_norm": 0.6065399646759033, + "learning_rate": 1.4184931506849317e-05, + "loss": 0.31609444618225097, + "mean_token_accuracy": 0.9152237951755524, + "num_tokens": 10170436.0, + "step": 5280 + }, + { + "entropy": 0.3278017446398735, + "epoch": 0.7197278911564626, + "grad_norm": 0.5555611848831177, + "learning_rate": 1.4116438356164386e-05, + "loss": 0.3270564556121826, + "mean_token_accuracy": 0.9139165192842483, + "num_tokens": 10188702.0, + "step": 5290 + }, + { + "entropy": 0.29700450971722603, + "epoch": 0.7210884353741497, + "grad_norm": 0.5627049207687378, + "learning_rate": 1.4047945205479452e-05, + "loss": 0.29241275787353516, + "mean_token_accuracy": 0.9209558874368667, + "num_tokens": 10208040.0, + "step": 5300 + }, + { + "entropy": 0.30556924641132355, + "epoch": 0.7224489795918367, + "grad_norm": 0.4673140048980713, + "learning_rate": 1.397945205479452e-05, + "loss": 0.30704848766326903, + "mean_token_accuracy": 0.9185676604509354, + "num_tokens": 10225913.0, + "step": 5310 + }, + { + "entropy": 0.3303392879664898, + "epoch": 0.7238095238095238, + "grad_norm": 0.4551662504673004, + "learning_rate": 1.391095890410959e-05, + "loss": 0.34713623523712156, + "mean_token_accuracy": 0.9099254727363586, + "num_tokens": 10243438.0, + "step": 5320 + }, + { + "entropy": 0.3365324631333351, + "epoch": 0.7251700680272108, + "grad_norm": 0.4157435894012451, + "learning_rate": 1.3842465753424657e-05, + "loss": 0.3378427505493164, + "mean_token_accuracy": 0.909062597155571, + "num_tokens": 10262805.0, + "step": 5330 + }, + { + "entropy": 0.32411291301250456, + "epoch": 0.726530612244898, + "grad_norm": 0.38107967376708984, + "learning_rate": 1.3773972602739727e-05, + "loss": 0.3202343225479126, + "mean_token_accuracy": 0.9115140378475189, + "num_tokens": 10282508.0, + "step": 5340 + }, + { + "entropy": 0.3038345813751221, + "epoch": 0.7278911564625851, + "grad_norm": 0.4391093850135803, + "learning_rate": 1.3705479452054794e-05, + "loss": 0.29564452171325684, + "mean_token_accuracy": 0.9200590491294861, + "num_tokens": 10303702.0, + "step": 5350 + }, + { + "entropy": 0.30474315136671065, + "epoch": 0.7292517006802721, + "grad_norm": 0.3891599178314209, + "learning_rate": 1.3636986301369864e-05, + "loss": 0.3134526491165161, + "mean_token_accuracy": 0.920561146736145, + "num_tokens": 10321825.0, + "step": 5360 + }, + { + "entropy": 0.3209965944290161, + "epoch": 0.7306122448979592, + "grad_norm": 0.6909648180007935, + "learning_rate": 1.3568493150684933e-05, + "loss": 0.33262181282043457, + "mean_token_accuracy": 0.9147113114595413, + "num_tokens": 10343513.0, + "step": 5370 + }, + { + "entropy": 0.2940310426056385, + "epoch": 0.7319727891156462, + "grad_norm": 0.3770340085029602, + "learning_rate": 1.3500000000000001e-05, + "loss": 0.2746826887130737, + "mean_token_accuracy": 0.9232329875230789, + "num_tokens": 10363756.0, + "step": 5380 + }, + { + "entropy": 0.3722431115806103, + "epoch": 0.7333333333333333, + "grad_norm": 0.5671103596687317, + "learning_rate": 1.343150684931507e-05, + "loss": 0.3857078790664673, + "mean_token_accuracy": 0.9042104989290237, + "num_tokens": 10382027.0, + "step": 5390 + }, + { + "entropy": 0.3080087319016457, + "epoch": 0.7346938775510204, + "grad_norm": 0.4369518458843231, + "learning_rate": 1.3363013698630137e-05, + "loss": 0.3244903802871704, + "mean_token_accuracy": 0.9169433772563934, + "num_tokens": 10401036.0, + "step": 5400 + }, + { + "entropy": 0.30762851536273955, + "epoch": 0.7360544217687075, + "grad_norm": 0.42277780175209045, + "learning_rate": 1.3294520547945205e-05, + "loss": 0.3111214876174927, + "mean_token_accuracy": 0.920467808842659, + "num_tokens": 10420343.0, + "step": 5410 + }, + { + "entropy": 0.2915546834468842, + "epoch": 0.7374149659863946, + "grad_norm": 0.6937938928604126, + "learning_rate": 1.3226027397260274e-05, + "loss": 0.29332144260406495, + "mean_token_accuracy": 0.928145581483841, + "num_tokens": 10435914.0, + "step": 5420 + }, + { + "entropy": 0.3241829499602318, + "epoch": 0.7387755102040816, + "grad_norm": 0.28303202986717224, + "learning_rate": 1.3157534246575342e-05, + "loss": 0.3249546527862549, + "mean_token_accuracy": 0.9146370053291321, + "num_tokens": 10456518.0, + "step": 5430 + }, + { + "entropy": 0.27756612598896024, + "epoch": 0.7401360544217687, + "grad_norm": 0.4878402054309845, + "learning_rate": 1.3089041095890411e-05, + "loss": 0.281647253036499, + "mean_token_accuracy": 0.9224755674600601, + "num_tokens": 10477480.0, + "step": 5440 + }, + { + "entropy": 0.31852116584777834, + "epoch": 0.7414965986394558, + "grad_norm": 0.5137626528739929, + "learning_rate": 1.302054794520548e-05, + "loss": 0.31983242034912107, + "mean_token_accuracy": 0.9154018461704254, + "num_tokens": 10496416.0, + "step": 5450 + }, + { + "entropy": 0.3167009152472019, + "epoch": 0.7428571428571429, + "grad_norm": 0.6255688667297363, + "learning_rate": 1.2952054794520549e-05, + "loss": 0.3200571060180664, + "mean_token_accuracy": 0.9154588907957077, + "num_tokens": 10515961.0, + "step": 5460 + }, + { + "entropy": 0.375573131069541, + "epoch": 0.7442176870748299, + "grad_norm": 0.42178791761398315, + "learning_rate": 1.2883561643835618e-05, + "loss": 0.40090117454528806, + "mean_token_accuracy": 0.8996903270483017, + "num_tokens": 10534924.0, + "step": 5470 + }, + { + "entropy": 0.304948503151536, + "epoch": 0.745578231292517, + "grad_norm": 0.3416388928890228, + "learning_rate": 1.2815068493150686e-05, + "loss": 0.3036719799041748, + "mean_token_accuracy": 0.9170698165893555, + "num_tokens": 10556668.0, + "step": 5480 + }, + { + "entropy": 0.3498163715004921, + "epoch": 0.746938775510204, + "grad_norm": 0.4322749376296997, + "learning_rate": 1.2746575342465755e-05, + "loss": 0.35872642993927, + "mean_token_accuracy": 0.9104368448257446, + "num_tokens": 10574438.0, + "step": 5490 + }, + { + "entropy": 0.2817006319761276, + "epoch": 0.7482993197278912, + "grad_norm": 0.3572882413864136, + "learning_rate": 1.2678082191780821e-05, + "loss": 0.2768646001815796, + "mean_token_accuracy": 0.9242710262537003, + "num_tokens": 10595536.0, + "step": 5500 + }, + { + "epoch": 0.7482993197278912, + "eval_entropy": 0.3340467913697163, + "eval_loss": 0.3254561424255371, + "eval_mean_token_accuracy": 0.9137606120109558, + "eval_num_tokens": 10595536.0, + "eval_runtime": 39.9117, + "eval_samples_per_second": 15.033, + "eval_steps_per_second": 7.517, + "step": 5500 + }, + { + "entropy": 0.3420073136687279, + "epoch": 0.7496598639455783, + "grad_norm": 0.6343442797660828, + "learning_rate": 1.260958904109589e-05, + "loss": 0.3322641611099243, + "mean_token_accuracy": 0.9081849426031112, + "num_tokens": 10616452.0, + "step": 5510 + }, + { + "entropy": 0.3741004906594753, + "epoch": 0.7510204081632653, + "grad_norm": 0.3708195388317108, + "learning_rate": 1.2541095890410959e-05, + "loss": 0.370376443862915, + "mean_token_accuracy": 0.9020725101232528, + "num_tokens": 10636869.0, + "step": 5520 + }, + { + "entropy": 0.36120699867606165, + "epoch": 0.7523809523809524, + "grad_norm": 0.3880722224712372, + "learning_rate": 1.2472602739726028e-05, + "loss": 0.3640472173690796, + "mean_token_accuracy": 0.9033250749111176, + "num_tokens": 10659405.0, + "step": 5530 + }, + { + "entropy": 0.3709873460233212, + "epoch": 0.7537414965986394, + "grad_norm": 0.4630112946033478, + "learning_rate": 1.2404109589041096e-05, + "loss": 0.3700759172439575, + "mean_token_accuracy": 0.9041523218154908, + "num_tokens": 10677909.0, + "step": 5540 + }, + { + "entropy": 0.2900518998503685, + "epoch": 0.7551020408163265, + "grad_norm": 0.5871257781982422, + "learning_rate": 1.2335616438356165e-05, + "loss": 0.28741438388824464, + "mean_token_accuracy": 0.9231601834297181, + "num_tokens": 10696772.0, + "step": 5550 + }, + { + "entropy": 0.31697488874197005, + "epoch": 0.7564625850340136, + "grad_norm": 0.4777441620826721, + "learning_rate": 1.2267123287671233e-05, + "loss": 0.32103445529937746, + "mean_token_accuracy": 0.9166768312454223, + "num_tokens": 10716784.0, + "step": 5560 + }, + { + "entropy": 0.3552616074681282, + "epoch": 0.7578231292517007, + "grad_norm": 0.4223809838294983, + "learning_rate": 1.2198630136986303e-05, + "loss": 0.37296199798583984, + "mean_token_accuracy": 0.9043572753667831, + "num_tokens": 10735450.0, + "step": 5570 + }, + { + "entropy": 0.2962200723588467, + "epoch": 0.7591836734693878, + "grad_norm": 0.43479984998703003, + "learning_rate": 1.213013698630137e-05, + "loss": 0.2950768947601318, + "mean_token_accuracy": 0.91934255361557, + "num_tokens": 10755247.0, + "step": 5580 + }, + { + "entropy": 0.31132856346666815, + "epoch": 0.7605442176870748, + "grad_norm": 0.4492180645465851, + "learning_rate": 1.2061643835616438e-05, + "loss": 0.3084630012512207, + "mean_token_accuracy": 0.9199709206819534, + "num_tokens": 10774582.0, + "step": 5590 + }, + { + "entropy": 0.30692355558276174, + "epoch": 0.7619047619047619, + "grad_norm": 0.5301331281661987, + "learning_rate": 1.1993150684931508e-05, + "loss": 0.2932703971862793, + "mean_token_accuracy": 0.9213265508413315, + "num_tokens": 10793235.0, + "step": 5600 + }, + { + "entropy": 0.299712085723877, + "epoch": 0.763265306122449, + "grad_norm": 0.38069605827331543, + "learning_rate": 1.1924657534246577e-05, + "loss": 0.30311241149902346, + "mean_token_accuracy": 0.9191953390836716, + "num_tokens": 10813845.0, + "step": 5610 + }, + { + "entropy": 0.34512806236743926, + "epoch": 0.7646258503401361, + "grad_norm": 0.472634881734848, + "learning_rate": 1.1856164383561643e-05, + "loss": 0.3428913354873657, + "mean_token_accuracy": 0.9112067639827728, + "num_tokens": 10832987.0, + "step": 5620 + }, + { + "entropy": 0.3049409069120884, + "epoch": 0.7659863945578231, + "grad_norm": 0.4912843704223633, + "learning_rate": 1.1787671232876713e-05, + "loss": 0.3134711503982544, + "mean_token_accuracy": 0.9179065018892288, + "num_tokens": 10851395.0, + "step": 5630 + }, + { + "entropy": 0.31765844002366067, + "epoch": 0.7673469387755102, + "grad_norm": 0.5902607440948486, + "learning_rate": 1.1719178082191782e-05, + "loss": 0.3220794916152954, + "mean_token_accuracy": 0.9152461349964142, + "num_tokens": 10869970.0, + "step": 5640 + }, + { + "entropy": 0.32936450466513634, + "epoch": 0.7687074829931972, + "grad_norm": 0.48000672459602356, + "learning_rate": 1.165068493150685e-05, + "loss": 0.3320637226104736, + "mean_token_accuracy": 0.9145504355430603, + "num_tokens": 10888631.0, + "step": 5650 + }, + { + "entropy": 0.3303429067134857, + "epoch": 0.7700680272108843, + "grad_norm": 0.33110785484313965, + "learning_rate": 1.158219178082192e-05, + "loss": 0.32364022731781006, + "mean_token_accuracy": 0.9085778921842576, + "num_tokens": 10911541.0, + "step": 5660 + }, + { + "entropy": 0.3667861148715019, + "epoch": 0.7714285714285715, + "grad_norm": 0.41618260741233826, + "learning_rate": 1.1513698630136985e-05, + "loss": 0.36133155822753904, + "mean_token_accuracy": 0.9050101220607758, + "num_tokens": 10931770.0, + "step": 5670 + }, + { + "entropy": 0.34156960174441336, + "epoch": 0.7727891156462585, + "grad_norm": 0.4888611435890198, + "learning_rate": 1.1445205479452055e-05, + "loss": 0.35315518379211425, + "mean_token_accuracy": 0.9090872585773468, + "num_tokens": 10950198.0, + "step": 5680 + }, + { + "entropy": 0.3624446362257004, + "epoch": 0.7741496598639456, + "grad_norm": 0.6312777996063232, + "learning_rate": 1.1376712328767124e-05, + "loss": 0.3629330635070801, + "mean_token_accuracy": 0.9079570889472961, + "num_tokens": 10968770.0, + "step": 5690 + }, + { + "entropy": 0.30860179737210275, + "epoch": 0.7755102040816326, + "grad_norm": 0.6905585527420044, + "learning_rate": 1.1308219178082192e-05, + "loss": 0.31810925006866453, + "mean_token_accuracy": 0.916623106598854, + "num_tokens": 10986842.0, + "step": 5700 + }, + { + "entropy": 0.3661921538412571, + "epoch": 0.7768707482993197, + "grad_norm": 0.6159297823905945, + "learning_rate": 1.1239726027397262e-05, + "loss": 0.3769394397735596, + "mean_token_accuracy": 0.9014403194189071, + "num_tokens": 11008087.0, + "step": 5710 + }, + { + "entropy": 0.3508593410253525, + "epoch": 0.7782312925170068, + "grad_norm": 0.34525948762893677, + "learning_rate": 1.117123287671233e-05, + "loss": 0.33626294136047363, + "mean_token_accuracy": 0.9080008000135422, + "num_tokens": 11030274.0, + "step": 5720 + }, + { + "entropy": 0.36780777871608733, + "epoch": 0.7795918367346939, + "grad_norm": 0.5597181916236877, + "learning_rate": 1.1102739726027397e-05, + "loss": 0.38009593486785886, + "mean_token_accuracy": 0.9044328391551971, + "num_tokens": 11049724.0, + "step": 5730 + }, + { + "entropy": 0.29728768691420554, + "epoch": 0.780952380952381, + "grad_norm": 0.5715142488479614, + "learning_rate": 1.1034246575342467e-05, + "loss": 0.30108027458190917, + "mean_token_accuracy": 0.9220935791730881, + "num_tokens": 11066979.0, + "step": 5740 + }, + { + "entropy": 0.33334632143378257, + "epoch": 0.782312925170068, + "grad_norm": 0.464752733707428, + "learning_rate": 1.0965753424657534e-05, + "loss": 0.3300285339355469, + "mean_token_accuracy": 0.9103938549757004, + "num_tokens": 11087708.0, + "step": 5750 + }, + { + "epoch": 0.782312925170068, + "eval_entropy": 0.33076301847894984, + "eval_loss": 0.3248400092124939, + "eval_mean_token_accuracy": 0.9138080885012945, + "eval_num_tokens": 11087708.0, + "eval_runtime": 39.9646, + "eval_samples_per_second": 15.013, + "eval_steps_per_second": 7.507, + "step": 5750 + }, + { + "entropy": 0.3186205253005028, + "epoch": 0.7836734693877551, + "grad_norm": 0.42055535316467285, + "learning_rate": 1.0897260273972604e-05, + "loss": 0.3032658815383911, + "mean_token_accuracy": 0.9145529806613922, + "num_tokens": 11108724.0, + "step": 5760 + }, + { + "entropy": 0.3299889124929905, + "epoch": 0.7850340136054422, + "grad_norm": 0.42789825797080994, + "learning_rate": 1.0828767123287672e-05, + "loss": 0.34171323776245116, + "mean_token_accuracy": 0.9122124373912811, + "num_tokens": 11126528.0, + "step": 5770 + }, + { + "entropy": 0.33234588280320165, + "epoch": 0.7863945578231293, + "grad_norm": 0.5156826376914978, + "learning_rate": 1.076027397260274e-05, + "loss": 0.3394943714141846, + "mean_token_accuracy": 0.9124648332595825, + "num_tokens": 11144913.0, + "step": 5780 + }, + { + "entropy": 0.3698764108121395, + "epoch": 0.7877551020408163, + "grad_norm": 0.3107636272907257, + "learning_rate": 1.0691780821917809e-05, + "loss": 0.3694345235824585, + "mean_token_accuracy": 0.9042896002531051, + "num_tokens": 11167390.0, + "step": 5790 + }, + { + "entropy": 0.3120001643896103, + "epoch": 0.7891156462585034, + "grad_norm": 0.5672539472579956, + "learning_rate": 1.0623287671232877e-05, + "loss": 0.3034272909164429, + "mean_token_accuracy": 0.9210961252450943, + "num_tokens": 11186470.0, + "step": 5800 + }, + { + "entropy": 0.31296552121639254, + "epoch": 0.7904761904761904, + "grad_norm": 0.6534621119499207, + "learning_rate": 1.0554794520547946e-05, + "loss": 0.32099902629852295, + "mean_token_accuracy": 0.9163517713546753, + "num_tokens": 11204042.0, + "step": 5810 + }, + { + "entropy": 0.2991756394505501, + "epoch": 0.7918367346938775, + "grad_norm": 0.5420209169387817, + "learning_rate": 1.0486301369863014e-05, + "loss": 0.2895534038543701, + "mean_token_accuracy": 0.9218130499124527, + "num_tokens": 11222662.0, + "step": 5820 + }, + { + "entropy": 0.29351976960897447, + "epoch": 0.7931972789115647, + "grad_norm": 0.41789329051971436, + "learning_rate": 1.0417808219178082e-05, + "loss": 0.2951449155807495, + "mean_token_accuracy": 0.9238580882549285, + "num_tokens": 11240403.0, + "step": 5830 + }, + { + "entropy": 0.36796221882104874, + "epoch": 0.7945578231292517, + "grad_norm": 0.3995990753173828, + "learning_rate": 1.0349315068493151e-05, + "loss": 0.38138272762298586, + "mean_token_accuracy": 0.9061296254396438, + "num_tokens": 11259380.0, + "step": 5840 + }, + { + "entropy": 0.3072926219552755, + "epoch": 0.7959183673469388, + "grad_norm": 0.4705047011375427, + "learning_rate": 1.028082191780822e-05, + "loss": 0.3008893489837646, + "mean_token_accuracy": 0.9149341195821762, + "num_tokens": 11280378.0, + "step": 5850 + }, + { + "entropy": 0.2909562915563583, + "epoch": 0.7972789115646258, + "grad_norm": 0.44585728645324707, + "learning_rate": 1.0212328767123288e-05, + "loss": 0.28569257259368896, + "mean_token_accuracy": 0.9219981044530868, + "num_tokens": 11298956.0, + "step": 5860 + }, + { + "entropy": 0.30830206871032717, + "epoch": 0.7986394557823129, + "grad_norm": 0.6111286282539368, + "learning_rate": 1.0143835616438356e-05, + "loss": 0.3069111108779907, + "mean_token_accuracy": 0.920006063580513, + "num_tokens": 11318152.0, + "step": 5870 + }, + { + "entropy": 0.3211949087679386, + "epoch": 0.8, + "grad_norm": 0.5721073150634766, + "learning_rate": 1.0075342465753424e-05, + "loss": 0.3205984592437744, + "mean_token_accuracy": 0.9189320415258407, + "num_tokens": 11335961.0, + "step": 5880 + }, + { + "entropy": 0.3145246207714081, + "epoch": 0.8013605442176871, + "grad_norm": 0.4170847237110138, + "learning_rate": 1.0006849315068494e-05, + "loss": 0.3303114652633667, + "mean_token_accuracy": 0.9154134333133698, + "num_tokens": 11354734.0, + "step": 5890 + }, + { + "entropy": 0.2957589320838451, + "epoch": 0.8027210884353742, + "grad_norm": 0.5433431267738342, + "learning_rate": 9.938356164383563e-06, + "loss": 0.29770326614379883, + "mean_token_accuracy": 0.9195897787809372, + "num_tokens": 11373075.0, + "step": 5900 + }, + { + "entropy": 0.30681760758161547, + "epoch": 0.8040816326530612, + "grad_norm": 0.6477324366569519, + "learning_rate": 9.86986301369863e-06, + "loss": 0.31293673515319825, + "mean_token_accuracy": 0.9200047612190246, + "num_tokens": 11390279.0, + "step": 5910 + }, + { + "entropy": 0.3382511667907238, + "epoch": 0.8054421768707483, + "grad_norm": 0.5619089603424072, + "learning_rate": 9.801369863013699e-06, + "loss": 0.34138176441192625, + "mean_token_accuracy": 0.9094976276159287, + "num_tokens": 11409791.0, + "step": 5920 + }, + { + "entropy": 0.2938502006232738, + "epoch": 0.8068027210884354, + "grad_norm": 0.3731178045272827, + "learning_rate": 9.732876712328768e-06, + "loss": 0.2872425317764282, + "mean_token_accuracy": 0.9224807500839234, + "num_tokens": 11429401.0, + "step": 5930 + }, + { + "entropy": 0.33451291769742963, + "epoch": 0.8081632653061225, + "grad_norm": 0.43806591629981995, + "learning_rate": 9.664383561643836e-06, + "loss": 0.33567047119140625, + "mean_token_accuracy": 0.9116082489490509, + "num_tokens": 11449058.0, + "step": 5940 + }, + { + "entropy": 0.30286668688058854, + "epoch": 0.8095238095238095, + "grad_norm": 0.587124228477478, + "learning_rate": 9.595890410958905e-06, + "loss": 0.31200466156005857, + "mean_token_accuracy": 0.9177083373069763, + "num_tokens": 11470294.0, + "step": 5950 + }, + { + "entropy": 0.3303552411496639, + "epoch": 0.8108843537414966, + "grad_norm": 0.6207278966903687, + "learning_rate": 9.527397260273973e-06, + "loss": 0.33328967094421386, + "mean_token_accuracy": 0.9125414162874221, + "num_tokens": 11489357.0, + "step": 5960 + }, + { + "entropy": 0.31682623624801637, + "epoch": 0.8122448979591836, + "grad_norm": 0.44043925404548645, + "learning_rate": 9.458904109589041e-06, + "loss": 0.31499733924865725, + "mean_token_accuracy": 0.9134839475154877, + "num_tokens": 11508233.0, + "step": 5970 + }, + { + "entropy": 0.3149236857891083, + "epoch": 0.8136054421768707, + "grad_norm": 0.36899179220199585, + "learning_rate": 9.39041095890411e-06, + "loss": 0.3117794990539551, + "mean_token_accuracy": 0.9213671326637268, + "num_tokens": 11525907.0, + "step": 5980 + }, + { + "entropy": 0.3045039355754852, + "epoch": 0.8149659863945579, + "grad_norm": 0.49508801102638245, + "learning_rate": 9.321917808219178e-06, + "loss": 0.2972161531448364, + "mean_token_accuracy": 0.9218720376491547, + "num_tokens": 11545454.0, + "step": 5990 + }, + { + "entropy": 0.33072281926870345, + "epoch": 0.8163265306122449, + "grad_norm": 0.3706882894039154, + "learning_rate": 9.253424657534248e-06, + "loss": 0.339734411239624, + "mean_token_accuracy": 0.9110476523637772, + "num_tokens": 11565392.0, + "step": 6000 + }, + { + "epoch": 0.8163265306122449, + "eval_entropy": 0.33086989077428974, + "eval_loss": 0.32422059774398804, + "eval_mean_token_accuracy": 0.913934031923612, + "eval_num_tokens": 11565392.0, + "eval_runtime": 38.7107, + "eval_samples_per_second": 15.5, + "eval_steps_per_second": 7.75, + "step": 6000 + }, + { + "entropy": 0.298756917193532, + "epoch": 0.817687074829932, + "grad_norm": 0.42621785402297974, + "learning_rate": 9.184931506849315e-06, + "loss": 0.29668822288513186, + "mean_token_accuracy": 0.9230580598115921, + "num_tokens": 11583377.0, + "step": 6010 + }, + { + "entropy": 0.3184999957680702, + "epoch": 0.819047619047619, + "grad_norm": 0.5035575032234192, + "learning_rate": 9.116438356164383e-06, + "loss": 0.3027308702468872, + "mean_token_accuracy": 0.9170263260602951, + "num_tokens": 11603965.0, + "step": 6020 + }, + { + "entropy": 0.3802847608923912, + "epoch": 0.8204081632653061, + "grad_norm": 0.49796271324157715, + "learning_rate": 9.047945205479453e-06, + "loss": 0.38724322319030763, + "mean_token_accuracy": 0.9009683400392532, + "num_tokens": 11624271.0, + "step": 6030 + }, + { + "entropy": 0.27982044592499733, + "epoch": 0.8217687074829932, + "grad_norm": 0.5557689666748047, + "learning_rate": 8.97945205479452e-06, + "loss": 0.2826409816741943, + "mean_token_accuracy": 0.9224202632904053, + "num_tokens": 11644456.0, + "step": 6040 + }, + { + "entropy": 0.3160627827048302, + "epoch": 0.8231292517006803, + "grad_norm": 0.576414167881012, + "learning_rate": 8.91095890410959e-06, + "loss": 0.31594884395599365, + "mean_token_accuracy": 0.9167442142963409, + "num_tokens": 11664317.0, + "step": 6050 + }, + { + "entropy": 0.27179846093058585, + "epoch": 0.8244897959183674, + "grad_norm": 0.5361799597740173, + "learning_rate": 8.84246575342466e-06, + "loss": 0.2700121641159058, + "mean_token_accuracy": 0.9278384625911713, + "num_tokens": 11682377.0, + "step": 6060 + }, + { + "entropy": 0.30858047008514405, + "epoch": 0.8258503401360544, + "grad_norm": 0.4765695333480835, + "learning_rate": 8.773972602739725e-06, + "loss": 0.30114083290100097, + "mean_token_accuracy": 0.9185465186834335, + "num_tokens": 11700972.0, + "step": 6070 + }, + { + "entropy": 0.3052784204483032, + "epoch": 0.8272108843537415, + "grad_norm": 0.5827853679656982, + "learning_rate": 8.705479452054795e-06, + "loss": 0.30794899463653563, + "mean_token_accuracy": 0.917677515745163, + "num_tokens": 11721322.0, + "step": 6080 + }, + { + "entropy": 0.31110915169119835, + "epoch": 0.8285714285714286, + "grad_norm": 0.5326927900314331, + "learning_rate": 8.636986301369863e-06, + "loss": 0.32460036277771, + "mean_token_accuracy": 0.9198139071464538, + "num_tokens": 11738982.0, + "step": 6090 + }, + { + "entropy": 0.33925400823354723, + "epoch": 0.8299319727891157, + "grad_norm": 0.36450883746147156, + "learning_rate": 8.568493150684932e-06, + "loss": 0.3317252635955811, + "mean_token_accuracy": 0.9089361041784286, + "num_tokens": 11760916.0, + "step": 6100 + }, + { + "entropy": 0.3158437326550484, + "epoch": 0.8312925170068027, + "grad_norm": 0.491068959236145, + "learning_rate": 8.500000000000002e-06, + "loss": 0.3143324851989746, + "mean_token_accuracy": 0.9161739617586135, + "num_tokens": 11779949.0, + "step": 6110 + }, + { + "entropy": 0.3439742915332317, + "epoch": 0.8326530612244898, + "grad_norm": 0.39802658557891846, + "learning_rate": 8.431506849315068e-06, + "loss": 0.3491824626922607, + "mean_token_accuracy": 0.9078180462121963, + "num_tokens": 11799970.0, + "step": 6120 + }, + { + "entropy": 0.2836092166602612, + "epoch": 0.8340136054421768, + "grad_norm": 0.4259534478187561, + "learning_rate": 8.363013698630137e-06, + "loss": 0.2912003040313721, + "mean_token_accuracy": 0.9252801060676574, + "num_tokens": 11820426.0, + "step": 6130 + }, + { + "entropy": 0.3005498096346855, + "epoch": 0.8353741496598639, + "grad_norm": 0.4778730273246765, + "learning_rate": 8.294520547945207e-06, + "loss": 0.28653755187988283, + "mean_token_accuracy": 0.9214383721351623, + "num_tokens": 11839890.0, + "step": 6140 + }, + { + "entropy": 0.3486380882561207, + "epoch": 0.8367346938775511, + "grad_norm": 0.6083812117576599, + "learning_rate": 8.226027397260274e-06, + "loss": 0.35432674884796145, + "mean_token_accuracy": 0.9074101686477661, + "num_tokens": 11859990.0, + "step": 6150 + }, + { + "entropy": 0.32456274405121804, + "epoch": 0.8380952380952381, + "grad_norm": 0.4606610834598541, + "learning_rate": 8.157534246575344e-06, + "loss": 0.3331348419189453, + "mean_token_accuracy": 0.9130833357572555, + "num_tokens": 11878717.0, + "step": 6160 + }, + { + "entropy": 0.32492469996213913, + "epoch": 0.8394557823129252, + "grad_norm": 0.4846050441265106, + "learning_rate": 8.089041095890412e-06, + "loss": 0.3428283929824829, + "mean_token_accuracy": 0.9127777367830276, + "num_tokens": 11897621.0, + "step": 6170 + }, + { + "entropy": 0.3137147091329098, + "epoch": 0.8408163265306122, + "grad_norm": 0.45196622610092163, + "learning_rate": 8.02054794520548e-06, + "loss": 0.3091363191604614, + "mean_token_accuracy": 0.9139469265937805, + "num_tokens": 11916535.0, + "step": 6180 + }, + { + "entropy": 0.29806752055883406, + "epoch": 0.8421768707482993, + "grad_norm": 0.454557865858078, + "learning_rate": 7.952054794520549e-06, + "loss": 0.29362752437591555, + "mean_token_accuracy": 0.9199837118387222, + "num_tokens": 11935369.0, + "step": 6190 + }, + { + "entropy": 0.28466799929738046, + "epoch": 0.8435374149659864, + "grad_norm": 0.5338443517684937, + "learning_rate": 7.883561643835617e-06, + "loss": 0.29237349033355714, + "mean_token_accuracy": 0.9232655674219131, + "num_tokens": 11953135.0, + "step": 6200 + }, + { + "entropy": 0.3033223427832127, + "epoch": 0.8448979591836735, + "grad_norm": 0.6123531460762024, + "learning_rate": 7.815068493150686e-06, + "loss": 0.3017022371292114, + "mean_token_accuracy": 0.920993834733963, + "num_tokens": 11970670.0, + "step": 6210 + }, + { + "entropy": 0.36309069730341437, + "epoch": 0.8462585034013606, + "grad_norm": 0.6074350476264954, + "learning_rate": 7.746575342465754e-06, + "loss": 0.3690043926239014, + "mean_token_accuracy": 0.907643273472786, + "num_tokens": 11988089.0, + "step": 6220 + }, + { + "entropy": 0.32631669789552686, + "epoch": 0.8476190476190476, + "grad_norm": 0.5981218814849854, + "learning_rate": 7.678082191780822e-06, + "loss": 0.337539529800415, + "mean_token_accuracy": 0.9157464295625687, + "num_tokens": 12005832.0, + "step": 6230 + }, + { + "entropy": 0.34066905826330185, + "epoch": 0.8489795918367347, + "grad_norm": 0.626277506351471, + "learning_rate": 7.609589041095891e-06, + "loss": 0.3323520660400391, + "mean_token_accuracy": 0.9154704749584198, + "num_tokens": 12022879.0, + "step": 6240 + }, + { + "entropy": 0.3810716487467289, + "epoch": 0.8503401360544217, + "grad_norm": 0.4984925091266632, + "learning_rate": 7.54109589041096e-06, + "loss": 0.4056093215942383, + "mean_token_accuracy": 0.9004001200199128, + "num_tokens": 12042173.0, + "step": 6250 + }, + { + "epoch": 0.8503401360544217, + "eval_entropy": 0.32555937600632506, + "eval_loss": 0.32379451394081116, + "eval_mean_token_accuracy": 0.9141173479954402, + "eval_num_tokens": 12042173.0, + "eval_runtime": 38.9464, + "eval_samples_per_second": 15.406, + "eval_steps_per_second": 7.703, + "step": 6250 + }, + { + "entropy": 0.30119421854615214, + "epoch": 0.8517006802721089, + "grad_norm": 0.45719167590141296, + "learning_rate": 7.4726027397260285e-06, + "loss": 0.2935132265090942, + "mean_token_accuracy": 0.9244018375873566, + "num_tokens": 12059658.0, + "step": 6260 + }, + { + "entropy": 0.3010863531380892, + "epoch": 0.8530612244897959, + "grad_norm": 0.5109034776687622, + "learning_rate": 7.404109589041095e-06, + "loss": 0.31348700523376466, + "mean_token_accuracy": 0.9156623929738998, + "num_tokens": 12078793.0, + "step": 6270 + }, + { + "entropy": 0.30613968297839167, + "epoch": 0.854421768707483, + "grad_norm": 0.5922409892082214, + "learning_rate": 7.335616438356165e-06, + "loss": 0.31641838550567625, + "mean_token_accuracy": 0.9163318037986755, + "num_tokens": 12098446.0, + "step": 6280 + }, + { + "entropy": 0.3765171080827713, + "epoch": 0.85578231292517, + "grad_norm": 0.5758050084114075, + "learning_rate": 7.2671232876712335e-06, + "loss": 0.37129898071289064, + "mean_token_accuracy": 0.9047569125890732, + "num_tokens": 12118576.0, + "step": 6290 + }, + { + "entropy": 0.3128926806151867, + "epoch": 0.8571428571428571, + "grad_norm": 0.3453630208969116, + "learning_rate": 7.198630136986302e-06, + "loss": 0.31790430545806886, + "mean_token_accuracy": 0.9154722660779953, + "num_tokens": 12137064.0, + "step": 6300 + }, + { + "entropy": 0.34110006168484686, + "epoch": 0.8585034013605443, + "grad_norm": 0.7402700185775757, + "learning_rate": 7.130136986301371e-06, + "loss": 0.3416825532913208, + "mean_token_accuracy": 0.910080087184906, + "num_tokens": 12157405.0, + "step": 6310 + }, + { + "entropy": 0.3634137712419033, + "epoch": 0.8598639455782313, + "grad_norm": 0.6401071548461914, + "learning_rate": 7.0616438356164386e-06, + "loss": 0.36912858486175537, + "mean_token_accuracy": 0.9012720108032226, + "num_tokens": 12176452.0, + "step": 6320 + }, + { + "entropy": 0.3082569859921932, + "epoch": 0.8612244897959184, + "grad_norm": 0.5355600118637085, + "learning_rate": 6.993150684931507e-06, + "loss": 0.29816470146179197, + "mean_token_accuracy": 0.9203158468008041, + "num_tokens": 12194612.0, + "step": 6330 + }, + { + "entropy": 0.3393115036189556, + "epoch": 0.8625850340136054, + "grad_norm": 0.38070395588874817, + "learning_rate": 6.924657534246576e-06, + "loss": 0.32619943618774416, + "mean_token_accuracy": 0.9085036665201187, + "num_tokens": 12217956.0, + "step": 6340 + }, + { + "entropy": 0.3250208295881748, + "epoch": 0.8639455782312925, + "grad_norm": 0.7538926005363464, + "learning_rate": 6.8561643835616444e-06, + "loss": 0.33664188385009763, + "mean_token_accuracy": 0.9119222640991211, + "num_tokens": 12236318.0, + "step": 6350 + }, + { + "entropy": 0.2986321195960045, + "epoch": 0.8653061224489796, + "grad_norm": 0.4130662679672241, + "learning_rate": 6.787671232876712e-06, + "loss": 0.28825740814208983, + "mean_token_accuracy": 0.9217880070209503, + "num_tokens": 12255826.0, + "step": 6360 + }, + { + "entropy": 0.2841433063149452, + "epoch": 0.8666666666666667, + "grad_norm": 0.6472411155700684, + "learning_rate": 6.719178082191781e-06, + "loss": 0.27176086902618407, + "mean_token_accuracy": 0.9227411568164825, + "num_tokens": 12276465.0, + "step": 6370 + }, + { + "entropy": 0.34614059180021284, + "epoch": 0.8680272108843538, + "grad_norm": 0.4979070723056793, + "learning_rate": 6.6506849315068495e-06, + "loss": 0.3492851495742798, + "mean_token_accuracy": 0.9061255156993866, + "num_tokens": 12295950.0, + "step": 6380 + }, + { + "entropy": 0.3491402305662632, + "epoch": 0.8693877551020408, + "grad_norm": 0.5168848633766174, + "learning_rate": 6.582191780821918e-06, + "loss": 0.356025767326355, + "mean_token_accuracy": 0.9081326395273208, + "num_tokens": 12315074.0, + "step": 6390 + }, + { + "entropy": 0.3183485813438892, + "epoch": 0.8707482993197279, + "grad_norm": 0.3737778961658478, + "learning_rate": 6.513698630136987e-06, + "loss": 0.3137796401977539, + "mean_token_accuracy": 0.9168394416570663, + "num_tokens": 12333113.0, + "step": 6400 + }, + { + "entropy": 0.3234575852751732, + "epoch": 0.8721088435374149, + "grad_norm": 0.4080883860588074, + "learning_rate": 6.4452054794520545e-06, + "loss": 0.3283559799194336, + "mean_token_accuracy": 0.910961264371872, + "num_tokens": 12354119.0, + "step": 6410 + }, + { + "entropy": 0.33070197626948356, + "epoch": 0.8734693877551021, + "grad_norm": 0.6628795862197876, + "learning_rate": 6.376712328767123e-06, + "loss": 0.33696324825286866, + "mean_token_accuracy": 0.9128307223320007, + "num_tokens": 12372396.0, + "step": 6420 + }, + { + "entropy": 0.296940540522337, + "epoch": 0.8748299319727891, + "grad_norm": 0.5327218174934387, + "learning_rate": 6.308219178082192e-06, + "loss": 0.29673740863800047, + "mean_token_accuracy": 0.9188735485076904, + "num_tokens": 12391926.0, + "step": 6430 + }, + { + "entropy": 0.37082950621843336, + "epoch": 0.8761904761904762, + "grad_norm": 0.5576393008232117, + "learning_rate": 6.23972602739726e-06, + "loss": 0.37212495803833007, + "mean_token_accuracy": 0.903967159986496, + "num_tokens": 12411397.0, + "step": 6440 + }, + { + "entropy": 0.292638061195612, + "epoch": 0.8775510204081632, + "grad_norm": 0.35143810510635376, + "learning_rate": 6.171232876712329e-06, + "loss": 0.2931572675704956, + "mean_token_accuracy": 0.9227148920297623, + "num_tokens": 12432121.0, + "step": 6450 + }, + { + "entropy": 0.3156722396612167, + "epoch": 0.8789115646258503, + "grad_norm": 0.774771511554718, + "learning_rate": 6.102739726027398e-06, + "loss": 0.31828532218933103, + "mean_token_accuracy": 0.9142497092485428, + "num_tokens": 12451601.0, + "step": 6460 + }, + { + "entropy": 0.292581620067358, + "epoch": 0.8802721088435375, + "grad_norm": 0.4632379710674286, + "learning_rate": 6.034246575342466e-06, + "loss": 0.28803262710571287, + "mean_token_accuracy": 0.9243631362915039, + "num_tokens": 12470105.0, + "step": 6470 + }, + { + "entropy": 0.3136554330587387, + "epoch": 0.8816326530612245, + "grad_norm": 0.5806280374526978, + "learning_rate": 5.965753424657534e-06, + "loss": 0.31753370761871336, + "mean_token_accuracy": 0.9147692441940307, + "num_tokens": 12488804.0, + "step": 6480 + }, + { + "entropy": 0.33160665780305865, + "epoch": 0.8829931972789116, + "grad_norm": 0.47363021969795227, + "learning_rate": 5.8972602739726035e-06, + "loss": 0.3279127836227417, + "mean_token_accuracy": 0.9173679798841476, + "num_tokens": 12507279.0, + "step": 6490 + }, + { + "entropy": 0.30978685468435285, + "epoch": 0.8843537414965986, + "grad_norm": 0.3723524212837219, + "learning_rate": 5.828767123287671e-06, + "loss": 0.311632776260376, + "mean_token_accuracy": 0.9137750864028931, + "num_tokens": 12527509.0, + "step": 6500 + }, + { + "epoch": 0.8843537414965986, + "eval_entropy": 0.3286215581248204, + "eval_loss": 0.32335928082466125, + "eval_mean_token_accuracy": 0.9142575333515803, + "eval_num_tokens": 12527509.0, + "eval_runtime": 38.7448, + "eval_samples_per_second": 15.486, + "eval_steps_per_second": 7.743, + "step": 6500 + }, + { + "entropy": 0.35880630612373354, + "epoch": 0.8857142857142857, + "grad_norm": 0.514137864112854, + "learning_rate": 5.76027397260274e-06, + "loss": 0.3609093427658081, + "mean_token_accuracy": 0.908568149805069, + "num_tokens": 12546024.0, + "step": 6510 + }, + { + "entropy": 0.2908146843314171, + "epoch": 0.8870748299319728, + "grad_norm": 0.4764300286769867, + "learning_rate": 5.6917808219178085e-06, + "loss": 0.29313054084777834, + "mean_token_accuracy": 0.9210966646671295, + "num_tokens": 12564957.0, + "step": 6520 + }, + { + "entropy": 0.3947236731648445, + "epoch": 0.8884353741496599, + "grad_norm": 0.564361572265625, + "learning_rate": 5.623287671232877e-06, + "loss": 0.3897683620452881, + "mean_token_accuracy": 0.9005183577537537, + "num_tokens": 12584511.0, + "step": 6530 + }, + { + "entropy": 0.35087050646543505, + "epoch": 0.889795918367347, + "grad_norm": 0.39907175302505493, + "learning_rate": 5.554794520547946e-06, + "loss": 0.35727748870849607, + "mean_token_accuracy": 0.9082259654998779, + "num_tokens": 12603539.0, + "step": 6540 + }, + { + "entropy": 0.30452171340584755, + "epoch": 0.891156462585034, + "grad_norm": 0.40334832668304443, + "learning_rate": 5.486301369863014e-06, + "loss": 0.31457273960113524, + "mean_token_accuracy": 0.9177052706480027, + "num_tokens": 12622546.0, + "step": 6550 + }, + { + "entropy": 0.34483264610171316, + "epoch": 0.8925170068027211, + "grad_norm": 0.5541892051696777, + "learning_rate": 5.417808219178082e-06, + "loss": 0.35019664764404296, + "mean_token_accuracy": 0.9109074980020523, + "num_tokens": 12641267.0, + "step": 6560 + }, + { + "entropy": 0.3339083693921566, + "epoch": 0.8938775510204081, + "grad_norm": 0.4268859624862671, + "learning_rate": 5.349315068493151e-06, + "loss": 0.33824708461761477, + "mean_token_accuracy": 0.9065180271863937, + "num_tokens": 12662141.0, + "step": 6570 + }, + { + "entropy": 0.32709172666072844, + "epoch": 0.8952380952380953, + "grad_norm": 0.4115796387195587, + "learning_rate": 5.2808219178082195e-06, + "loss": 0.33449928760528563, + "mean_token_accuracy": 0.9107886403799057, + "num_tokens": 12683863.0, + "step": 6580 + }, + { + "entropy": 0.359021432697773, + "epoch": 0.8965986394557823, + "grad_norm": 0.5937194228172302, + "learning_rate": 5.212328767123288e-06, + "loss": 0.34922354221343993, + "mean_token_accuracy": 0.9072045892477035, + "num_tokens": 12703543.0, + "step": 6590 + }, + { + "entropy": 0.39199258685112, + "epoch": 0.8979591836734694, + "grad_norm": 0.6500011086463928, + "learning_rate": 5.143835616438356e-06, + "loss": 0.3893763542175293, + "mean_token_accuracy": 0.9033649802207947, + "num_tokens": 12721280.0, + "step": 6600 + }, + { + "entropy": 0.2838050812482834, + "epoch": 0.8993197278911564, + "grad_norm": 0.4997044503688812, + "learning_rate": 5.075342465753425e-06, + "loss": 0.28191652297973635, + "mean_token_accuracy": 0.9257587313652038, + "num_tokens": 12738942.0, + "step": 6610 + }, + { + "entropy": 0.3208158388733864, + "epoch": 0.9006802721088435, + "grad_norm": 0.463008850812912, + "learning_rate": 5.006849315068493e-06, + "loss": 0.3163119316101074, + "mean_token_accuracy": 0.9138482451438904, + "num_tokens": 12758737.0, + "step": 6620 + }, + { + "entropy": 0.32699171006679534, + "epoch": 0.9020408163265307, + "grad_norm": 0.46865853667259216, + "learning_rate": 4.938356164383562e-06, + "loss": 0.3282167434692383, + "mean_token_accuracy": 0.917575141787529, + "num_tokens": 12776236.0, + "step": 6630 + }, + { + "entropy": 0.33992503136396407, + "epoch": 0.9034013605442177, + "grad_norm": 0.622612714767456, + "learning_rate": 4.86986301369863e-06, + "loss": 0.3454226732254028, + "mean_token_accuracy": 0.9105749785900116, + "num_tokens": 12793838.0, + "step": 6640 + }, + { + "entropy": 0.29390531592071056, + "epoch": 0.9047619047619048, + "grad_norm": 0.44798150658607483, + "learning_rate": 4.801369863013699e-06, + "loss": 0.27416980266571045, + "mean_token_accuracy": 0.9256226003170014, + "num_tokens": 12813036.0, + "step": 6650 + }, + { + "entropy": 0.350774671882391, + "epoch": 0.9061224489795918, + "grad_norm": 0.38738077878952026, + "learning_rate": 4.732876712328768e-06, + "loss": 0.33705477714538573, + "mean_token_accuracy": 0.9067309200763702, + "num_tokens": 12834574.0, + "step": 6660 + }, + { + "entropy": 0.3244813568890095, + "epoch": 0.9074829931972789, + "grad_norm": 0.4696221649646759, + "learning_rate": 4.664383561643835e-06, + "loss": 0.31435041427612304, + "mean_token_accuracy": 0.9171089708805085, + "num_tokens": 12852605.0, + "step": 6670 + }, + { + "entropy": 0.305752881616354, + "epoch": 0.908843537414966, + "grad_norm": 0.4763096868991852, + "learning_rate": 4.595890410958904e-06, + "loss": 0.3067082643508911, + "mean_token_accuracy": 0.918914520740509, + "num_tokens": 12870795.0, + "step": 6680 + }, + { + "entropy": 0.31308693066239357, + "epoch": 0.9102040816326531, + "grad_norm": 0.5077400803565979, + "learning_rate": 4.5273972602739735e-06, + "loss": 0.30920541286468506, + "mean_token_accuracy": 0.9194491058588028, + "num_tokens": 12890313.0, + "step": 6690 + }, + { + "entropy": 0.2930800631642342, + "epoch": 0.9115646258503401, + "grad_norm": 0.5321216583251953, + "learning_rate": 4.458904109589041e-06, + "loss": 0.2862373352050781, + "mean_token_accuracy": 0.9227749168872833, + "num_tokens": 12909441.0, + "step": 6700 + }, + { + "entropy": 0.32441584542393687, + "epoch": 0.9129251700680272, + "grad_norm": 0.5052834153175354, + "learning_rate": 4.39041095890411e-06, + "loss": 0.33454344272613523, + "mean_token_accuracy": 0.9128939062356949, + "num_tokens": 12928259.0, + "step": 6710 + }, + { + "entropy": 0.3400576122105122, + "epoch": 0.9142857142857143, + "grad_norm": 0.42697256803512573, + "learning_rate": 4.321917808219178e-06, + "loss": 0.3487342357635498, + "mean_token_accuracy": 0.9104131639003754, + "num_tokens": 12946832.0, + "step": 6720 + }, + { + "entropy": 0.3238037794828415, + "epoch": 0.9156462585034013, + "grad_norm": 0.4467569887638092, + "learning_rate": 4.253424657534247e-06, + "loss": 0.3275768518447876, + "mean_token_accuracy": 0.9114355087280274, + "num_tokens": 12967923.0, + "step": 6730 + }, + { + "entropy": 0.3033832423388958, + "epoch": 0.9170068027210885, + "grad_norm": 0.5587620139122009, + "learning_rate": 4.184931506849316e-06, + "loss": 0.3123866319656372, + "mean_token_accuracy": 0.9181416839361191, + "num_tokens": 12986379.0, + "step": 6740 + }, + { + "entropy": 0.32793152034282685, + "epoch": 0.9183673469387755, + "grad_norm": 0.4971228241920471, + "learning_rate": 4.116438356164384e-06, + "loss": 0.32430260181427, + "mean_token_accuracy": 0.9148997128009796, + "num_tokens": 13006250.0, + "step": 6750 + }, + { + "epoch": 0.9183673469387755, + "eval_entropy": 0.32498093172907827, + "eval_loss": 0.3230104446411133, + "eval_mean_token_accuracy": 0.9144533753395081, + "eval_num_tokens": 13006250.0, + "eval_runtime": 38.9222, + "eval_samples_per_second": 15.415, + "eval_steps_per_second": 7.708, + "step": 6750 + }, + { + "entropy": 0.35129692405462265, + "epoch": 0.9197278911564626, + "grad_norm": 0.450941264629364, + "learning_rate": 4.047945205479452e-06, + "loss": 0.36153857707977294, + "mean_token_accuracy": 0.9063918352127075, + "num_tokens": 13026583.0, + "step": 6760 + }, + { + "entropy": 0.3283290073275566, + "epoch": 0.9210884353741496, + "grad_norm": 0.5283523797988892, + "learning_rate": 3.979452054794521e-06, + "loss": 0.3139669179916382, + "mean_token_accuracy": 0.9148370325565338, + "num_tokens": 13047823.0, + "step": 6770 + }, + { + "entropy": 0.33190669044852256, + "epoch": 0.9224489795918367, + "grad_norm": 0.361507773399353, + "learning_rate": 3.9109589041095895e-06, + "loss": 0.34453206062316893, + "mean_token_accuracy": 0.9123291075229645, + "num_tokens": 13065483.0, + "step": 6780 + }, + { + "entropy": 0.3849423386156559, + "epoch": 0.9238095238095239, + "grad_norm": 0.5551365613937378, + "learning_rate": 3.842465753424658e-06, + "loss": 0.3853173017501831, + "mean_token_accuracy": 0.9038680106401443, + "num_tokens": 13084658.0, + "step": 6790 + }, + { + "entropy": 0.3309860430657864, + "epoch": 0.9251700680272109, + "grad_norm": 0.5244467258453369, + "learning_rate": 3.7739726027397263e-06, + "loss": 0.32932875156402586, + "mean_token_accuracy": 0.909374451637268, + "num_tokens": 13105539.0, + "step": 6800 + }, + { + "entropy": 0.26563679054379463, + "epoch": 0.926530612244898, + "grad_norm": 0.4336610436439514, + "learning_rate": 3.705479452054795e-06, + "loss": 0.25863826274871826, + "mean_token_accuracy": 0.9274649411439896, + "num_tokens": 13124934.0, + "step": 6810 + }, + { + "entropy": 0.3203303225338459, + "epoch": 0.927891156462585, + "grad_norm": 0.39582449197769165, + "learning_rate": 3.636986301369863e-06, + "loss": 0.32346789836883544, + "mean_token_accuracy": 0.9153842777013779, + "num_tokens": 13143409.0, + "step": 6820 + }, + { + "entropy": 0.33626803755760193, + "epoch": 0.9292517006802721, + "grad_norm": 0.6105754375457764, + "learning_rate": 3.5684931506849317e-06, + "loss": 0.3388401508331299, + "mean_token_accuracy": 0.9112157255411149, + "num_tokens": 13163858.0, + "step": 6830 + }, + { + "entropy": 0.3165531374514103, + "epoch": 0.9306122448979591, + "grad_norm": 0.6469919681549072, + "learning_rate": 3.5000000000000004e-06, + "loss": 0.3066595792770386, + "mean_token_accuracy": 0.9175763040781021, + "num_tokens": 13183825.0, + "step": 6840 + }, + { + "entropy": 0.38997668847441674, + "epoch": 0.9319727891156463, + "grad_norm": 0.49790993332862854, + "learning_rate": 3.4315068493150686e-06, + "loss": 0.40053672790527345, + "mean_token_accuracy": 0.8986356943845749, + "num_tokens": 13203094.0, + "step": 6850 + }, + { + "entropy": 0.2587420903146267, + "epoch": 0.9333333333333333, + "grad_norm": 0.4957631230354309, + "learning_rate": 3.363013698630137e-06, + "loss": 0.25925211906433104, + "mean_token_accuracy": 0.9277909904718399, + "num_tokens": 13224483.0, + "step": 6860 + }, + { + "entropy": 0.2768689256161451, + "epoch": 0.9346938775510204, + "grad_norm": 0.398699551820755, + "learning_rate": 3.2945205479452054e-06, + "loss": 0.27082579135894774, + "mean_token_accuracy": 0.9245598196983338, + "num_tokens": 13243719.0, + "step": 6870 + }, + { + "entropy": 0.2864796228706837, + "epoch": 0.9360544217687075, + "grad_norm": 0.4619617164134979, + "learning_rate": 3.226027397260274e-06, + "loss": 0.28009073734283446, + "mean_token_accuracy": 0.9254051715135574, + "num_tokens": 13263387.0, + "step": 6880 + }, + { + "entropy": 0.2629375524818897, + "epoch": 0.9374149659863945, + "grad_norm": 0.5026074647903442, + "learning_rate": 3.1575342465753422e-06, + "loss": 0.2672743558883667, + "mean_token_accuracy": 0.9294937402009964, + "num_tokens": 13280810.0, + "step": 6890 + }, + { + "entropy": 0.3043747998774052, + "epoch": 0.9387755102040817, + "grad_norm": 0.6281470656394958, + "learning_rate": 3.089041095890411e-06, + "loss": 0.3025246858596802, + "mean_token_accuracy": 0.9180625200271606, + "num_tokens": 13299231.0, + "step": 6900 + }, + { + "entropy": 0.33138156682252884, + "epoch": 0.9401360544217687, + "grad_norm": 0.5016866326332092, + "learning_rate": 3.0205479452054795e-06, + "loss": 0.34398717880249025, + "mean_token_accuracy": 0.911366480588913, + "num_tokens": 13319077.0, + "step": 6910 + }, + { + "entropy": 0.324043782800436, + "epoch": 0.9414965986394558, + "grad_norm": 0.3665086328983307, + "learning_rate": 2.952054794520548e-06, + "loss": 0.3297460079193115, + "mean_token_accuracy": 0.9116989314556122, + "num_tokens": 13339163.0, + "step": 6920 + }, + { + "entropy": 0.31357899233698844, + "epoch": 0.9428571428571428, + "grad_norm": 0.4561605155467987, + "learning_rate": 2.8835616438356167e-06, + "loss": 0.3067751169204712, + "mean_token_accuracy": 0.9200424224138259, + "num_tokens": 13357061.0, + "step": 6930 + }, + { + "entropy": 0.33805915862321856, + "epoch": 0.9442176870748299, + "grad_norm": 0.3674219846725464, + "learning_rate": 2.815068493150685e-06, + "loss": 0.3418180704116821, + "mean_token_accuracy": 0.908822026848793, + "num_tokens": 13377792.0, + "step": 6940 + }, + { + "entropy": 0.3485445693135262, + "epoch": 0.9455782312925171, + "grad_norm": 0.3852236568927765, + "learning_rate": 2.7465753424657536e-06, + "loss": 0.3584400177001953, + "mean_token_accuracy": 0.9075007975101471, + "num_tokens": 13398215.0, + "step": 6950 + }, + { + "entropy": 0.30585007295012473, + "epoch": 0.9469387755102041, + "grad_norm": 0.5300541520118713, + "learning_rate": 2.6780821917808218e-06, + "loss": 0.3072829246520996, + "mean_token_accuracy": 0.9181619912385941, + "num_tokens": 13418511.0, + "step": 6960 + }, + { + "entropy": 0.3123452767729759, + "epoch": 0.9482993197278912, + "grad_norm": 0.5543433427810669, + "learning_rate": 2.6095890410958904e-06, + "loss": 0.3134234666824341, + "mean_token_accuracy": 0.9177923172712326, + "num_tokens": 13438071.0, + "step": 6970 + }, + { + "entropy": 0.3253453135490417, + "epoch": 0.9496598639455782, + "grad_norm": 0.6263024806976318, + "learning_rate": 2.541095890410959e-06, + "loss": 0.3312222480773926, + "mean_token_accuracy": 0.9184915989637374, + "num_tokens": 13455223.0, + "step": 6980 + }, + { + "entropy": 0.30098590180277823, + "epoch": 0.9510204081632653, + "grad_norm": 0.41422611474990845, + "learning_rate": 2.4726027397260277e-06, + "loss": 0.29193758964538574, + "mean_token_accuracy": 0.9190629869699478, + "num_tokens": 13476518.0, + "step": 6990 + }, + { + "entropy": 0.2495936904102564, + "epoch": 0.9523809523809523, + "grad_norm": 0.5148512125015259, + "learning_rate": 2.404109589041096e-06, + "loss": 0.25398826599121094, + "mean_token_accuracy": 0.931014546751976, + "num_tokens": 13495349.0, + "step": 7000 + }, + { + "epoch": 0.9523809523809523, + "eval_entropy": 0.32407169657448925, + "eval_loss": 0.3227205276489258, + "eval_mean_token_accuracy": 0.9144436609745026, + "eval_num_tokens": 13495349.0, + "eval_runtime": 40.5516, + "eval_samples_per_second": 14.796, + "eval_steps_per_second": 7.398, + "step": 7000 + }, + { + "entropy": 0.39397928565740586, + "epoch": 0.9537414965986395, + "grad_norm": 0.49788719415664673, + "learning_rate": 2.3356164383561645e-06, + "loss": 0.3860008955001831, + "mean_token_accuracy": 0.897650420665741, + "num_tokens": 13514703.0, + "step": 7010 + }, + { + "entropy": 0.3235211636871099, + "epoch": 0.9551020408163265, + "grad_norm": 0.7469091415405273, + "learning_rate": 2.2671232876712327e-06, + "loss": 0.33283085823059083, + "mean_token_accuracy": 0.9115709811449051, + "num_tokens": 13534485.0, + "step": 7020 + }, + { + "entropy": 0.28874940276145933, + "epoch": 0.9564625850340136, + "grad_norm": 0.3857863247394562, + "learning_rate": 2.1986301369863017e-06, + "loss": 0.2785567045211792, + "mean_token_accuracy": 0.9208014994859696, + "num_tokens": 13554148.0, + "step": 7030 + }, + { + "entropy": 0.3946863390505314, + "epoch": 0.9578231292517007, + "grad_norm": 0.5933367609977722, + "learning_rate": 2.13013698630137e-06, + "loss": 0.41825008392333984, + "mean_token_accuracy": 0.8936791986227035, + "num_tokens": 13575347.0, + "step": 7040 + }, + { + "entropy": 0.31831209883093836, + "epoch": 0.9591836734693877, + "grad_norm": 0.5279243588447571, + "learning_rate": 2.0616438356164386e-06, + "loss": 0.3086137056350708, + "mean_token_accuracy": 0.9153400510549545, + "num_tokens": 13594286.0, + "step": 7050 + }, + { + "entropy": 0.3001719556748867, + "epoch": 0.9605442176870749, + "grad_norm": 0.5925727486610413, + "learning_rate": 1.9931506849315068e-06, + "loss": 0.29816312789916993, + "mean_token_accuracy": 0.9233756989240647, + "num_tokens": 13612101.0, + "step": 7060 + }, + { + "entropy": 0.2860943116247654, + "epoch": 0.9619047619047619, + "grad_norm": 0.4324999749660492, + "learning_rate": 1.9246575342465754e-06, + "loss": 0.28548643589019773, + "mean_token_accuracy": 0.9229339271783829, + "num_tokens": 13632308.0, + "step": 7070 + }, + { + "entropy": 0.2696798574179411, + "epoch": 0.963265306122449, + "grad_norm": 0.47976070642471313, + "learning_rate": 1.856164383561644e-06, + "loss": 0.2697856903076172, + "mean_token_accuracy": 0.9258852988481522, + "num_tokens": 13652430.0, + "step": 7080 + }, + { + "entropy": 0.3074658364057541, + "epoch": 0.964625850340136, + "grad_norm": 0.5789731740951538, + "learning_rate": 1.7876712328767124e-06, + "loss": 0.30829975605010984, + "mean_token_accuracy": 0.9198275089263916, + "num_tokens": 13670932.0, + "step": 7090 + }, + { + "entropy": 0.2700748883187771, + "epoch": 0.9659863945578231, + "grad_norm": 0.5600951313972473, + "learning_rate": 1.7191780821917809e-06, + "loss": 0.2561317205429077, + "mean_token_accuracy": 0.9302997469902039, + "num_tokens": 13689050.0, + "step": 7100 + }, + { + "entropy": 0.25459411814808847, + "epoch": 0.9673469387755103, + "grad_norm": 0.4180262088775635, + "learning_rate": 1.6506849315068493e-06, + "loss": 0.26971304416656494, + "mean_token_accuracy": 0.9278194218873977, + "num_tokens": 13707330.0, + "step": 7110 + }, + { + "entropy": 0.3380099833011627, + "epoch": 0.9687074829931973, + "grad_norm": 0.3550525903701782, + "learning_rate": 1.5821917808219177e-06, + "loss": 0.3356317043304443, + "mean_token_accuracy": 0.9100026935338974, + "num_tokens": 13729267.0, + "step": 7120 + }, + { + "entropy": 0.30718842446804046, + "epoch": 0.9700680272108844, + "grad_norm": 0.3944030702114105, + "learning_rate": 1.5136986301369863e-06, + "loss": 0.3045744180679321, + "mean_token_accuracy": 0.9190286636352539, + "num_tokens": 13749923.0, + "step": 7130 + }, + { + "entropy": 0.2982238419353962, + "epoch": 0.9714285714285714, + "grad_norm": 0.5965185165405273, + "learning_rate": 1.4452054794520547e-06, + "loss": 0.2974499225616455, + "mean_token_accuracy": 0.9249072134494781, + "num_tokens": 13766285.0, + "step": 7140 + }, + { + "entropy": 0.30790726840496063, + "epoch": 0.9727891156462585, + "grad_norm": 0.47756844758987427, + "learning_rate": 1.3767123287671234e-06, + "loss": 0.3204644203186035, + "mean_token_accuracy": 0.9175480782985688, + "num_tokens": 13785433.0, + "step": 7150 + }, + { + "entropy": 0.34362565353512764, + "epoch": 0.9741496598639455, + "grad_norm": 0.4054695963859558, + "learning_rate": 1.3082191780821918e-06, + "loss": 0.3412818431854248, + "mean_token_accuracy": 0.9061507374048233, + "num_tokens": 13807369.0, + "step": 7160 + }, + { + "entropy": 0.2839637853205204, + "epoch": 0.9755102040816327, + "grad_norm": 0.5588259100914001, + "learning_rate": 1.2397260273972604e-06, + "loss": 0.27560198307037354, + "mean_token_accuracy": 0.9249170988798141, + "num_tokens": 13827276.0, + "step": 7170 + }, + { + "entropy": 0.343147137016058, + "epoch": 0.9768707482993197, + "grad_norm": 0.8266026973724365, + "learning_rate": 1.1712328767123288e-06, + "loss": 0.36100358963012696, + "mean_token_accuracy": 0.9057711988687516, + "num_tokens": 13845606.0, + "step": 7180 + }, + { + "entropy": 0.35489280000329015, + "epoch": 0.9782312925170068, + "grad_norm": 0.4694916605949402, + "learning_rate": 1.1027397260273972e-06, + "loss": 0.3540391445159912, + "mean_token_accuracy": 0.913532018661499, + "num_tokens": 13865551.0, + "step": 7190 + }, + { + "entropy": 0.33037103042006494, + "epoch": 0.9795918367346939, + "grad_norm": 0.5188440680503845, + "learning_rate": 1.0342465753424659e-06, + "loss": 0.3330535888671875, + "mean_token_accuracy": 0.9100096374750137, + "num_tokens": 13884552.0, + "step": 7200 + }, + { + "entropy": 0.3238568499684334, + "epoch": 0.9809523809523809, + "grad_norm": 0.5132967233657837, + "learning_rate": 9.657534246575343e-07, + "loss": 0.3343978404998779, + "mean_token_accuracy": 0.9120694696903229, + "num_tokens": 13905871.0, + "step": 7210 + }, + { + "entropy": 0.2768064573407173, + "epoch": 0.9823129251700681, + "grad_norm": 0.5923624038696289, + "learning_rate": 8.972602739726027e-07, + "loss": 0.2772003412246704, + "mean_token_accuracy": 0.9270251214504241, + "num_tokens": 13924266.0, + "step": 7220 + }, + { + "entropy": 0.36717668995261193, + "epoch": 0.9836734693877551, + "grad_norm": 0.47369247674942017, + "learning_rate": 8.287671232876713e-07, + "loss": 0.3650016784667969, + "mean_token_accuracy": 0.9068687915802002, + "num_tokens": 13944354.0, + "step": 7230 + }, + { + "entropy": 0.286948836222291, + "epoch": 0.9850340136054422, + "grad_norm": 0.5255698561668396, + "learning_rate": 7.602739726027397e-07, + "loss": 0.2726752519607544, + "mean_token_accuracy": 0.925079733133316, + "num_tokens": 13963359.0, + "step": 7240 + }, + { + "entropy": 0.3442313067615032, + "epoch": 0.9863945578231292, + "grad_norm": 0.47418609261512756, + "learning_rate": 6.917808219178082e-07, + "loss": 0.35138564109802245, + "mean_token_accuracy": 0.9129589021205902, + "num_tokens": 13981101.0, + "step": 7250 + }, + { + "epoch": 0.9863945578231292, + "eval_entropy": 0.32534752316772936, + "eval_loss": 0.32265961170196533, + "eval_mean_token_accuracy": 0.9144763757785161, + "eval_num_tokens": 13981101.0, + "eval_runtime": 40.4232, + "eval_samples_per_second": 14.843, + "eval_steps_per_second": 7.421, + "step": 7250 + } + ], + "logging_steps": 10, + "max_steps": 7350, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 250, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.785795654131794e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-7250/training_args.bin b/checkpoint-7250/training_args.bin new file mode 100644 index 0000000..7178ddc --- /dev/null +++ b/checkpoint-7250/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b54c85194e9ec9f646206be51009ed347eb345ba25a878f68633e3a3a2bb102 +size 5841 diff --git a/checkpoint-7350/README.md b/checkpoint-7350/README.md new file mode 100644 index 0000000..63ca3ea --- /dev/null +++ b/checkpoint-7350/README.md @@ -0,0 +1,209 @@ +--- +base_model: HuggingFaceTB/SmolLM3-3B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:HuggingFaceTB/SmolLM3-3B +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.21.1 \ No newline at end of file diff --git a/checkpoint-7350/adapter_config.json b/checkpoint-7350/adapter_config.json new file mode 100644 index 0000000..284088d --- /dev/null +++ b/checkpoint-7350/adapter_config.json @@ -0,0 +1,51 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "HuggingFaceTB/SmolLM3-3B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "kasa_config": null, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.21.1", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/checkpoint-7350/adapter_model.safetensors b/checkpoint-7350/adapter_model.safetensors new file mode 100644 index 0000000..dc64993 --- /dev/null +++ b/checkpoint-7350/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7028124bee7fd70d390c977b6b3dd5af197879500c92e4ecd46841830572aa5 +size 60523464 diff --git a/checkpoint-7350/chat_template.jinja b/checkpoint-7350/chat_template.jinja new file mode 100644 index 0000000..e01e3a1 --- /dev/null +++ b/checkpoint-7350/chat_template.jinja @@ -0,0 +1,94 @@ +{# ───── defaults ───── #} +{%- if enable_thinking is not defined -%} +{%- set enable_thinking = true -%} +{%- endif -%} + +{# ───── reasoning mode ───── #} +{%- if enable_thinking -%} + {%- set reasoning_mode = "/think" -%} +{%- else -%} + {%- set reasoning_mode = "/no_think" -%} +{%- endif -%} + +{# ───── header (system message) ───── #} +{{- "<|im_start|>system\n" -}} + +{%- if messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} + {%- if "/no_think" in system_message -%} + {%- set reasoning_mode = "/no_think" -%} + {%- elif "/think" in system_message -%} + {%- set reasoning_mode = "/think" -%} + {%- endif -%} + {%- set custom_instructions = system_message.replace("/no_think", "").replace("/think", "").rstrip() -%} +{%- endif -%} + +{%- if "/system_override" in system_message -%} + {{- custom_instructions.replace("/system_override", "").rstrip() -}} + {{- "<|im_end|>\n" -}} +{%- else -%} + {{- "## Metadata\n\n" -}} + {{- "Knowledge Cutoff Date: June 2025\n" -}} + {%- set today = strftime_now("%d %B %Y") -%} + {{- "Today Date: " ~ today ~ "\n" -}} + {{- "Reasoning Mode: " + reasoning_mode + "\n\n" -}} + + {{- "## Custom Instructions\n\n" -}} + {%- if custom_instructions -%} + {{- custom_instructions + "\n\n" -}} + {%- elif reasoning_mode == "/think" -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracking, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: Thought section Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion.\n\n" -}} + {%- else -%} + {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face.\n\n" -}} + {%- endif -%} + + {%- if xml_tools or python_tools or tools -%} + {{- "### Tools\n\n" -}} + {%- if xml_tools or tools -%} + {%- if tools -%} + {%- set xml_tools = tools -%} + {%- endif -%} + {%- set ns = namespace(xml_tool_string="You may call one or more functions to assist with the user query.\nYou are provided with function signatures within XML tags:\n\n\n") -%} + {%- for tool in xml_tools[:] -%} {# The slicing makes sure that xml_tools is a list #} + {%- set ns.xml_tool_string = ns.xml_tool_string ~ (tool | string) ~ "\n" -%} + {%- endfor -%} + {%- set xml_tool_string = ns.xml_tool_string + "\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n" -%} + {{- xml_tool_string -}} + {%- endif -%} + {%- if python_tools -%} + {%- set ns = namespace(python_tool_string="When you send a message containing Python code between '' and '' tags, it will be executed in a stateful Jupyter notebook environment, and you will then be given the output to continued reasoning in an agentic loop.\n\nYou can use the following tools in your python code like regular functions:\n\n") -%} + {%- for tool in python_tools[:] -%} {# The slicing makes sure that python_tools is a list #} + {%- set ns.python_tool_string = ns.python_tool_string ~ (tool | string) ~ "\n" -%} + {%- endfor -%} + {%- set python_tool_string = ns.python_tool_string + "\n\nThe state persists between code executions: so variables that you define in one step are still available thereafter." -%} + {{- python_tool_string -}} + {%- endif -%} + {{- "\n\n" -}} + {{- "<|im_end|>\n" -}} + {%- endif -%} +{%- endif -%} +{# ───── main loop ───── #} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{ "<|im_start|>" + message.role + "\n" + content + "<|im_end|>\n" }} + {%- elif message.role == "assistant" -%} + {% generation %} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" + content.lstrip("\n") + "<|im_end|>\n" }} + {%- endif -%} + {% endgeneration %} + {%- elif message.role == "tool" -%} + {{ "<|im_start|>" + "user\n" + content + "<|im_end|>\n" }} + {%- endif -%} +{%- endfor -%} +{# ───── generation prompt ───── #} +{%- if add_generation_prompt -%} + {%- if reasoning_mode == "/think" -%} + {{ "<|im_start|>assistant\n" }} + {%- else -%} + {{ "<|im_start|>assistant\n" + "\n\n\n" }} + {%- endif -%} +{%- endif -%} \ No newline at end of file diff --git a/checkpoint-7350/optimizer.pt b/checkpoint-7350/optimizer.pt new file mode 100644 index 0000000..e666e3e --- /dev/null +++ b/checkpoint-7350/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc0a5e6485831d7de5a30b2af276e397977179d41235f8b50469431b76bc1e3b +size 121338699 diff --git a/checkpoint-7350/rng_state.pth b/checkpoint-7350/rng_state.pth new file mode 100644 index 0000000..4cb6841 --- /dev/null +++ b/checkpoint-7350/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900b02d17efd529ceee92c1c4f31d650d3f1169076d80bc628961de99e5b67fa +size 14645 diff --git a/checkpoint-7350/scheduler.pt b/checkpoint-7350/scheduler.pt new file mode 100644 index 0000000..2c25cbc --- /dev/null +++ b/checkpoint-7350/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:315b43904fc68d9afde42fd7166dc61789434f8e4c3d178d2fc702a1b37585f8 +size 1465 diff --git a/checkpoint-7350/tokenizer.json b/checkpoint-7350/tokenizer.json new file mode 100644 index 0000000..074fa66 --- /dev/null +++ b/checkpoint-7350/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b6a500b662a34eb3f0374db856ba4ad7de4c81040571d78dc0d357238930005 +size 17208819 diff --git a/checkpoint-7350/tokenizer_config.json b/checkpoint-7350/tokenizer_config.json new file mode 100644 index 0000000..8804a73 --- /dev/null +++ b/checkpoint-7350/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": true, + "eos_token": "<|im_end|>", + "fast": false, + "is_local": false, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|im_end|>", + "tokenizer_class": "TokenizersBackend" +} diff --git a/checkpoint-7350/trainer_state.json b/checkpoint-7350/trainer_state.json new file mode 100644 index 0000000..38e5a70 --- /dev/null +++ b/checkpoint-7350/trainer_state.json @@ -0,0 +1,7714 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 250, + "global_step": 7350, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.3966208942234516, + "epoch": 0.0013605442176870747, + "grad_norm": 1.5996246337890625, + "learning_rate": 9e-06, + "loss": 0.8333851814270019, + "mean_token_accuracy": 0.8561436295509338, + "num_tokens": 19866.0, + "step": 10 + }, + { + "entropy": 0.34238167852163315, + "epoch": 0.0027210884353741495, + "grad_norm": 1.296663522720337, + "learning_rate": 1.9e-05, + "loss": 0.7776159763336181, + "mean_token_accuracy": 0.8674830228090287, + "num_tokens": 39758.0, + "step": 20 + }, + { + "entropy": 0.42244544103741644, + "epoch": 0.004081632653061225, + "grad_norm": 0.7037413716316223, + "learning_rate": 2.9e-05, + "loss": 0.7272251605987549, + "mean_token_accuracy": 0.8528120934963226, + "num_tokens": 60338.0, + "step": 30 + }, + { + "entropy": 0.40994860902428626, + "epoch": 0.005442176870748299, + "grad_norm": 0.47693488001823425, + "learning_rate": 3.9000000000000006e-05, + "loss": 0.5927484512329102, + "mean_token_accuracy": 0.865676474571228, + "num_tokens": 79139.0, + "step": 40 + }, + { + "entropy": 0.4373441830277443, + "epoch": 0.006802721088435374, + "grad_norm": 0.4747143089771271, + "learning_rate": 4.9e-05, + "loss": 0.515117883682251, + "mean_token_accuracy": 0.8772250235080719, + "num_tokens": 98338.0, + "step": 50 + }, + { + "entropy": 0.5012069299817086, + "epoch": 0.00816326530612245, + "grad_norm": 0.5569216012954712, + "learning_rate": 4.9938356164383565e-05, + "loss": 0.5295265197753907, + "mean_token_accuracy": 0.8785856813192368, + "num_tokens": 115658.0, + "step": 60 + }, + { + "entropy": 0.4689371645450592, + "epoch": 0.009523809523809525, + "grad_norm": 0.2908697724342346, + "learning_rate": 4.986986301369863e-05, + "loss": 0.42938008308410647, + "mean_token_accuracy": 0.8960865825414658, + "num_tokens": 135179.0, + "step": 70 + }, + { + "entropy": 0.4583378002047539, + "epoch": 0.010884353741496598, + "grad_norm": 0.402289479970932, + "learning_rate": 4.9801369863013704e-05, + "loss": 0.4500479221343994, + "mean_token_accuracy": 0.8930670320987701, + "num_tokens": 154492.0, + "step": 80 + }, + { + "entropy": 0.45134490057826043, + "epoch": 0.012244897959183673, + "grad_norm": 0.4460085928440094, + "learning_rate": 4.973287671232877e-05, + "loss": 0.46530923843383787, + "mean_token_accuracy": 0.8883364647626877, + "num_tokens": 172732.0, + "step": 90 + }, + { + "entropy": 0.421974216401577, + "epoch": 0.013605442176870748, + "grad_norm": 0.4011005759239197, + "learning_rate": 4.966438356164384e-05, + "loss": 0.4124259948730469, + "mean_token_accuracy": 0.8977983146905899, + "num_tokens": 192922.0, + "step": 100 + }, + { + "entropy": 0.38947183191776275, + "epoch": 0.014965986394557823, + "grad_norm": 0.38355737924575806, + "learning_rate": 4.959589041095891e-05, + "loss": 0.3860264778137207, + "mean_token_accuracy": 0.9041510850191117, + "num_tokens": 210310.0, + "step": 110 + }, + { + "entropy": 0.474651126563549, + "epoch": 0.0163265306122449, + "grad_norm": 0.45214158296585083, + "learning_rate": 4.9527397260273975e-05, + "loss": 0.48367671966552733, + "mean_token_accuracy": 0.8782862812280655, + "num_tokens": 229495.0, + "step": 120 + }, + { + "entropy": 0.34797751232981683, + "epoch": 0.017687074829931974, + "grad_norm": 0.40691450238227844, + "learning_rate": 4.945890410958904e-05, + "loss": 0.32767112255096437, + "mean_token_accuracy": 0.9148984670639038, + "num_tokens": 247661.0, + "step": 130 + }, + { + "entropy": 0.40742513462901114, + "epoch": 0.01904761904761905, + "grad_norm": 0.45683032274246216, + "learning_rate": 4.939041095890411e-05, + "loss": 0.40177297592163086, + "mean_token_accuracy": 0.8949861347675323, + "num_tokens": 268110.0, + "step": 140 + }, + { + "entropy": 0.4079647034406662, + "epoch": 0.02040816326530612, + "grad_norm": 0.32131096720695496, + "learning_rate": 4.932191780821918e-05, + "loss": 0.4140042781829834, + "mean_token_accuracy": 0.8966580182313919, + "num_tokens": 286814.0, + "step": 150 + }, + { + "entropy": 0.38506424576044085, + "epoch": 0.021768707482993196, + "grad_norm": 0.4252924919128418, + "learning_rate": 4.9253424657534246e-05, + "loss": 0.3659359931945801, + "mean_token_accuracy": 0.9046422451734543, + "num_tokens": 306080.0, + "step": 160 + }, + { + "entropy": 0.42693412154912946, + "epoch": 0.02312925170068027, + "grad_norm": 0.49110063910484314, + "learning_rate": 4.918493150684932e-05, + "loss": 0.4523786067962646, + "mean_token_accuracy": 0.8923947751522064, + "num_tokens": 324055.0, + "step": 170 + }, + { + "entropy": 0.43308429792523384, + "epoch": 0.024489795918367346, + "grad_norm": 0.39848849177360535, + "learning_rate": 4.9116438356164385e-05, + "loss": 0.44591054916381834, + "mean_token_accuracy": 0.8914232641458512, + "num_tokens": 342105.0, + "step": 180 + }, + { + "entropy": 0.3832189217209816, + "epoch": 0.02585034013605442, + "grad_norm": 0.4442504346370697, + "learning_rate": 4.904794520547946e-05, + "loss": 0.38175168037414553, + "mean_token_accuracy": 0.9029964536428452, + "num_tokens": 360113.0, + "step": 190 + }, + { + "entropy": 0.35205388739705085, + "epoch": 0.027210884353741496, + "grad_norm": 0.3656632602214813, + "learning_rate": 4.8979452054794524e-05, + "loss": 0.3521461009979248, + "mean_token_accuracy": 0.9092369765043259, + "num_tokens": 377600.0, + "step": 200 + }, + { + "entropy": 0.37196114510297773, + "epoch": 0.02857142857142857, + "grad_norm": 0.44447311758995056, + "learning_rate": 4.8910958904109597e-05, + "loss": 0.3701155185699463, + "mean_token_accuracy": 0.9007940888404846, + "num_tokens": 398788.0, + "step": 210 + }, + { + "entropy": 0.38314035460352897, + "epoch": 0.029931972789115645, + "grad_norm": 0.3941226303577423, + "learning_rate": 4.884246575342466e-05, + "loss": 0.3909285068511963, + "mean_token_accuracy": 0.8981142967939377, + "num_tokens": 419141.0, + "step": 220 + }, + { + "entropy": 0.3607197545468807, + "epoch": 0.031292517006802724, + "grad_norm": 0.3219183683395386, + "learning_rate": 4.877397260273973e-05, + "loss": 0.37545225620269773, + "mean_token_accuracy": 0.9047404289245605, + "num_tokens": 437786.0, + "step": 230 + }, + { + "entropy": 0.3778640143573284, + "epoch": 0.0326530612244898, + "grad_norm": 0.49071386456489563, + "learning_rate": 4.8705479452054795e-05, + "loss": 0.3791273832321167, + "mean_token_accuracy": 0.9014792263507843, + "num_tokens": 456747.0, + "step": 240 + }, + { + "entropy": 0.4029653251171112, + "epoch": 0.034013605442176874, + "grad_norm": 0.5018599629402161, + "learning_rate": 4.863698630136986e-05, + "loss": 0.4035783290863037, + "mean_token_accuracy": 0.8966485053300858, + "num_tokens": 475585.0, + "step": 250 + }, + { + "epoch": 0.034013605442176874, + "eval_entropy": 0.3818362482388814, + "eval_loss": 0.37777262926101685, + "eval_mean_token_accuracy": 0.9024270363648732, + "eval_num_tokens": 475585.0, + "eval_runtime": 39.4306, + "eval_samples_per_second": 15.217, + "eval_steps_per_second": 7.608, + "step": 250 + }, + { + "entropy": 0.348661782592535, + "epoch": 0.03537414965986395, + "grad_norm": 0.45828038454055786, + "learning_rate": 4.8568493150684934e-05, + "loss": 0.3377650737762451, + "mean_token_accuracy": 0.9096396565437317, + "num_tokens": 494035.0, + "step": 260 + }, + { + "entropy": 0.3596964508295059, + "epoch": 0.036734693877551024, + "grad_norm": 0.4269905984401703, + "learning_rate": 4.85e-05, + "loss": 0.3601596593856812, + "mean_token_accuracy": 0.9069953888654709, + "num_tokens": 512948.0, + "step": 270 + }, + { + "entropy": 0.3371978670358658, + "epoch": 0.0380952380952381, + "grad_norm": 0.3644143342971802, + "learning_rate": 4.843150684931507e-05, + "loss": 0.33878345489501954, + "mean_token_accuracy": 0.9105350911617279, + "num_tokens": 532063.0, + "step": 280 + }, + { + "entropy": 0.35116567388176917, + "epoch": 0.03945578231292517, + "grad_norm": 0.3923342823982239, + "learning_rate": 4.836301369863014e-05, + "loss": 0.3478651523590088, + "mean_token_accuracy": 0.9095128774642944, + "num_tokens": 549043.0, + "step": 290 + }, + { + "entropy": 0.34563929960131645, + "epoch": 0.04081632653061224, + "grad_norm": 0.3414367735385895, + "learning_rate": 4.829452054794521e-05, + "loss": 0.33946025371551514, + "mean_token_accuracy": 0.9125081688165665, + "num_tokens": 567832.0, + "step": 300 + }, + { + "entropy": 0.37908021062612535, + "epoch": 0.04217687074829932, + "grad_norm": 0.4030672311782837, + "learning_rate": 4.822602739726028e-05, + "loss": 0.38597617149353025, + "mean_token_accuracy": 0.900176802277565, + "num_tokens": 588457.0, + "step": 310 + }, + { + "entropy": 0.38542170226573946, + "epoch": 0.04353741496598639, + "grad_norm": 0.40530654788017273, + "learning_rate": 4.8157534246575344e-05, + "loss": 0.39215831756591796, + "mean_token_accuracy": 0.8950019955635071, + "num_tokens": 608720.0, + "step": 320 + }, + { + "entropy": 0.40009846463799476, + "epoch": 0.044897959183673466, + "grad_norm": 0.5024587512016296, + "learning_rate": 4.808904109589042e-05, + "loss": 0.41683034896850585, + "mean_token_accuracy": 0.893940594792366, + "num_tokens": 628467.0, + "step": 330 + }, + { + "entropy": 0.3844646617770195, + "epoch": 0.04625850340136054, + "grad_norm": 0.37910065054893494, + "learning_rate": 4.802054794520548e-05, + "loss": 0.3854831695556641, + "mean_token_accuracy": 0.901634418964386, + "num_tokens": 648387.0, + "step": 340 + }, + { + "entropy": 0.3825081571936607, + "epoch": 0.047619047619047616, + "grad_norm": 0.455219030380249, + "learning_rate": 4.795205479452055e-05, + "loss": 0.38410975933074953, + "mean_token_accuracy": 0.9046315401792526, + "num_tokens": 665488.0, + "step": 350 + }, + { + "entropy": 0.3589709296822548, + "epoch": 0.04897959183673469, + "grad_norm": 0.3238779902458191, + "learning_rate": 4.7883561643835615e-05, + "loss": 0.35581753253936765, + "mean_token_accuracy": 0.9074301540851593, + "num_tokens": 684284.0, + "step": 360 + }, + { + "entropy": 0.3434913203120232, + "epoch": 0.050340136054421766, + "grad_norm": 0.349220871925354, + "learning_rate": 4.781506849315069e-05, + "loss": 0.33537404537200927, + "mean_token_accuracy": 0.9093591779470444, + "num_tokens": 702876.0, + "step": 370 + }, + { + "entropy": 0.3294819757342339, + "epoch": 0.05170068027210884, + "grad_norm": 0.35057687759399414, + "learning_rate": 4.7746575342465754e-05, + "loss": 0.33455510139465333, + "mean_token_accuracy": 0.9156662911176682, + "num_tokens": 720840.0, + "step": 380 + }, + { + "entropy": 0.3731862798333168, + "epoch": 0.053061224489795916, + "grad_norm": 0.4629971981048584, + "learning_rate": 4.767808219178083e-05, + "loss": 0.3719052791595459, + "mean_token_accuracy": 0.903752988576889, + "num_tokens": 739745.0, + "step": 390 + }, + { + "entropy": 0.3066830538213253, + "epoch": 0.05442176870748299, + "grad_norm": 0.4535863697528839, + "learning_rate": 4.760958904109589e-05, + "loss": 0.30530486106872556, + "mean_token_accuracy": 0.9210470616817474, + "num_tokens": 757386.0, + "step": 400 + }, + { + "entropy": 0.33964752927422526, + "epoch": 0.055782312925170066, + "grad_norm": 0.326674222946167, + "learning_rate": 4.7541095890410966e-05, + "loss": 0.3501870632171631, + "mean_token_accuracy": 0.9105898469686509, + "num_tokens": 774681.0, + "step": 410 + }, + { + "entropy": 0.34910165071487426, + "epoch": 0.05714285714285714, + "grad_norm": 0.4973655939102173, + "learning_rate": 4.747260273972603e-05, + "loss": 0.35225639343261717, + "mean_token_accuracy": 0.9095459759235383, + "num_tokens": 794694.0, + "step": 420 + }, + { + "entropy": 0.3515663854777813, + "epoch": 0.058503401360544216, + "grad_norm": 0.3093283772468567, + "learning_rate": 4.74041095890411e-05, + "loss": 0.3447835922241211, + "mean_token_accuracy": 0.9086586356163024, + "num_tokens": 813789.0, + "step": 430 + }, + { + "entropy": 0.3582890912890434, + "epoch": 0.05986394557823129, + "grad_norm": 0.29822006821632385, + "learning_rate": 4.733561643835617e-05, + "loss": 0.3475156784057617, + "mean_token_accuracy": 0.9078772723674774, + "num_tokens": 833162.0, + "step": 440 + }, + { + "entropy": 0.3461380168795586, + "epoch": 0.061224489795918366, + "grad_norm": 0.350153386592865, + "learning_rate": 4.726712328767123e-05, + "loss": 0.3462402820587158, + "mean_token_accuracy": 0.9049428641796112, + "num_tokens": 853866.0, + "step": 450 + }, + { + "entropy": 0.30043474584817886, + "epoch": 0.06258503401360545, + "grad_norm": 0.3542056679725647, + "learning_rate": 4.71986301369863e-05, + "loss": 0.3105747222900391, + "mean_token_accuracy": 0.9163908660411835, + "num_tokens": 873028.0, + "step": 460 + }, + { + "entropy": 0.3481592245399952, + "epoch": 0.06394557823129252, + "grad_norm": 0.3953518271446228, + "learning_rate": 4.713013698630137e-05, + "loss": 0.3386308908462524, + "mean_token_accuracy": 0.9098592340946198, + "num_tokens": 891721.0, + "step": 470 + }, + { + "entropy": 0.3410739690065384, + "epoch": 0.0653061224489796, + "grad_norm": 0.4479163885116577, + "learning_rate": 4.706164383561644e-05, + "loss": 0.3482985496520996, + "mean_token_accuracy": 0.9046086817979813, + "num_tokens": 913549.0, + "step": 480 + }, + { + "entropy": 0.36930672004818915, + "epoch": 0.06666666666666667, + "grad_norm": 0.3130194842815399, + "learning_rate": 4.699315068493151e-05, + "loss": 0.36626100540161133, + "mean_token_accuracy": 0.9040644109249115, + "num_tokens": 932950.0, + "step": 490 + }, + { + "entropy": 0.397816364467144, + "epoch": 0.06802721088435375, + "grad_norm": 0.45680442452430725, + "learning_rate": 4.692465753424658e-05, + "loss": 0.40360026359558104, + "mean_token_accuracy": 0.9006230294704437, + "num_tokens": 951066.0, + "step": 500 + }, + { + "epoch": 0.06802721088435375, + "eval_entropy": 0.3723491595685482, + "eval_loss": 0.3646194040775299, + "eval_mean_token_accuracy": 0.9050061551729838, + "eval_num_tokens": 951066.0, + "eval_runtime": 39.4141, + "eval_samples_per_second": 15.223, + "eval_steps_per_second": 7.611, + "step": 500 + }, + { + "entropy": 0.4003289297223091, + "epoch": 0.06938775510204082, + "grad_norm": 0.37886762619018555, + "learning_rate": 4.685616438356165e-05, + "loss": 0.39484944343566897, + "mean_token_accuracy": 0.8952762842178345, + "num_tokens": 971117.0, + "step": 510 + }, + { + "entropy": 0.3746891237795353, + "epoch": 0.0707482993197279, + "grad_norm": 0.39949730038642883, + "learning_rate": 4.678767123287671e-05, + "loss": 0.3683069467544556, + "mean_token_accuracy": 0.903772982954979, + "num_tokens": 990103.0, + "step": 520 + }, + { + "entropy": 0.31653870418667795, + "epoch": 0.07210884353741497, + "grad_norm": 0.33652064204216003, + "learning_rate": 4.6719178082191786e-05, + "loss": 0.3186372995376587, + "mean_token_accuracy": 0.9144774466753006, + "num_tokens": 1010804.0, + "step": 530 + }, + { + "entropy": 0.3711325004696846, + "epoch": 0.07346938775510205, + "grad_norm": 0.3641034662723541, + "learning_rate": 4.665068493150685e-05, + "loss": 0.3789762735366821, + "mean_token_accuracy": 0.9045424968004226, + "num_tokens": 1029560.0, + "step": 540 + }, + { + "entropy": 0.37221564501523974, + "epoch": 0.07482993197278912, + "grad_norm": 0.4707719087600708, + "learning_rate": 4.658219178082192e-05, + "loss": 0.3638460636138916, + "mean_token_accuracy": 0.9029160022735596, + "num_tokens": 1049933.0, + "step": 550 + }, + { + "entropy": 0.3466314420104027, + "epoch": 0.0761904761904762, + "grad_norm": 0.5033849477767944, + "learning_rate": 4.6513698630136984e-05, + "loss": 0.352337908744812, + "mean_token_accuracy": 0.9113892763853073, + "num_tokens": 1067783.0, + "step": 560 + }, + { + "entropy": 0.277361162006855, + "epoch": 0.07755102040816327, + "grad_norm": 0.3210407793521881, + "learning_rate": 4.644520547945206e-05, + "loss": 0.28125414848327634, + "mean_token_accuracy": 0.922634044289589, + "num_tokens": 1088771.0, + "step": 570 + }, + { + "entropy": 0.3498281344771385, + "epoch": 0.07891156462585033, + "grad_norm": 0.43742120265960693, + "learning_rate": 4.637671232876712e-05, + "loss": 0.3471531391143799, + "mean_token_accuracy": 0.9098026067018509, + "num_tokens": 1108964.0, + "step": 580 + }, + { + "entropy": 0.34425888285040857, + "epoch": 0.08027210884353742, + "grad_norm": 0.32993149757385254, + "learning_rate": 4.6308219178082196e-05, + "loss": 0.35619263648986815, + "mean_token_accuracy": 0.9035626143217087, + "num_tokens": 1130205.0, + "step": 590 + }, + { + "entropy": 0.3691210441291332, + "epoch": 0.08163265306122448, + "grad_norm": 0.4786720871925354, + "learning_rate": 4.623972602739726e-05, + "loss": 0.3622753381729126, + "mean_token_accuracy": 0.9080120652914048, + "num_tokens": 1150314.0, + "step": 600 + }, + { + "entropy": 0.389055623114109, + "epoch": 0.08299319727891157, + "grad_norm": 0.3758779764175415, + "learning_rate": 4.6171232876712335e-05, + "loss": 0.3914887189865112, + "mean_token_accuracy": 0.901269006729126, + "num_tokens": 1168468.0, + "step": 610 + }, + { + "entropy": 0.3246720366179943, + "epoch": 0.08435374149659863, + "grad_norm": 0.4306440055370331, + "learning_rate": 4.61027397260274e-05, + "loss": 0.32867746353149413, + "mean_token_accuracy": 0.9110509842634201, + "num_tokens": 1188023.0, + "step": 620 + }, + { + "entropy": 0.32952589765191076, + "epoch": 0.08571428571428572, + "grad_norm": 0.3729093670845032, + "learning_rate": 4.603424657534247e-05, + "loss": 0.3338437795639038, + "mean_token_accuracy": 0.9136817961931228, + "num_tokens": 1205997.0, + "step": 630 + }, + { + "entropy": 0.34219127148389816, + "epoch": 0.08707482993197278, + "grad_norm": 0.33127057552337646, + "learning_rate": 4.596575342465754e-05, + "loss": 0.3417861223220825, + "mean_token_accuracy": 0.9103910386562347, + "num_tokens": 1225286.0, + "step": 640 + }, + { + "entropy": 0.396539293974638, + "epoch": 0.08843537414965986, + "grad_norm": 0.40578168630599976, + "learning_rate": 4.5897260273972606e-05, + "loss": 0.4101886749267578, + "mean_token_accuracy": 0.8973281621932984, + "num_tokens": 1244108.0, + "step": 650 + }, + { + "entropy": 0.38693875148892404, + "epoch": 0.08979591836734693, + "grad_norm": 0.3798186779022217, + "learning_rate": 4.582876712328767e-05, + "loss": 0.3816530704498291, + "mean_token_accuracy": 0.9010074496269226, + "num_tokens": 1264868.0, + "step": 660 + }, + { + "entropy": 0.3229499511420727, + "epoch": 0.09115646258503401, + "grad_norm": 0.3040688931941986, + "learning_rate": 4.576027397260274e-05, + "loss": 0.3126213073730469, + "mean_token_accuracy": 0.9153517127037049, + "num_tokens": 1285486.0, + "step": 670 + }, + { + "entropy": 0.356215351074934, + "epoch": 0.09251700680272108, + "grad_norm": 0.4674758017063141, + "learning_rate": 4.569178082191781e-05, + "loss": 0.3552473306655884, + "mean_token_accuracy": 0.9089611619710922, + "num_tokens": 1303745.0, + "step": 680 + }, + { + "entropy": 0.40253775492310523, + "epoch": 0.09387755102040816, + "grad_norm": 0.42166969180107117, + "learning_rate": 4.562328767123288e-05, + "loss": 0.41041126251220705, + "mean_token_accuracy": 0.8952078282833099, + "num_tokens": 1324171.0, + "step": 690 + }, + { + "entropy": 0.34822252616286276, + "epoch": 0.09523809523809523, + "grad_norm": 0.3444162905216217, + "learning_rate": 4.555479452054795e-05, + "loss": 0.35946462154388426, + "mean_token_accuracy": 0.9083791345357894, + "num_tokens": 1343401.0, + "step": 700 + }, + { + "entropy": 0.32225404009222985, + "epoch": 0.09659863945578231, + "grad_norm": 0.4599993824958801, + "learning_rate": 4.5486301369863016e-05, + "loss": 0.32996740341186526, + "mean_token_accuracy": 0.9141956597566605, + "num_tokens": 1363084.0, + "step": 710 + }, + { + "entropy": 0.3677783004939556, + "epoch": 0.09795918367346938, + "grad_norm": 0.435357928276062, + "learning_rate": 4.541780821917808e-05, + "loss": 0.37146801948547364, + "mean_token_accuracy": 0.9019740343093872, + "num_tokens": 1381194.0, + "step": 720 + }, + { + "entropy": 0.33645370081067083, + "epoch": 0.09931972789115646, + "grad_norm": 0.42202693223953247, + "learning_rate": 4.5349315068493155e-05, + "loss": 0.33685150146484377, + "mean_token_accuracy": 0.9091235578060151, + "num_tokens": 1400075.0, + "step": 730 + }, + { + "entropy": 0.3678745202720165, + "epoch": 0.10068027210884353, + "grad_norm": 0.45290741324424744, + "learning_rate": 4.528082191780822e-05, + "loss": 0.35833916664123533, + "mean_token_accuracy": 0.9041007101535797, + "num_tokens": 1420258.0, + "step": 740 + }, + { + "entropy": 0.36107349693775176, + "epoch": 0.10204081632653061, + "grad_norm": 0.4961569309234619, + "learning_rate": 4.5212328767123294e-05, + "loss": 0.3725413799285889, + "mean_token_accuracy": 0.90306396484375, + "num_tokens": 1439035.0, + "step": 750 + }, + { + "epoch": 0.10204081632653061, + "eval_entropy": 0.34408375874161723, + "eval_loss": 0.3582100570201874, + "eval_mean_token_accuracy": 0.9064499346415201, + "eval_num_tokens": 1439035.0, + "eval_runtime": 39.3818, + "eval_samples_per_second": 15.235, + "eval_steps_per_second": 7.618, + "step": 750 + }, + { + "entropy": 0.3133292868733406, + "epoch": 0.10340136054421768, + "grad_norm": 0.6212469935417175, + "learning_rate": 4.514383561643836e-05, + "loss": 0.32587475776672364, + "mean_token_accuracy": 0.9168930113315582, + "num_tokens": 1455982.0, + "step": 760 + }, + { + "entropy": 0.3137072093784809, + "epoch": 0.10476190476190476, + "grad_norm": 0.3927687406539917, + "learning_rate": 4.5075342465753426e-05, + "loss": 0.3104274034500122, + "mean_token_accuracy": 0.919685822725296, + "num_tokens": 1474473.0, + "step": 770 + }, + { + "entropy": 0.39127727299928666, + "epoch": 0.10612244897959183, + "grad_norm": 0.47527554631233215, + "learning_rate": 4.500684931506849e-05, + "loss": 0.3964341640472412, + "mean_token_accuracy": 0.9029917776584625, + "num_tokens": 1492907.0, + "step": 780 + }, + { + "entropy": 0.36204536259174347, + "epoch": 0.10748299319727891, + "grad_norm": 0.38715481758117676, + "learning_rate": 4.4938356164383565e-05, + "loss": 0.3774503469467163, + "mean_token_accuracy": 0.9045975625514984, + "num_tokens": 1511281.0, + "step": 790 + }, + { + "entropy": 0.3681113593280315, + "epoch": 0.10884353741496598, + "grad_norm": 0.3628789782524109, + "learning_rate": 4.486986301369863e-05, + "loss": 0.3720179557800293, + "mean_token_accuracy": 0.903636223077774, + "num_tokens": 1531835.0, + "step": 800 + }, + { + "entropy": 0.3314031042158604, + "epoch": 0.11020408163265306, + "grad_norm": 0.34054824709892273, + "learning_rate": 4.4801369863013704e-05, + "loss": 0.3269455909729004, + "mean_token_accuracy": 0.9134721159934998, + "num_tokens": 1550387.0, + "step": 810 + }, + { + "entropy": 0.2866345688700676, + "epoch": 0.11156462585034013, + "grad_norm": 0.7705225944519043, + "learning_rate": 4.473287671232877e-05, + "loss": 0.2993777275085449, + "mean_token_accuracy": 0.9186601221561432, + "num_tokens": 1571096.0, + "step": 820 + }, + { + "entropy": 0.38486768677830696, + "epoch": 0.11292517006802721, + "grad_norm": 0.45260295271873474, + "learning_rate": 4.4664383561643836e-05, + "loss": 0.3849661350250244, + "mean_token_accuracy": 0.9009180992841721, + "num_tokens": 1590859.0, + "step": 830 + }, + { + "entropy": 0.38537542670965197, + "epoch": 0.11428571428571428, + "grad_norm": 0.3428184688091278, + "learning_rate": 4.459589041095891e-05, + "loss": 0.37544195652008056, + "mean_token_accuracy": 0.8985641360282898, + "num_tokens": 1612476.0, + "step": 840 + }, + { + "entropy": 0.3693395532667637, + "epoch": 0.11564625850340136, + "grad_norm": 0.48987624049186707, + "learning_rate": 4.4527397260273975e-05, + "loss": 0.37159268856048583, + "mean_token_accuracy": 0.9065731137990951, + "num_tokens": 1632486.0, + "step": 850 + }, + { + "entropy": 0.3446200601756573, + "epoch": 0.11700680272108843, + "grad_norm": 0.38551682233810425, + "learning_rate": 4.445890410958905e-05, + "loss": 0.340871262550354, + "mean_token_accuracy": 0.9108866393566132, + "num_tokens": 1651787.0, + "step": 860 + }, + { + "entropy": 0.3901903234422207, + "epoch": 0.11836734693877551, + "grad_norm": 0.44852742552757263, + "learning_rate": 4.439041095890411e-05, + "loss": 0.3871347665786743, + "mean_token_accuracy": 0.8977316886186599, + "num_tokens": 1672747.0, + "step": 870 + }, + { + "entropy": 0.35357545092701914, + "epoch": 0.11972789115646258, + "grad_norm": 0.6645485162734985, + "learning_rate": 4.432191780821918e-05, + "loss": 0.36792519092559817, + "mean_token_accuracy": 0.9066919326782227, + "num_tokens": 1692001.0, + "step": 880 + }, + { + "entropy": 0.4160300984978676, + "epoch": 0.12108843537414966, + "grad_norm": 0.4086649417877197, + "learning_rate": 4.4253424657534246e-05, + "loss": 0.4246100425720215, + "mean_token_accuracy": 0.8960783511400223, + "num_tokens": 1710300.0, + "step": 890 + }, + { + "entropy": 0.349920666962862, + "epoch": 0.12244897959183673, + "grad_norm": 0.5265305638313293, + "learning_rate": 4.418493150684932e-05, + "loss": 0.3457962989807129, + "mean_token_accuracy": 0.9070086061954499, + "num_tokens": 1729591.0, + "step": 900 + }, + { + "entropy": 0.32573084980249406, + "epoch": 0.12380952380952381, + "grad_norm": 0.45363783836364746, + "learning_rate": 4.4116438356164385e-05, + "loss": 0.32244632244110105, + "mean_token_accuracy": 0.9139437705278397, + "num_tokens": 1747204.0, + "step": 910 + }, + { + "entropy": 0.31426838859915734, + "epoch": 0.1251700680272109, + "grad_norm": 0.43578457832336426, + "learning_rate": 4.404794520547945e-05, + "loss": 0.31415719985961915, + "mean_token_accuracy": 0.919468006491661, + "num_tokens": 1765776.0, + "step": 920 + }, + { + "entropy": 0.3555323138833046, + "epoch": 0.12653061224489795, + "grad_norm": 0.41630497574806213, + "learning_rate": 4.3979452054794524e-05, + "loss": 0.36635076999664307, + "mean_token_accuracy": 0.9067310243844986, + "num_tokens": 1783746.0, + "step": 930 + }, + { + "entropy": 0.30401672422885895, + "epoch": 0.12789115646258503, + "grad_norm": 0.2979906499385834, + "learning_rate": 4.391095890410959e-05, + "loss": 0.2973449468612671, + "mean_token_accuracy": 0.91805320084095, + "num_tokens": 1805916.0, + "step": 940 + }, + { + "entropy": 0.32692537233233454, + "epoch": 0.1292517006802721, + "grad_norm": 0.415134459733963, + "learning_rate": 4.384246575342466e-05, + "loss": 0.3329088926315308, + "mean_token_accuracy": 0.9109864830970764, + "num_tokens": 1824061.0, + "step": 950 + }, + { + "entropy": 0.33077087327837945, + "epoch": 0.1306122448979592, + "grad_norm": 0.309216171503067, + "learning_rate": 4.377397260273973e-05, + "loss": 0.3365213871002197, + "mean_token_accuracy": 0.9099713116884232, + "num_tokens": 1845404.0, + "step": 960 + }, + { + "entropy": 0.3487712278962135, + "epoch": 0.13197278911564625, + "grad_norm": 0.46856454014778137, + "learning_rate": 4.3705479452054795e-05, + "loss": 0.3540360689163208, + "mean_token_accuracy": 0.9070927411317825, + "num_tokens": 1866651.0, + "step": 970 + }, + { + "entropy": 0.33622991368174554, + "epoch": 0.13333333333333333, + "grad_norm": 0.26888006925582886, + "learning_rate": 4.363698630136986e-05, + "loss": 0.345446252822876, + "mean_token_accuracy": 0.9077248781919479, + "num_tokens": 1886454.0, + "step": 980 + }, + { + "entropy": 0.3214356690645218, + "epoch": 0.1346938775510204, + "grad_norm": 0.3736676275730133, + "learning_rate": 4.3568493150684934e-05, + "loss": 0.30926587581634524, + "mean_token_accuracy": 0.9138902932405472, + "num_tokens": 1905432.0, + "step": 990 + }, + { + "entropy": 0.3608489066362381, + "epoch": 0.1360544217687075, + "grad_norm": 0.5109230279922485, + "learning_rate": 4.35e-05, + "loss": 0.3655172109603882, + "mean_token_accuracy": 0.906938511133194, + "num_tokens": 1925558.0, + "step": 1000 + }, + { + "epoch": 0.1360544217687075, + "eval_entropy": 0.34845481877525647, + "eval_loss": 0.3528926968574524, + "eval_mean_token_accuracy": 0.9076920866966247, + "eval_num_tokens": 1925558.0, + "eval_runtime": 39.6694, + "eval_samples_per_second": 15.125, + "eval_steps_per_second": 7.563, + "step": 1000 + }, + { + "entropy": 0.41318991109728814, + "epoch": 0.13741496598639455, + "grad_norm": 0.38739997148513794, + "learning_rate": 4.3431506849315066e-05, + "loss": 0.42798943519592286, + "mean_token_accuracy": 0.8936826854944229, + "num_tokens": 1944122.0, + "step": 1010 + }, + { + "entropy": 0.3004152253270149, + "epoch": 0.13877551020408163, + "grad_norm": 0.3833780586719513, + "learning_rate": 4.336301369863014e-05, + "loss": 0.3056038856506348, + "mean_token_accuracy": 0.9163402706384659, + "num_tokens": 1966172.0, + "step": 1020 + }, + { + "entropy": 0.33445904925465586, + "epoch": 0.1401360544217687, + "grad_norm": 0.4406833350658417, + "learning_rate": 4.3294520547945205e-05, + "loss": 0.3189589500427246, + "mean_token_accuracy": 0.9131261318922043, + "num_tokens": 1985410.0, + "step": 1030 + }, + { + "entropy": 0.3856867581605911, + "epoch": 0.1414965986394558, + "grad_norm": 0.32720640301704407, + "learning_rate": 4.322602739726028e-05, + "loss": 0.3763080358505249, + "mean_token_accuracy": 0.9028674513101578, + "num_tokens": 2004750.0, + "step": 1040 + }, + { + "entropy": 0.3344607546925545, + "epoch": 0.14285714285714285, + "grad_norm": 0.5312451720237732, + "learning_rate": 4.3157534246575344e-05, + "loss": 0.3377204656600952, + "mean_token_accuracy": 0.9150244653224945, + "num_tokens": 2022601.0, + "step": 1050 + }, + { + "entropy": 0.3213494800031185, + "epoch": 0.14421768707482993, + "grad_norm": 0.3459687829017639, + "learning_rate": 4.308904109589042e-05, + "loss": 0.3276652812957764, + "mean_token_accuracy": 0.9159560710191726, + "num_tokens": 2041553.0, + "step": 1060 + }, + { + "entropy": 0.4032025709748268, + "epoch": 0.145578231292517, + "grad_norm": 0.42302119731903076, + "learning_rate": 4.302054794520548e-05, + "loss": 0.40665340423583984, + "mean_token_accuracy": 0.8989956825971603, + "num_tokens": 2060111.0, + "step": 1070 + }, + { + "entropy": 0.30957555398344994, + "epoch": 0.1469387755102041, + "grad_norm": 0.540122389793396, + "learning_rate": 4.295205479452055e-05, + "loss": 0.31520886421203614, + "mean_token_accuracy": 0.9152782648801804, + "num_tokens": 2079453.0, + "step": 1080 + }, + { + "entropy": 0.37794227004051206, + "epoch": 0.14829931972789115, + "grad_norm": 0.34265509247779846, + "learning_rate": 4.2883561643835615e-05, + "loss": 0.3954782009124756, + "mean_token_accuracy": 0.9016141533851624, + "num_tokens": 2097734.0, + "step": 1090 + }, + { + "entropy": 0.34080238491296766, + "epoch": 0.14965986394557823, + "grad_norm": 0.40301233530044556, + "learning_rate": 4.281506849315069e-05, + "loss": 0.3306001901626587, + "mean_token_accuracy": 0.9116725891828537, + "num_tokens": 2115880.0, + "step": 1100 + }, + { + "entropy": 0.38523422852158545, + "epoch": 0.1510204081632653, + "grad_norm": 0.4232766330242157, + "learning_rate": 4.2746575342465754e-05, + "loss": 0.3976067304611206, + "mean_token_accuracy": 0.9017782896757126, + "num_tokens": 2134164.0, + "step": 1110 + }, + { + "entropy": 0.35867138430476186, + "epoch": 0.1523809523809524, + "grad_norm": 0.29137903451919556, + "learning_rate": 4.267808219178082e-05, + "loss": 0.34880263805389405, + "mean_token_accuracy": 0.9045450478792191, + "num_tokens": 2155503.0, + "step": 1120 + }, + { + "entropy": 0.4304305985569954, + "epoch": 0.15374149659863945, + "grad_norm": 0.4157155752182007, + "learning_rate": 4.260958904109589e-05, + "loss": 0.4243772029876709, + "mean_token_accuracy": 0.8891431093215942, + "num_tokens": 2174849.0, + "step": 1130 + }, + { + "entropy": 0.3275195978581905, + "epoch": 0.15510204081632653, + "grad_norm": 0.42673543095588684, + "learning_rate": 4.254109589041096e-05, + "loss": 0.32062151432037356, + "mean_token_accuracy": 0.9139250129461288, + "num_tokens": 2195274.0, + "step": 1140 + }, + { + "entropy": 0.31267223581671716, + "epoch": 0.1564625850340136, + "grad_norm": 0.4825468957424164, + "learning_rate": 4.247260273972603e-05, + "loss": 0.3127827882766724, + "mean_token_accuracy": 0.9189456045627594, + "num_tokens": 2213025.0, + "step": 1150 + }, + { + "entropy": 0.34783718213438986, + "epoch": 0.15782312925170067, + "grad_norm": 0.3787980377674103, + "learning_rate": 4.24041095890411e-05, + "loss": 0.33619091510772703, + "mean_token_accuracy": 0.9100428611040116, + "num_tokens": 2233726.0, + "step": 1160 + }, + { + "entropy": 0.25309042185544967, + "epoch": 0.15918367346938775, + "grad_norm": 0.3635607659816742, + "learning_rate": 4.233561643835617e-05, + "loss": 0.25443458557128906, + "mean_token_accuracy": 0.9310213536024093, + "num_tokens": 2252029.0, + "step": 1170 + }, + { + "entropy": 0.3736408993601799, + "epoch": 0.16054421768707483, + "grad_norm": 0.4148007333278656, + "learning_rate": 4.226712328767124e-05, + "loss": 0.38751490116119386, + "mean_token_accuracy": 0.9018953382968903, + "num_tokens": 2269954.0, + "step": 1180 + }, + { + "entropy": 0.31956026926636694, + "epoch": 0.1619047619047619, + "grad_norm": 0.4839099645614624, + "learning_rate": 4.21986301369863e-05, + "loss": 0.32132811546325685, + "mean_token_accuracy": 0.912396851181984, + "num_tokens": 2288506.0, + "step": 1190 + }, + { + "entropy": 0.3568165637552738, + "epoch": 0.16326530612244897, + "grad_norm": 0.24555467069149017, + "learning_rate": 4.213013698630137e-05, + "loss": 0.3542040824890137, + "mean_token_accuracy": 0.9109734386205673, + "num_tokens": 2307894.0, + "step": 1200 + }, + { + "entropy": 0.35158547312021254, + "epoch": 0.16462585034013605, + "grad_norm": 0.412095844745636, + "learning_rate": 4.2061643835616435e-05, + "loss": 0.3496436357498169, + "mean_token_accuracy": 0.9088854372501374, + "num_tokens": 2327240.0, + "step": 1210 + }, + { + "entropy": 0.34600893333554267, + "epoch": 0.16598639455782313, + "grad_norm": 0.3335567116737366, + "learning_rate": 4.199315068493151e-05, + "loss": 0.34185967445373533, + "mean_token_accuracy": 0.905845221877098, + "num_tokens": 2345594.0, + "step": 1220 + }, + { + "entropy": 0.3424117244780064, + "epoch": 0.1673469387755102, + "grad_norm": 0.4858080744743347, + "learning_rate": 4.1924657534246574e-05, + "loss": 0.33852126598358157, + "mean_token_accuracy": 0.913614547252655, + "num_tokens": 2363745.0, + "step": 1230 + }, + { + "entropy": 0.340135945379734, + "epoch": 0.16870748299319727, + "grad_norm": 0.4143927991390228, + "learning_rate": 4.185616438356165e-05, + "loss": 0.3389458417892456, + "mean_token_accuracy": 0.9139795809984207, + "num_tokens": 2382660.0, + "step": 1240 + }, + { + "entropy": 0.35108796805143355, + "epoch": 0.17006802721088435, + "grad_norm": 0.4392998516559601, + "learning_rate": 4.178767123287671e-05, + "loss": 0.356292724609375, + "mean_token_accuracy": 0.9068086326122284, + "num_tokens": 2403842.0, + "step": 1250 + }, + { + "epoch": 0.17006802721088435, + "eval_entropy": 0.34711426744858426, + "eval_loss": 0.349116712808609, + "eval_mean_token_accuracy": 0.9085024491945902, + "eval_num_tokens": 2403842.0, + "eval_runtime": 39.5145, + "eval_samples_per_second": 15.184, + "eval_steps_per_second": 7.592, + "step": 1250 + }, + { + "entropy": 0.3387380100786686, + "epoch": 0.17142857142857143, + "grad_norm": 0.4315812587738037, + "learning_rate": 4.1719178082191786e-05, + "loss": 0.3605659484863281, + "mean_token_accuracy": 0.9103223711252213, + "num_tokens": 2422375.0, + "step": 1260 + }, + { + "entropy": 0.3285294592380524, + "epoch": 0.1727891156462585, + "grad_norm": 0.28438887000083923, + "learning_rate": 4.165068493150685e-05, + "loss": 0.32664496898651124, + "mean_token_accuracy": 0.9147592782974243, + "num_tokens": 2441057.0, + "step": 1270 + }, + { + "entropy": 0.33626926839351656, + "epoch": 0.17414965986394557, + "grad_norm": 0.3710070848464966, + "learning_rate": 4.1582191780821925e-05, + "loss": 0.3354215621948242, + "mean_token_accuracy": 0.9121009469032287, + "num_tokens": 2459887.0, + "step": 1280 + }, + { + "entropy": 0.32256170734763145, + "epoch": 0.17551020408163265, + "grad_norm": 0.3689749240875244, + "learning_rate": 4.151369863013699e-05, + "loss": 0.334096360206604, + "mean_token_accuracy": 0.9107798606157302, + "num_tokens": 2481764.0, + "step": 1290 + }, + { + "entropy": 0.3187907926738262, + "epoch": 0.17687074829931973, + "grad_norm": 0.45171478390693665, + "learning_rate": 4.144520547945206e-05, + "loss": 0.31167116165161135, + "mean_token_accuracy": 0.916716742515564, + "num_tokens": 2501140.0, + "step": 1300 + }, + { + "entropy": 0.3347592830657959, + "epoch": 0.1782312925170068, + "grad_norm": 0.3578483760356903, + "learning_rate": 4.1376712328767123e-05, + "loss": 0.3300051689147949, + "mean_token_accuracy": 0.9117922872304917, + "num_tokens": 2520373.0, + "step": 1310 + }, + { + "entropy": 0.34503707885742185, + "epoch": 0.17959183673469387, + "grad_norm": 0.48708587884902954, + "learning_rate": 4.130821917808219e-05, + "loss": 0.35149393081665037, + "mean_token_accuracy": 0.9106566429138183, + "num_tokens": 2539252.0, + "step": 1320 + }, + { + "entropy": 0.2878501363098621, + "epoch": 0.18095238095238095, + "grad_norm": 0.295443594455719, + "learning_rate": 4.123972602739726e-05, + "loss": 0.28683664798736574, + "mean_token_accuracy": 0.9232668906450272, + "num_tokens": 2559632.0, + "step": 1330 + }, + { + "entropy": 0.35331556648015977, + "epoch": 0.18231292517006803, + "grad_norm": 0.39578038454055786, + "learning_rate": 4.117123287671233e-05, + "loss": 0.3638563871383667, + "mean_token_accuracy": 0.9076030403375626, + "num_tokens": 2578101.0, + "step": 1340 + }, + { + "entropy": 0.2722251106053591, + "epoch": 0.1836734693877551, + "grad_norm": 0.36816802620887756, + "learning_rate": 4.11027397260274e-05, + "loss": 0.2725792169570923, + "mean_token_accuracy": 0.9225607991218567, + "num_tokens": 2596872.0, + "step": 1350 + }, + { + "entropy": 0.31183348074555395, + "epoch": 0.18503401360544217, + "grad_norm": 0.31272897124290466, + "learning_rate": 4.103424657534247e-05, + "loss": 0.3018923044204712, + "mean_token_accuracy": 0.918904721736908, + "num_tokens": 2616948.0, + "step": 1360 + }, + { + "entropy": 0.3306739188730717, + "epoch": 0.18639455782312925, + "grad_norm": 0.49856531620025635, + "learning_rate": 4.096575342465754e-05, + "loss": 0.326325798034668, + "mean_token_accuracy": 0.9133409053087235, + "num_tokens": 2635194.0, + "step": 1370 + }, + { + "entropy": 0.3449342973530293, + "epoch": 0.18775510204081633, + "grad_norm": 0.3672386705875397, + "learning_rate": 4.0897260273972606e-05, + "loss": 0.34992690086364747, + "mean_token_accuracy": 0.910362508893013, + "num_tokens": 2653415.0, + "step": 1380 + }, + { + "entropy": 0.34170059114694595, + "epoch": 0.1891156462585034, + "grad_norm": 0.438903272151947, + "learning_rate": 4.082876712328767e-05, + "loss": 0.362008261680603, + "mean_token_accuracy": 0.9066114306449891, + "num_tokens": 2672556.0, + "step": 1390 + }, + { + "entropy": 0.36418116837739944, + "epoch": 0.19047619047619047, + "grad_norm": 0.3735516369342804, + "learning_rate": 4.076027397260274e-05, + "loss": 0.36933226585388185, + "mean_token_accuracy": 0.903956550359726, + "num_tokens": 2691019.0, + "step": 1400 + }, + { + "entropy": 0.33693169951438906, + "epoch": 0.19183673469387755, + "grad_norm": 0.3407398760318756, + "learning_rate": 4.0691780821917805e-05, + "loss": 0.3302572965621948, + "mean_token_accuracy": 0.9152946561574936, + "num_tokens": 2710242.0, + "step": 1410 + }, + { + "entropy": 0.3172802679240704, + "epoch": 0.19319727891156463, + "grad_norm": 0.3666602373123169, + "learning_rate": 4.062328767123288e-05, + "loss": 0.3129624366760254, + "mean_token_accuracy": 0.9118194162845612, + "num_tokens": 2731126.0, + "step": 1420 + }, + { + "entropy": 0.39142856150865557, + "epoch": 0.1945578231292517, + "grad_norm": 0.3450668752193451, + "learning_rate": 4.0554794520547944e-05, + "loss": 0.3807058572769165, + "mean_token_accuracy": 0.899824932217598, + "num_tokens": 2752136.0, + "step": 1430 + }, + { + "entropy": 0.354360631108284, + "epoch": 0.19591836734693877, + "grad_norm": 0.41342902183532715, + "learning_rate": 4.0486301369863016e-05, + "loss": 0.3667241096496582, + "mean_token_accuracy": 0.9088259905576705, + "num_tokens": 2769136.0, + "step": 1440 + }, + { + "entropy": 0.38877039551734927, + "epoch": 0.19727891156462585, + "grad_norm": 0.49252015352249146, + "learning_rate": 4.041780821917808e-05, + "loss": 0.3996612548828125, + "mean_token_accuracy": 0.898360651731491, + "num_tokens": 2788101.0, + "step": 1450 + }, + { + "entropy": 0.3267239861190319, + "epoch": 0.19863945578231293, + "grad_norm": 0.3556021451950073, + "learning_rate": 4.0349315068493155e-05, + "loss": 0.31743640899658204, + "mean_token_accuracy": 0.9156175047159195, + "num_tokens": 2809422.0, + "step": 1460 + }, + { + "entropy": 0.3570724129676819, + "epoch": 0.2, + "grad_norm": 0.41443583369255066, + "learning_rate": 4.028082191780822e-05, + "loss": 0.34733829498291013, + "mean_token_accuracy": 0.9090855300426484, + "num_tokens": 2828897.0, + "step": 1470 + }, + { + "entropy": 0.36629826575517654, + "epoch": 0.20136054421768707, + "grad_norm": 0.43215256929397583, + "learning_rate": 4.0212328767123294e-05, + "loss": 0.37642669677734375, + "mean_token_accuracy": 0.9042314618825913, + "num_tokens": 2846647.0, + "step": 1480 + }, + { + "entropy": 0.3530550479888916, + "epoch": 0.20272108843537415, + "grad_norm": 0.3693477213382721, + "learning_rate": 4.014383561643836e-05, + "loss": 0.35866584777832033, + "mean_token_accuracy": 0.9062743157148361, + "num_tokens": 2866552.0, + "step": 1490 + }, + { + "entropy": 0.3125508435070515, + "epoch": 0.20408163265306123, + "grad_norm": 0.48748907446861267, + "learning_rate": 4.0075342465753427e-05, + "loss": 0.3137276887893677, + "mean_token_accuracy": 0.9152947783470153, + "num_tokens": 2887091.0, + "step": 1500 + }, + { + "epoch": 0.20408163265306123, + "eval_entropy": 0.3448227975765864, + "eval_loss": 0.34629908204078674, + "eval_mean_token_accuracy": 0.9089537409941355, + "eval_num_tokens": 2887091.0, + "eval_runtime": 39.562, + "eval_samples_per_second": 15.166, + "eval_steps_per_second": 7.583, + "step": 1500 + }, + { + "entropy": 0.3338837943971157, + "epoch": 0.2054421768707483, + "grad_norm": 0.453998327255249, + "learning_rate": 4.000684931506849e-05, + "loss": 0.33314809799194334, + "mean_token_accuracy": 0.9141378700733185, + "num_tokens": 2906558.0, + "step": 1510 + }, + { + "entropy": 0.35243152529001237, + "epoch": 0.20680272108843537, + "grad_norm": 0.514372706413269, + "learning_rate": 3.993835616438356e-05, + "loss": 0.36109921932220457, + "mean_token_accuracy": 0.909517452120781, + "num_tokens": 2924701.0, + "step": 1520 + }, + { + "entropy": 0.35977045372128486, + "epoch": 0.20816326530612245, + "grad_norm": 0.47331997752189636, + "learning_rate": 3.986986301369863e-05, + "loss": 0.3542989730834961, + "mean_token_accuracy": 0.9023140043020248, + "num_tokens": 2944323.0, + "step": 1530 + }, + { + "entropy": 0.3476133652031422, + "epoch": 0.20952380952380953, + "grad_norm": 0.39329883456230164, + "learning_rate": 3.98013698630137e-05, + "loss": 0.35582053661346436, + "mean_token_accuracy": 0.9074772745370865, + "num_tokens": 2962847.0, + "step": 1540 + }, + { + "entropy": 0.36178177297115327, + "epoch": 0.2108843537414966, + "grad_norm": 0.42516037821769714, + "learning_rate": 3.973287671232877e-05, + "loss": 0.3781341552734375, + "mean_token_accuracy": 0.9034180849790573, + "num_tokens": 2982036.0, + "step": 1550 + }, + { + "entropy": 0.3484080970287323, + "epoch": 0.21224489795918366, + "grad_norm": 0.4276966452598572, + "learning_rate": 3.9664383561643837e-05, + "loss": 0.34070544242858886, + "mean_token_accuracy": 0.9100886523723603, + "num_tokens": 3001246.0, + "step": 1560 + }, + { + "entropy": 0.296015814691782, + "epoch": 0.21360544217687075, + "grad_norm": 0.3481068015098572, + "learning_rate": 3.959589041095891e-05, + "loss": 0.3041130542755127, + "mean_token_accuracy": 0.9180979192256927, + "num_tokens": 3021457.0, + "step": 1570 + }, + { + "entropy": 0.33566633649170396, + "epoch": 0.21496598639455783, + "grad_norm": 0.46466073393821716, + "learning_rate": 3.9527397260273976e-05, + "loss": 0.3371039152145386, + "mean_token_accuracy": 0.9121709406375885, + "num_tokens": 3040277.0, + "step": 1580 + }, + { + "entropy": 0.37958977892994883, + "epoch": 0.2163265306122449, + "grad_norm": 0.38823264837265015, + "learning_rate": 3.945890410958905e-05, + "loss": 0.36861262321472166, + "mean_token_accuracy": 0.9052564293146134, + "num_tokens": 3060403.0, + "step": 1590 + }, + { + "entropy": 0.35140516981482506, + "epoch": 0.21768707482993196, + "grad_norm": 0.3407391607761383, + "learning_rate": 3.9390410958904114e-05, + "loss": 0.3561867237091064, + "mean_token_accuracy": 0.9101497054100036, + "num_tokens": 3079560.0, + "step": 1600 + }, + { + "entropy": 0.32827898040413855, + "epoch": 0.21904761904761905, + "grad_norm": 0.3683736324310303, + "learning_rate": 3.932191780821918e-05, + "loss": 0.3281137704849243, + "mean_token_accuracy": 0.9150744706392289, + "num_tokens": 3096762.0, + "step": 1610 + }, + { + "entropy": 0.38666338548064233, + "epoch": 0.22040816326530613, + "grad_norm": 0.4814898669719696, + "learning_rate": 3.925342465753425e-05, + "loss": 0.3958434104919434, + "mean_token_accuracy": 0.9005985498428345, + "num_tokens": 3116551.0, + "step": 1620 + }, + { + "entropy": 0.3289907656610012, + "epoch": 0.2217687074829932, + "grad_norm": 0.34105587005615234, + "learning_rate": 3.918493150684931e-05, + "loss": 0.33880622386932374, + "mean_token_accuracy": 0.9120993465185165, + "num_tokens": 3136954.0, + "step": 1630 + }, + { + "entropy": 0.4120957344770432, + "epoch": 0.22312925170068026, + "grad_norm": 0.2596409022808075, + "learning_rate": 3.9116438356164386e-05, + "loss": 0.411406946182251, + "mean_token_accuracy": 0.8971390217542649, + "num_tokens": 3157285.0, + "step": 1640 + }, + { + "entropy": 0.32644312381744384, + "epoch": 0.22448979591836735, + "grad_norm": 0.3950192332267761, + "learning_rate": 3.904794520547945e-05, + "loss": 0.31403555870056155, + "mean_token_accuracy": 0.9128539711236954, + "num_tokens": 3176288.0, + "step": 1650 + }, + { + "entropy": 0.35888044387102125, + "epoch": 0.22585034013605443, + "grad_norm": 0.35740935802459717, + "learning_rate": 3.8979452054794525e-05, + "loss": 0.3686177968978882, + "mean_token_accuracy": 0.9048507601022721, + "num_tokens": 3196362.0, + "step": 1660 + }, + { + "entropy": 0.3604126326739788, + "epoch": 0.2272108843537415, + "grad_norm": 0.4237874746322632, + "learning_rate": 3.891095890410959e-05, + "loss": 0.3526318073272705, + "mean_token_accuracy": 0.9068311870098114, + "num_tokens": 3214053.0, + "step": 1670 + }, + { + "entropy": 0.34602590948343276, + "epoch": 0.22857142857142856, + "grad_norm": 0.36912399530410767, + "learning_rate": 3.8842465753424663e-05, + "loss": 0.3596346378326416, + "mean_token_accuracy": 0.905837458372116, + "num_tokens": 3233180.0, + "step": 1680 + }, + { + "entropy": 0.3587561517953873, + "epoch": 0.22993197278911565, + "grad_norm": 0.5846858024597168, + "learning_rate": 3.877397260273973e-05, + "loss": 0.3572895288467407, + "mean_token_accuracy": 0.9061489969491958, + "num_tokens": 3253148.0, + "step": 1690 + }, + { + "entropy": 0.31483854800462724, + "epoch": 0.23129251700680273, + "grad_norm": 0.4492280185222626, + "learning_rate": 3.8705479452054796e-05, + "loss": 0.3197118520736694, + "mean_token_accuracy": 0.91756871342659, + "num_tokens": 3272318.0, + "step": 1700 + }, + { + "entropy": 0.399021328240633, + "epoch": 0.23265306122448978, + "grad_norm": 0.44325029850006104, + "learning_rate": 3.863698630136987e-05, + "loss": 0.40597949028015134, + "mean_token_accuracy": 0.895935806632042, + "num_tokens": 3291607.0, + "step": 1710 + }, + { + "entropy": 0.3489011608064175, + "epoch": 0.23401360544217686, + "grad_norm": 0.4789123237133026, + "learning_rate": 3.856849315068493e-05, + "loss": 0.3425743579864502, + "mean_token_accuracy": 0.9078541338443756, + "num_tokens": 3311423.0, + "step": 1720 + }, + { + "entropy": 0.3557139202952385, + "epoch": 0.23537414965986395, + "grad_norm": 0.3092873990535736, + "learning_rate": 3.85e-05, + "loss": 0.36077446937561036, + "mean_token_accuracy": 0.9053440719842911, + "num_tokens": 3333426.0, + "step": 1730 + }, + { + "entropy": 0.3405990473926067, + "epoch": 0.23673469387755103, + "grad_norm": 0.4567224383354187, + "learning_rate": 3.843150684931507e-05, + "loss": 0.3459942817687988, + "mean_token_accuracy": 0.9079800844192505, + "num_tokens": 3352658.0, + "step": 1740 + }, + { + "entropy": 0.3876554772257805, + "epoch": 0.23809523809523808, + "grad_norm": 0.5339434146881104, + "learning_rate": 3.836301369863014e-05, + "loss": 0.39633448123931886, + "mean_token_accuracy": 0.902654567360878, + "num_tokens": 3371460.0, + "step": 1750 + }, + { + "epoch": 0.23809523809523808, + "eval_entropy": 0.3414275799691677, + "eval_loss": 0.34352293610572815, + "eval_mean_token_accuracy": 0.9096791291236878, + "eval_num_tokens": 3371460.0, + "eval_runtime": 39.7497, + "eval_samples_per_second": 15.094, + "eval_steps_per_second": 7.547, + "step": 1750 + }, + { + "entropy": 0.3382601335644722, + "epoch": 0.23945578231292516, + "grad_norm": 0.4466438591480255, + "learning_rate": 3.8294520547945206e-05, + "loss": 0.3433582305908203, + "mean_token_accuracy": 0.911861115694046, + "num_tokens": 3389275.0, + "step": 1760 + }, + { + "entropy": 0.42951491847634315, + "epoch": 0.24081632653061225, + "grad_norm": 0.37805283069610596, + "learning_rate": 3.822602739726028e-05, + "loss": 0.42836475372314453, + "mean_token_accuracy": 0.8932142198085785, + "num_tokens": 3408444.0, + "step": 1770 + }, + { + "entropy": 0.35968755930662155, + "epoch": 0.24217687074829933, + "grad_norm": 0.40297001600265503, + "learning_rate": 3.8157534246575345e-05, + "loss": 0.358404278755188, + "mean_token_accuracy": 0.9076766610145569, + "num_tokens": 3427335.0, + "step": 1780 + }, + { + "entropy": 0.3270475797355175, + "epoch": 0.24353741496598638, + "grad_norm": 0.3639969825744629, + "learning_rate": 3.808904109589042e-05, + "loss": 0.31723456382751464, + "mean_token_accuracy": 0.9145333647727967, + "num_tokens": 3446864.0, + "step": 1790 + }, + { + "entropy": 0.30234274081885815, + "epoch": 0.24489795918367346, + "grad_norm": 0.443294882774353, + "learning_rate": 3.8020547945205484e-05, + "loss": 0.3082306385040283, + "mean_token_accuracy": 0.9216614037752151, + "num_tokens": 3464556.0, + "step": 1800 + }, + { + "entropy": 0.29795855954289435, + "epoch": 0.24625850340136055, + "grad_norm": 0.39209097623825073, + "learning_rate": 3.795205479452055e-05, + "loss": 0.3015660285949707, + "mean_token_accuracy": 0.9175312757492066, + "num_tokens": 3482284.0, + "step": 1810 + }, + { + "entropy": 0.31364143155515195, + "epoch": 0.24761904761904763, + "grad_norm": 0.522806704044342, + "learning_rate": 3.7883561643835616e-05, + "loss": 0.3150556802749634, + "mean_token_accuracy": 0.9156683564186097, + "num_tokens": 3500454.0, + "step": 1820 + }, + { + "entropy": 0.31209259331226347, + "epoch": 0.24897959183673468, + "grad_norm": 0.36000123620033264, + "learning_rate": 3.781506849315068e-05, + "loss": 0.3044718027114868, + "mean_token_accuracy": 0.9180886089801789, + "num_tokens": 3519086.0, + "step": 1830 + }, + { + "entropy": 0.3723416179418564, + "epoch": 0.2503401360544218, + "grad_norm": 0.4354872703552246, + "learning_rate": 3.7746575342465755e-05, + "loss": 0.3991520404815674, + "mean_token_accuracy": 0.9001334726810455, + "num_tokens": 3537252.0, + "step": 1840 + }, + { + "entropy": 0.375165493786335, + "epoch": 0.25170068027210885, + "grad_norm": 0.46084484457969666, + "learning_rate": 3.767808219178082e-05, + "loss": 0.3683051586151123, + "mean_token_accuracy": 0.902921998500824, + "num_tokens": 3555749.0, + "step": 1850 + }, + { + "entropy": 0.3341012619435787, + "epoch": 0.2530612244897959, + "grad_norm": 0.4521940052509308, + "learning_rate": 3.7609589041095894e-05, + "loss": 0.3300362825393677, + "mean_token_accuracy": 0.9145215421915054, + "num_tokens": 3573805.0, + "step": 1860 + }, + { + "entropy": 0.3390877529978752, + "epoch": 0.254421768707483, + "grad_norm": 0.392527312040329, + "learning_rate": 3.754109589041096e-05, + "loss": 0.3547995090484619, + "mean_token_accuracy": 0.9103965133428573, + "num_tokens": 3592844.0, + "step": 1870 + }, + { + "entropy": 0.27256989255547526, + "epoch": 0.25578231292517006, + "grad_norm": 0.4466555118560791, + "learning_rate": 3.747260273972603e-05, + "loss": 0.27302353382110595, + "mean_token_accuracy": 0.9277669697999954, + "num_tokens": 3612185.0, + "step": 1880 + }, + { + "entropy": 0.24652124755084515, + "epoch": 0.2571428571428571, + "grad_norm": 0.4279734194278717, + "learning_rate": 3.74041095890411e-05, + "loss": 0.23633365631103515, + "mean_token_accuracy": 0.9341515839099884, + "num_tokens": 3630518.0, + "step": 1890 + }, + { + "entropy": 0.35038353204727174, + "epoch": 0.2585034013605442, + "grad_norm": 0.5714797973632812, + "learning_rate": 3.7335616438356165e-05, + "loss": 0.344893741607666, + "mean_token_accuracy": 0.907752126455307, + "num_tokens": 3650072.0, + "step": 1900 + }, + { + "entropy": 0.3278027452528477, + "epoch": 0.2598639455782313, + "grad_norm": 0.37882331013679504, + "learning_rate": 3.726712328767124e-05, + "loss": 0.330623722076416, + "mean_token_accuracy": 0.9118026465177536, + "num_tokens": 3670750.0, + "step": 1910 + }, + { + "entropy": 0.3359388172626495, + "epoch": 0.2612244897959184, + "grad_norm": 0.5028720498085022, + "learning_rate": 3.7198630136986304e-05, + "loss": 0.3352250576019287, + "mean_token_accuracy": 0.9122653841972351, + "num_tokens": 3690610.0, + "step": 1920 + }, + { + "entropy": 0.3143607035279274, + "epoch": 0.26258503401360545, + "grad_norm": 0.46486079692840576, + "learning_rate": 3.713013698630137e-05, + "loss": 0.30714452266693115, + "mean_token_accuracy": 0.9196935653686523, + "num_tokens": 3708272.0, + "step": 1930 + }, + { + "entropy": 0.35282155945897103, + "epoch": 0.2639455782312925, + "grad_norm": 0.45841529965400696, + "learning_rate": 3.7061643835616436e-05, + "loss": 0.3545929670333862, + "mean_token_accuracy": 0.9059949398040772, + "num_tokens": 3726869.0, + "step": 1940 + }, + { + "entropy": 0.2984108135104179, + "epoch": 0.2653061224489796, + "grad_norm": 0.42509031295776367, + "learning_rate": 3.699315068493151e-05, + "loss": 0.2950887203216553, + "mean_token_accuracy": 0.9201341837644577, + "num_tokens": 3746172.0, + "step": 1950 + }, + { + "entropy": 0.31338903233408927, + "epoch": 0.26666666666666666, + "grad_norm": 0.37572163343429565, + "learning_rate": 3.6924657534246575e-05, + "loss": 0.3111075162887573, + "mean_token_accuracy": 0.916234827041626, + "num_tokens": 3765690.0, + "step": 1960 + }, + { + "entropy": 0.31969371140003205, + "epoch": 0.2680272108843537, + "grad_norm": 0.4484844207763672, + "learning_rate": 3.685616438356165e-05, + "loss": 0.32388155460357665, + "mean_token_accuracy": 0.9160292863845825, + "num_tokens": 3783682.0, + "step": 1970 + }, + { + "entropy": 0.34558562338352206, + "epoch": 0.2693877551020408, + "grad_norm": 0.6314212679862976, + "learning_rate": 3.6787671232876714e-05, + "loss": 0.3459887742996216, + "mean_token_accuracy": 0.9086391866207123, + "num_tokens": 3801830.0, + "step": 1980 + }, + { + "entropy": 0.315334652364254, + "epoch": 0.2707482993197279, + "grad_norm": 0.9491491317749023, + "learning_rate": 3.671917808219179e-05, + "loss": 0.3217782974243164, + "mean_token_accuracy": 0.914330929517746, + "num_tokens": 3821971.0, + "step": 1990 + }, + { + "entropy": 0.3376677267253399, + "epoch": 0.272108843537415, + "grad_norm": 0.4743102192878723, + "learning_rate": 3.665068493150685e-05, + "loss": 0.3347081184387207, + "mean_token_accuracy": 0.9096920937299728, + "num_tokens": 3841489.0, + "step": 2000 + }, + { + "epoch": 0.272108843537415, + "eval_entropy": 0.3519680585463842, + "eval_loss": 0.342134028673172, + "eval_mean_token_accuracy": 0.9097507733106613, + "eval_num_tokens": 3841489.0, + "eval_runtime": 39.6137, + "eval_samples_per_second": 15.146, + "eval_steps_per_second": 7.573, + "step": 2000 + }, + { + "entropy": 0.35166340619325637, + "epoch": 0.27346938775510204, + "grad_norm": 0.2940121591091156, + "learning_rate": 3.658219178082192e-05, + "loss": 0.3434895515441895, + "mean_token_accuracy": 0.9088268965482712, + "num_tokens": 3861515.0, + "step": 2010 + }, + { + "entropy": 0.3252212144434452, + "epoch": 0.2748299319727891, + "grad_norm": 0.33630767464637756, + "learning_rate": 3.651369863013699e-05, + "loss": 0.3320222616195679, + "mean_token_accuracy": 0.9136297762393951, + "num_tokens": 3880492.0, + "step": 2020 + }, + { + "entropy": 0.30923423618078233, + "epoch": 0.2761904761904762, + "grad_norm": 0.3880252242088318, + "learning_rate": 3.644520547945206e-05, + "loss": 0.30057878494262696, + "mean_token_accuracy": 0.921365550160408, + "num_tokens": 3900283.0, + "step": 2030 + }, + { + "entropy": 0.3484115995466709, + "epoch": 0.27755102040816326, + "grad_norm": 0.4368172883987427, + "learning_rate": 3.6376712328767124e-05, + "loss": 0.33826982975006104, + "mean_token_accuracy": 0.9120531916618347, + "num_tokens": 3918622.0, + "step": 2040 + }, + { + "entropy": 0.301137863099575, + "epoch": 0.2789115646258503, + "grad_norm": 0.44099488854408264, + "learning_rate": 3.630821917808219e-05, + "loss": 0.3043475866317749, + "mean_token_accuracy": 0.9190519124269485, + "num_tokens": 3936916.0, + "step": 2050 + }, + { + "entropy": 0.32400831282138826, + "epoch": 0.2802721088435374, + "grad_norm": 0.3683919310569763, + "learning_rate": 3.623972602739726e-05, + "loss": 0.32993350028991697, + "mean_token_accuracy": 0.9162216484546661, + "num_tokens": 3954118.0, + "step": 2060 + }, + { + "entropy": 0.36117612645030023, + "epoch": 0.2816326530612245, + "grad_norm": 0.36590471863746643, + "learning_rate": 3.617123287671233e-05, + "loss": 0.366199254989624, + "mean_token_accuracy": 0.9073205977678299, + "num_tokens": 3972244.0, + "step": 2070 + }, + { + "entropy": 0.30845518559217455, + "epoch": 0.2829931972789116, + "grad_norm": 0.4904245436191559, + "learning_rate": 3.61027397260274e-05, + "loss": 0.3000732660293579, + "mean_token_accuracy": 0.9213456630706787, + "num_tokens": 3991933.0, + "step": 2080 + }, + { + "entropy": 0.33342998176813127, + "epoch": 0.28435374149659864, + "grad_norm": 0.3451215326786041, + "learning_rate": 3.603424657534247e-05, + "loss": 0.3519883155822754, + "mean_token_accuracy": 0.9118590801954269, + "num_tokens": 4009549.0, + "step": 2090 + }, + { + "entropy": 0.3225520886480808, + "epoch": 0.2857142857142857, + "grad_norm": 0.41656896471977234, + "learning_rate": 3.5965753424657534e-05, + "loss": 0.33789401054382323, + "mean_token_accuracy": 0.9126103073358536, + "num_tokens": 4028579.0, + "step": 2100 + }, + { + "entropy": 0.33171583339571953, + "epoch": 0.2870748299319728, + "grad_norm": 0.44855958223342896, + "learning_rate": 3.589726027397261e-05, + "loss": 0.33625202178955077, + "mean_token_accuracy": 0.9129469394683838, + "num_tokens": 4047155.0, + "step": 2110 + }, + { + "entropy": 0.3181050315499306, + "epoch": 0.28843537414965986, + "grad_norm": 0.45970842242240906, + "learning_rate": 3.582876712328767e-05, + "loss": 0.3234997749328613, + "mean_token_accuracy": 0.9154122680425644, + "num_tokens": 4065666.0, + "step": 2120 + }, + { + "entropy": 0.3713026508688927, + "epoch": 0.2897959183673469, + "grad_norm": 0.4234248399734497, + "learning_rate": 3.5760273972602746e-05, + "loss": 0.36728248596191404, + "mean_token_accuracy": 0.9066497653722763, + "num_tokens": 4084684.0, + "step": 2130 + }, + { + "entropy": 0.36940672621130943, + "epoch": 0.291156462585034, + "grad_norm": 0.490075021982193, + "learning_rate": 3.569178082191781e-05, + "loss": 0.3767258644104004, + "mean_token_accuracy": 0.9038274347782135, + "num_tokens": 4104584.0, + "step": 2140 + }, + { + "entropy": 0.36020471081137656, + "epoch": 0.2925170068027211, + "grad_norm": 0.45468005537986755, + "learning_rate": 3.562328767123288e-05, + "loss": 0.36363468170166013, + "mean_token_accuracy": 0.907564926147461, + "num_tokens": 4122918.0, + "step": 2150 + }, + { + "entropy": 0.320191890001297, + "epoch": 0.2938775510204082, + "grad_norm": 0.4919210374355316, + "learning_rate": 3.5554794520547944e-05, + "loss": 0.3147299766540527, + "mean_token_accuracy": 0.919364845752716, + "num_tokens": 4140528.0, + "step": 2160 + }, + { + "entropy": 0.39545504078269006, + "epoch": 0.29523809523809524, + "grad_norm": 0.4305582344532013, + "learning_rate": 3.548630136986302e-05, + "loss": 0.4035975456237793, + "mean_token_accuracy": 0.8999768882989884, + "num_tokens": 4159136.0, + "step": 2170 + }, + { + "entropy": 0.32939055263996125, + "epoch": 0.2965986394557823, + "grad_norm": 0.5001799464225769, + "learning_rate": 3.541780821917808e-05, + "loss": 0.32146081924438474, + "mean_token_accuracy": 0.9139656722545624, + "num_tokens": 4178728.0, + "step": 2180 + }, + { + "entropy": 0.3295467995107174, + "epoch": 0.2979591836734694, + "grad_norm": 0.5351600050926208, + "learning_rate": 3.5349315068493156e-05, + "loss": 0.32642207145690916, + "mean_token_accuracy": 0.9078761160373687, + "num_tokens": 4199352.0, + "step": 2190 + }, + { + "entropy": 0.3639287866652012, + "epoch": 0.29931972789115646, + "grad_norm": 0.5141214728355408, + "learning_rate": 3.528082191780822e-05, + "loss": 0.3758009195327759, + "mean_token_accuracy": 0.9029638439416885, + "num_tokens": 4220071.0, + "step": 2200 + }, + { + "entropy": 0.31801966652274133, + "epoch": 0.3006802721088435, + "grad_norm": 0.4239181876182556, + "learning_rate": 3.521232876712329e-05, + "loss": 0.3060357093811035, + "mean_token_accuracy": 0.9188521355390549, + "num_tokens": 4238856.0, + "step": 2210 + }, + { + "entropy": 0.3254566751420498, + "epoch": 0.3020408163265306, + "grad_norm": 0.5890150666236877, + "learning_rate": 3.514383561643836e-05, + "loss": 0.33809313774108884, + "mean_token_accuracy": 0.9106950730085372, + "num_tokens": 4258568.0, + "step": 2220 + }, + { + "entropy": 0.3605205237865448, + "epoch": 0.3034013605442177, + "grad_norm": 0.4173797070980072, + "learning_rate": 3.507534246575343e-05, + "loss": 0.3492074251174927, + "mean_token_accuracy": 0.9083913922309875, + "num_tokens": 4277649.0, + "step": 2230 + }, + { + "entropy": 0.3326388791203499, + "epoch": 0.3047619047619048, + "grad_norm": 0.510657012462616, + "learning_rate": 3.500684931506849e-05, + "loss": 0.3288934946060181, + "mean_token_accuracy": 0.9091042637825012, + "num_tokens": 4297822.0, + "step": 2240 + }, + { + "entropy": 0.32781085297465323, + "epoch": 0.30612244897959184, + "grad_norm": 0.41473498940467834, + "learning_rate": 3.493835616438356e-05, + "loss": 0.320549750328064, + "mean_token_accuracy": 0.9140338152647018, + "num_tokens": 4316581.0, + "step": 2250 + }, + { + "epoch": 0.30612244897959184, + "eval_entropy": 0.3347464001675447, + "eval_loss": 0.3400094211101532, + "eval_mean_token_accuracy": 0.9105568333466848, + "eval_num_tokens": 4316581.0, + "eval_runtime": 39.5997, + "eval_samples_per_second": 15.152, + "eval_steps_per_second": 7.576, + "step": 2250 + }, + { + "entropy": 0.29784226417541504, + "epoch": 0.3074829931972789, + "grad_norm": 0.3205791711807251, + "learning_rate": 3.486986301369863e-05, + "loss": 0.2996628761291504, + "mean_token_accuracy": 0.918051677942276, + "num_tokens": 4335064.0, + "step": 2260 + }, + { + "entropy": 0.31014653742313386, + "epoch": 0.308843537414966, + "grad_norm": 0.4868067502975464, + "learning_rate": 3.48013698630137e-05, + "loss": 0.31834020614624026, + "mean_token_accuracy": 0.9162485599517822, + "num_tokens": 4355734.0, + "step": 2270 + }, + { + "entropy": 0.32516508251428605, + "epoch": 0.31020408163265306, + "grad_norm": 0.4637530446052551, + "learning_rate": 3.473287671232877e-05, + "loss": 0.33171901702880857, + "mean_token_accuracy": 0.9147695600986481, + "num_tokens": 4373625.0, + "step": 2280 + }, + { + "entropy": 0.3301460199058056, + "epoch": 0.3115646258503401, + "grad_norm": 0.39910921454429626, + "learning_rate": 3.466438356164384e-05, + "loss": 0.33153936862945554, + "mean_token_accuracy": 0.9145438969135284, + "num_tokens": 4391557.0, + "step": 2290 + }, + { + "entropy": 0.35131814479827883, + "epoch": 0.3129251700680272, + "grad_norm": 0.4119342267513275, + "learning_rate": 3.45958904109589e-05, + "loss": 0.36605679988861084, + "mean_token_accuracy": 0.9072954088449479, + "num_tokens": 4409285.0, + "step": 2300 + }, + { + "entropy": 0.38690097257494926, + "epoch": 0.3142857142857143, + "grad_norm": 0.4198577404022217, + "learning_rate": 3.4527397260273976e-05, + "loss": 0.39443681240081785, + "mean_token_accuracy": 0.9032483011484146, + "num_tokens": 4427549.0, + "step": 2310 + }, + { + "entropy": 0.3198526822030544, + "epoch": 0.31564625850340133, + "grad_norm": 0.5783278942108154, + "learning_rate": 3.445890410958904e-05, + "loss": 0.31236040592193604, + "mean_token_accuracy": 0.916850745677948, + "num_tokens": 4445150.0, + "step": 2320 + }, + { + "entropy": 0.37613857463002204, + "epoch": 0.31700680272108844, + "grad_norm": 0.4824889302253723, + "learning_rate": 3.4390410958904115e-05, + "loss": 0.38405096530914307, + "mean_token_accuracy": 0.9016139656305313, + "num_tokens": 4464890.0, + "step": 2330 + }, + { + "entropy": 0.3105509489774704, + "epoch": 0.3183673469387755, + "grad_norm": 0.40339168906211853, + "learning_rate": 3.432191780821918e-05, + "loss": 0.30131852626800537, + "mean_token_accuracy": 0.916943484544754, + "num_tokens": 4484914.0, + "step": 2340 + }, + { + "entropy": 0.34861349165439603, + "epoch": 0.3197278911564626, + "grad_norm": 0.35453712940216064, + "learning_rate": 3.425342465753425e-05, + "loss": 0.3576103687286377, + "mean_token_accuracy": 0.9065870314836502, + "num_tokens": 4503897.0, + "step": 2350 + }, + { + "entropy": 0.3504960760474205, + "epoch": 0.32108843537414966, + "grad_norm": 0.36962467432022095, + "learning_rate": 3.418493150684931e-05, + "loss": 0.3403719186782837, + "mean_token_accuracy": 0.9101202934980392, + "num_tokens": 4523958.0, + "step": 2360 + }, + { + "entropy": 0.3424271956086159, + "epoch": 0.3224489795918367, + "grad_norm": 0.4860788881778717, + "learning_rate": 3.4116438356164386e-05, + "loss": 0.3569652080535889, + "mean_token_accuracy": 0.9091333240270615, + "num_tokens": 4540850.0, + "step": 2370 + }, + { + "entropy": 0.3189090140163898, + "epoch": 0.3238095238095238, + "grad_norm": 0.4140557050704956, + "learning_rate": 3.404794520547945e-05, + "loss": 0.315256404876709, + "mean_token_accuracy": 0.9175257176160813, + "num_tokens": 4558898.0, + "step": 2380 + }, + { + "entropy": 0.33776298984885217, + "epoch": 0.3251700680272109, + "grad_norm": 0.36254385113716125, + "learning_rate": 3.397945205479452e-05, + "loss": 0.33310599327087403, + "mean_token_accuracy": 0.9075568825006485, + "num_tokens": 4578067.0, + "step": 2390 + }, + { + "entropy": 0.35714403837919234, + "epoch": 0.32653061224489793, + "grad_norm": 0.36844414472579956, + "learning_rate": 3.391095890410959e-05, + "loss": 0.36895968914031985, + "mean_token_accuracy": 0.9088596940040589, + "num_tokens": 4596553.0, + "step": 2400 + }, + { + "entropy": 0.34672648683190344, + "epoch": 0.32789115646258504, + "grad_norm": 0.5768862962722778, + "learning_rate": 3.384246575342466e-05, + "loss": 0.3418919801712036, + "mean_token_accuracy": 0.9105334728956223, + "num_tokens": 4617409.0, + "step": 2410 + }, + { + "entropy": 0.35704168528318403, + "epoch": 0.3292517006802721, + "grad_norm": 0.3797224760055542, + "learning_rate": 3.377397260273973e-05, + "loss": 0.35338263511657714, + "mean_token_accuracy": 0.907687571644783, + "num_tokens": 4636036.0, + "step": 2420 + }, + { + "entropy": 0.37071596682071684, + "epoch": 0.3306122448979592, + "grad_norm": 0.41465795040130615, + "learning_rate": 3.3705479452054796e-05, + "loss": 0.375014066696167, + "mean_token_accuracy": 0.9049693375825882, + "num_tokens": 4654270.0, + "step": 2430 + }, + { + "entropy": 0.3085879869759083, + "epoch": 0.33197278911564626, + "grad_norm": 0.5098187327384949, + "learning_rate": 3.363698630136987e-05, + "loss": 0.30675497055053713, + "mean_token_accuracy": 0.9197484374046325, + "num_tokens": 4672398.0, + "step": 2440 + }, + { + "entropy": 0.3546553373336792, + "epoch": 0.3333333333333333, + "grad_norm": 0.4272635877132416, + "learning_rate": 3.3568493150684935e-05, + "loss": 0.36507577896118165, + "mean_token_accuracy": 0.9074768930673599, + "num_tokens": 4690119.0, + "step": 2450 + }, + { + "entropy": 0.3424846582114697, + "epoch": 0.3346938775510204, + "grad_norm": 0.47268643975257874, + "learning_rate": 3.35e-05, + "loss": 0.33769257068634034, + "mean_token_accuracy": 0.912876409292221, + "num_tokens": 4709956.0, + "step": 2460 + }, + { + "entropy": 0.36993408799171446, + "epoch": 0.3360544217687075, + "grad_norm": 0.4779612421989441, + "learning_rate": 3.343150684931507e-05, + "loss": 0.37760152816772463, + "mean_token_accuracy": 0.9003741055727005, + "num_tokens": 4730284.0, + "step": 2470 + }, + { + "entropy": 0.34832309633493425, + "epoch": 0.33741496598639453, + "grad_norm": 0.6033436059951782, + "learning_rate": 3.336301369863014e-05, + "loss": 0.346789026260376, + "mean_token_accuracy": 0.9062613487243653, + "num_tokens": 4750695.0, + "step": 2480 + }, + { + "entropy": 0.3566281244158745, + "epoch": 0.33877551020408164, + "grad_norm": 0.5511232018470764, + "learning_rate": 3.3294520547945206e-05, + "loss": 0.33579089641571047, + "mean_token_accuracy": 0.9080673784017563, + "num_tokens": 4770493.0, + "step": 2490 + }, + { + "entropy": 0.3206308774650097, + "epoch": 0.3401360544217687, + "grad_norm": 0.3410138189792633, + "learning_rate": 3.322602739726027e-05, + "loss": 0.3231546401977539, + "mean_token_accuracy": 0.9120282381772995, + "num_tokens": 4791470.0, + "step": 2500 + }, + { + "epoch": 0.3401360544217687, + "eval_entropy": 0.3339786528050899, + "eval_loss": 0.3382870852947235, + "eval_mean_token_accuracy": 0.910913230975469, + "eval_num_tokens": 4791470.0, + "eval_runtime": 39.7078, + "eval_samples_per_second": 15.11, + "eval_steps_per_second": 7.555, + "step": 2500 + }, + { + "entropy": 0.3026216134428978, + "epoch": 0.3414965986394558, + "grad_norm": 0.49190741777420044, + "learning_rate": 3.3157534246575345e-05, + "loss": 0.31151907444000243, + "mean_token_accuracy": 0.9197212547063828, + "num_tokens": 4808923.0, + "step": 2510 + }, + { + "entropy": 0.35362531840801237, + "epoch": 0.34285714285714286, + "grad_norm": 0.4947229325771332, + "learning_rate": 3.308904109589041e-05, + "loss": 0.35634171962738037, + "mean_token_accuracy": 0.905270916223526, + "num_tokens": 4828698.0, + "step": 2520 + }, + { + "entropy": 0.2948874182999134, + "epoch": 0.3442176870748299, + "grad_norm": 0.588706910610199, + "learning_rate": 3.3020547945205484e-05, + "loss": 0.3044130802154541, + "mean_token_accuracy": 0.9255930632352829, + "num_tokens": 4846152.0, + "step": 2530 + }, + { + "entropy": 0.30128093957901003, + "epoch": 0.345578231292517, + "grad_norm": 0.38747715950012207, + "learning_rate": 3.295205479452055e-05, + "loss": 0.2940336227416992, + "mean_token_accuracy": 0.9189084142446518, + "num_tokens": 4866211.0, + "step": 2540 + }, + { + "entropy": 0.3247291907668114, + "epoch": 0.3469387755102041, + "grad_norm": 0.4447266161441803, + "learning_rate": 3.288356164383562e-05, + "loss": 0.32945706844329836, + "mean_token_accuracy": 0.912653061747551, + "num_tokens": 4886257.0, + "step": 2550 + }, + { + "entropy": 0.35191837064921855, + "epoch": 0.34829931972789113, + "grad_norm": 0.44626155495643616, + "learning_rate": 3.281506849315069e-05, + "loss": 0.33861584663391114, + "mean_token_accuracy": 0.9103022038936615, + "num_tokens": 4905249.0, + "step": 2560 + }, + { + "entropy": 0.2881168477237225, + "epoch": 0.34965986394557824, + "grad_norm": 0.44346174597740173, + "learning_rate": 3.2746575342465755e-05, + "loss": 0.28231775760650635, + "mean_token_accuracy": 0.9268552869558334, + "num_tokens": 4923178.0, + "step": 2570 + }, + { + "entropy": 0.3358376182615757, + "epoch": 0.3510204081632653, + "grad_norm": 0.32187992334365845, + "learning_rate": 3.267808219178082e-05, + "loss": 0.3331818342208862, + "mean_token_accuracy": 0.9085330307483673, + "num_tokens": 4942861.0, + "step": 2580 + }, + { + "entropy": 0.30966882929205897, + "epoch": 0.3523809523809524, + "grad_norm": 0.5043341517448425, + "learning_rate": 3.260958904109589e-05, + "loss": 0.31597659587860105, + "mean_token_accuracy": 0.9186759322881699, + "num_tokens": 4960880.0, + "step": 2590 + }, + { + "entropy": 0.31486412063241004, + "epoch": 0.35374149659863946, + "grad_norm": 0.3755909204483032, + "learning_rate": 3.254109589041096e-05, + "loss": 0.30836541652679444, + "mean_token_accuracy": 0.9174209177494049, + "num_tokens": 4979663.0, + "step": 2600 + }, + { + "entropy": 0.3569600723683834, + "epoch": 0.3551020408163265, + "grad_norm": 0.5074579119682312, + "learning_rate": 3.2472602739726026e-05, + "loss": 0.3442463159561157, + "mean_token_accuracy": 0.9093485355377198, + "num_tokens": 4999805.0, + "step": 2610 + }, + { + "entropy": 0.31667746156454085, + "epoch": 0.3564625850340136, + "grad_norm": 0.41301679611206055, + "learning_rate": 3.24041095890411e-05, + "loss": 0.3286094903945923, + "mean_token_accuracy": 0.9134628236293793, + "num_tokens": 5019199.0, + "step": 2620 + }, + { + "entropy": 0.321327581256628, + "epoch": 0.3578231292517007, + "grad_norm": 0.6529260277748108, + "learning_rate": 3.2335616438356165e-05, + "loss": 0.3306095838546753, + "mean_token_accuracy": 0.9112082034349441, + "num_tokens": 5038915.0, + "step": 2630 + }, + { + "entropy": 0.36806232705712316, + "epoch": 0.35918367346938773, + "grad_norm": 0.4105146527290344, + "learning_rate": 3.226712328767124e-05, + "loss": 0.37603011131286623, + "mean_token_accuracy": 0.903413000702858, + "num_tokens": 5057723.0, + "step": 2640 + }, + { + "entropy": 0.31727024130523207, + "epoch": 0.36054421768707484, + "grad_norm": 0.4973562955856323, + "learning_rate": 3.2198630136986304e-05, + "loss": 0.3162988662719727, + "mean_token_accuracy": 0.9186434358358383, + "num_tokens": 5076899.0, + "step": 2650 + }, + { + "entropy": 0.3479673244059086, + "epoch": 0.3619047619047619, + "grad_norm": 0.471534788608551, + "learning_rate": 3.213013698630137e-05, + "loss": 0.34523987770080566, + "mean_token_accuracy": 0.9076945811510087, + "num_tokens": 5098260.0, + "step": 2660 + }, + { + "entropy": 0.3287705913186073, + "epoch": 0.363265306122449, + "grad_norm": 0.3614688217639923, + "learning_rate": 3.2061643835616436e-05, + "loss": 0.3317812442779541, + "mean_token_accuracy": 0.9130234062671662, + "num_tokens": 5118009.0, + "step": 2670 + }, + { + "entropy": 0.3424487292766571, + "epoch": 0.36462585034013606, + "grad_norm": 0.473599910736084, + "learning_rate": 3.199315068493151e-05, + "loss": 0.3391443729400635, + "mean_token_accuracy": 0.9129292786121368, + "num_tokens": 5137099.0, + "step": 2680 + }, + { + "entropy": 0.370761601626873, + "epoch": 0.3659863945578231, + "grad_norm": 0.6083559989929199, + "learning_rate": 3.1924657534246575e-05, + "loss": 0.3879230499267578, + "mean_token_accuracy": 0.8995687663555145, + "num_tokens": 5157056.0, + "step": 2690 + }, + { + "entropy": 0.34785791113972664, + "epoch": 0.3673469387755102, + "grad_norm": 0.3559763431549072, + "learning_rate": 3.185616438356164e-05, + "loss": 0.34124653339385985, + "mean_token_accuracy": 0.909286892414093, + "num_tokens": 5177253.0, + "step": 2700 + }, + { + "entropy": 0.3293781645596027, + "epoch": 0.3687074829931973, + "grad_norm": 0.5223445296287537, + "learning_rate": 3.1787671232876714e-05, + "loss": 0.32764997482299807, + "mean_token_accuracy": 0.9157842248678207, + "num_tokens": 5194945.0, + "step": 2710 + }, + { + "entropy": 0.31182133629918096, + "epoch": 0.37006802721088433, + "grad_norm": 0.3634929358959198, + "learning_rate": 3.171917808219178e-05, + "loss": 0.31785967350006106, + "mean_token_accuracy": 0.9155223101377488, + "num_tokens": 5215068.0, + "step": 2720 + }, + { + "entropy": 0.39456270560622214, + "epoch": 0.37142857142857144, + "grad_norm": 1.0898386240005493, + "learning_rate": 3.165068493150685e-05, + "loss": 0.4019328594207764, + "mean_token_accuracy": 0.896870681643486, + "num_tokens": 5236257.0, + "step": 2730 + }, + { + "entropy": 0.3375735577195883, + "epoch": 0.3727891156462585, + "grad_norm": 0.5985054969787598, + "learning_rate": 3.158219178082192e-05, + "loss": 0.34434778690338136, + "mean_token_accuracy": 0.914423531293869, + "num_tokens": 5254512.0, + "step": 2740 + }, + { + "entropy": 0.3534530118107796, + "epoch": 0.3741496598639456, + "grad_norm": 0.6718593835830688, + "learning_rate": 3.151369863013699e-05, + "loss": 0.34789319038391114, + "mean_token_accuracy": 0.9068450212478638, + "num_tokens": 5273306.0, + "step": 2750 + }, + { + "epoch": 0.3741496598639456, + "eval_entropy": 0.34889839991927146, + "eval_loss": 0.33721673488616943, + "eval_mean_token_accuracy": 0.911351277033488, + "eval_num_tokens": 5273306.0, + "eval_runtime": 39.6807, + "eval_samples_per_second": 15.121, + "eval_steps_per_second": 7.56, + "step": 2750 + }, + { + "entropy": 0.34381640776991845, + "epoch": 0.37551020408163266, + "grad_norm": 0.3617115914821625, + "learning_rate": 3.144520547945206e-05, + "loss": 0.34432876110076904, + "mean_token_accuracy": 0.9133523643016815, + "num_tokens": 5293581.0, + "step": 2760 + }, + { + "entropy": 0.3725735809653997, + "epoch": 0.3768707482993197, + "grad_norm": 0.39663493633270264, + "learning_rate": 3.1376712328767124e-05, + "loss": 0.3688955545425415, + "mean_token_accuracy": 0.9088011533021927, + "num_tokens": 5312584.0, + "step": 2770 + }, + { + "entropy": 0.287184102833271, + "epoch": 0.3782312925170068, + "grad_norm": 0.5590073466300964, + "learning_rate": 3.130821917808219e-05, + "loss": 0.27388792037963866, + "mean_token_accuracy": 0.919337859749794, + "num_tokens": 5333734.0, + "step": 2780 + }, + { + "entropy": 0.31367961689829826, + "epoch": 0.3795918367346939, + "grad_norm": 0.4748844504356384, + "learning_rate": 3.1239726027397256e-05, + "loss": 0.3206137418746948, + "mean_token_accuracy": 0.9146426022052765, + "num_tokens": 5353231.0, + "step": 2790 + }, + { + "entropy": 0.33578074127435686, + "epoch": 0.38095238095238093, + "grad_norm": 0.5165243148803711, + "learning_rate": 3.117123287671233e-05, + "loss": 0.3436616897583008, + "mean_token_accuracy": 0.9078721046447754, + "num_tokens": 5373437.0, + "step": 2800 + }, + { + "entropy": 0.30183863267302513, + "epoch": 0.38231292517006804, + "grad_norm": 0.5405089259147644, + "learning_rate": 3.1102739726027395e-05, + "loss": 0.29791650772094724, + "mean_token_accuracy": 0.9211073070764542, + "num_tokens": 5394481.0, + "step": 2810 + }, + { + "entropy": 0.3263554021716118, + "epoch": 0.3836734693877551, + "grad_norm": 0.3398967981338501, + "learning_rate": 3.103424657534247e-05, + "loss": 0.33998684883117675, + "mean_token_accuracy": 0.911750203371048, + "num_tokens": 5414726.0, + "step": 2820 + }, + { + "entropy": 0.3496020220220089, + "epoch": 0.38503401360544215, + "grad_norm": 0.49333450198173523, + "learning_rate": 3.0965753424657534e-05, + "loss": 0.3456233263015747, + "mean_token_accuracy": 0.9098768889904022, + "num_tokens": 5433364.0, + "step": 2830 + }, + { + "entropy": 0.35282182693481445, + "epoch": 0.38639455782312926, + "grad_norm": 0.4709138572216034, + "learning_rate": 3.089726027397261e-05, + "loss": 0.3554483413696289, + "mean_token_accuracy": 0.9042170315980911, + "num_tokens": 5454586.0, + "step": 2840 + }, + { + "entropy": 0.2756012246012688, + "epoch": 0.3877551020408163, + "grad_norm": 0.3347083330154419, + "learning_rate": 3.082876712328767e-05, + "loss": 0.2776206016540527, + "mean_token_accuracy": 0.9246499836444855, + "num_tokens": 5472836.0, + "step": 2850 + }, + { + "entropy": 0.3471778355538845, + "epoch": 0.3891156462585034, + "grad_norm": 0.3866668939590454, + "learning_rate": 3.0760273972602746e-05, + "loss": 0.3517859697341919, + "mean_token_accuracy": 0.9072955340147019, + "num_tokens": 5491741.0, + "step": 2860 + }, + { + "entropy": 0.31448092609643935, + "epoch": 0.3904761904761905, + "grad_norm": 0.5152408480644226, + "learning_rate": 3.069178082191781e-05, + "loss": 0.32549118995666504, + "mean_token_accuracy": 0.9156864136457443, + "num_tokens": 5510575.0, + "step": 2870 + }, + { + "entropy": 0.37327151745557785, + "epoch": 0.39183673469387753, + "grad_norm": 0.4741872549057007, + "learning_rate": 3.062328767123288e-05, + "loss": 0.3845684289932251, + "mean_token_accuracy": 0.8996114999055862, + "num_tokens": 5529986.0, + "step": 2880 + }, + { + "entropy": 0.3675620459020138, + "epoch": 0.39319727891156464, + "grad_norm": 0.43704453110694885, + "learning_rate": 3.0554794520547944e-05, + "loss": 0.3695012092590332, + "mean_token_accuracy": 0.9072604596614837, + "num_tokens": 5549632.0, + "step": 2890 + }, + { + "entropy": 0.3603727616369724, + "epoch": 0.3945578231292517, + "grad_norm": 0.4204140901565552, + "learning_rate": 3.0486301369863014e-05, + "loss": 0.361452054977417, + "mean_token_accuracy": 0.9049343466758728, + "num_tokens": 5569455.0, + "step": 2900 + }, + { + "entropy": 0.31857734769582746, + "epoch": 0.39591836734693875, + "grad_norm": 0.543210506439209, + "learning_rate": 3.0417808219178083e-05, + "loss": 0.3216622114181519, + "mean_token_accuracy": 0.9114610522985458, + "num_tokens": 5590154.0, + "step": 2910 + }, + { + "entropy": 0.32558867931365965, + "epoch": 0.39727891156462586, + "grad_norm": 0.39182421565055847, + "learning_rate": 3.034931506849315e-05, + "loss": 0.3142657518386841, + "mean_token_accuracy": 0.9159083515405655, + "num_tokens": 5608934.0, + "step": 2920 + }, + { + "entropy": 0.35831537619233134, + "epoch": 0.3986394557823129, + "grad_norm": 0.43255022168159485, + "learning_rate": 3.0280821917808222e-05, + "loss": 0.3551504373550415, + "mean_token_accuracy": 0.9060514450073243, + "num_tokens": 5629700.0, + "step": 2930 + }, + { + "entropy": 0.32889301925897596, + "epoch": 0.4, + "grad_norm": 0.4353126883506775, + "learning_rate": 3.021232876712329e-05, + "loss": 0.33272716999053953, + "mean_token_accuracy": 0.9120247811079025, + "num_tokens": 5648205.0, + "step": 2940 + }, + { + "entropy": 0.4077422976493835, + "epoch": 0.4013605442176871, + "grad_norm": 0.5193459391593933, + "learning_rate": 3.0143835616438358e-05, + "loss": 0.42622690200805663, + "mean_token_accuracy": 0.8950145274400712, + "num_tokens": 5666658.0, + "step": 2950 + }, + { + "entropy": 0.2837504968047142, + "epoch": 0.40272108843537413, + "grad_norm": 0.4110262095928192, + "learning_rate": 3.0075342465753424e-05, + "loss": 0.2780358552932739, + "mean_token_accuracy": 0.9248676776885987, + "num_tokens": 5685895.0, + "step": 2960 + }, + { + "entropy": 0.2994197480380535, + "epoch": 0.40408163265306124, + "grad_norm": 0.5185297727584839, + "learning_rate": 3.0006849315068497e-05, + "loss": 0.3070257902145386, + "mean_token_accuracy": 0.9217073917388916, + "num_tokens": 5705410.0, + "step": 2970 + }, + { + "entropy": 0.3405400663614273, + "epoch": 0.4054421768707483, + "grad_norm": 0.446480929851532, + "learning_rate": 2.9938356164383563e-05, + "loss": 0.34464187622070314, + "mean_token_accuracy": 0.91073057949543, + "num_tokens": 5725937.0, + "step": 2980 + }, + { + "entropy": 0.3204948931932449, + "epoch": 0.40680272108843535, + "grad_norm": 0.4274948835372925, + "learning_rate": 2.986986301369863e-05, + "loss": 0.3086549758911133, + "mean_token_accuracy": 0.9216050624847412, + "num_tokens": 5744274.0, + "step": 2990 + }, + { + "entropy": 0.319742513448, + "epoch": 0.40816326530612246, + "grad_norm": 0.43988749384880066, + "learning_rate": 2.9801369863013702e-05, + "loss": 0.3238105535507202, + "mean_token_accuracy": 0.9127194285392761, + "num_tokens": 5764224.0, + "step": 3000 + }, + { + "epoch": 0.40816326530612246, + "eval_entropy": 0.33606284650663537, + "eval_loss": 0.33526092767715454, + "eval_mean_token_accuracy": 0.9117824711402257, + "eval_num_tokens": 5764224.0, + "eval_runtime": 39.6422, + "eval_samples_per_second": 15.135, + "eval_steps_per_second": 7.568, + "step": 3000 + }, + { + "entropy": 0.3738269850611687, + "epoch": 0.4095238095238095, + "grad_norm": 0.5675323605537415, + "learning_rate": 2.9732876712328768e-05, + "loss": 0.3804152488708496, + "mean_token_accuracy": 0.9026560753583908, + "num_tokens": 5781507.0, + "step": 3010 + }, + { + "entropy": 0.35528839454054834, + "epoch": 0.4108843537414966, + "grad_norm": 0.37698662281036377, + "learning_rate": 2.9664383561643837e-05, + "loss": 0.35562241077423096, + "mean_token_accuracy": 0.9055985480546951, + "num_tokens": 5801540.0, + "step": 3020 + }, + { + "entropy": 0.34376412332057954, + "epoch": 0.4122448979591837, + "grad_norm": 0.48629990220069885, + "learning_rate": 2.9595890410958904e-05, + "loss": 0.3413914680480957, + "mean_token_accuracy": 0.9106980264186859, + "num_tokens": 5820095.0, + "step": 3030 + }, + { + "entropy": 0.28705876022577287, + "epoch": 0.41360544217687073, + "grad_norm": 0.3930319845676422, + "learning_rate": 2.9527397260273976e-05, + "loss": 0.2918708801269531, + "mean_token_accuracy": 0.9254306346178055, + "num_tokens": 5836806.0, + "step": 3040 + }, + { + "entropy": 0.28956920132040975, + "epoch": 0.41496598639455784, + "grad_norm": 0.3387986421585083, + "learning_rate": 2.9458904109589042e-05, + "loss": 0.28510375022888185, + "mean_token_accuracy": 0.9249218046665192, + "num_tokens": 5855499.0, + "step": 3050 + }, + { + "entropy": 0.3717873513698578, + "epoch": 0.4163265306122449, + "grad_norm": 0.5153553485870361, + "learning_rate": 2.9390410958904112e-05, + "loss": 0.36185555458068847, + "mean_token_accuracy": 0.9043754696846008, + "num_tokens": 5875651.0, + "step": 3060 + }, + { + "entropy": 0.3651160977780819, + "epoch": 0.41768707482993195, + "grad_norm": 0.622138261795044, + "learning_rate": 2.9321917808219178e-05, + "loss": 0.36736748218536375, + "mean_token_accuracy": 0.9075702458620072, + "num_tokens": 5893916.0, + "step": 3070 + }, + { + "entropy": 0.2804809086024761, + "epoch": 0.41904761904761906, + "grad_norm": 0.4219469726085663, + "learning_rate": 2.9253424657534244e-05, + "loss": 0.2844717025756836, + "mean_token_accuracy": 0.921401646733284, + "num_tokens": 5912149.0, + "step": 3080 + }, + { + "entropy": 0.314235956966877, + "epoch": 0.4204081632653061, + "grad_norm": 0.5418047308921814, + "learning_rate": 2.9184931506849317e-05, + "loss": 0.3110402822494507, + "mean_token_accuracy": 0.9149896830320359, + "num_tokens": 5932858.0, + "step": 3090 + }, + { + "entropy": 0.32378696352243425, + "epoch": 0.4217687074829932, + "grad_norm": 0.5057699680328369, + "learning_rate": 2.9116438356164383e-05, + "loss": 0.31246180534362794, + "mean_token_accuracy": 0.917678028345108, + "num_tokens": 5950456.0, + "step": 3100 + }, + { + "entropy": 0.3295158065855503, + "epoch": 0.4231292517006803, + "grad_norm": 0.32313019037246704, + "learning_rate": 2.9047945205479456e-05, + "loss": 0.32827062606811525, + "mean_token_accuracy": 0.913808137178421, + "num_tokens": 5970315.0, + "step": 3110 + }, + { + "entropy": 0.3258155144751072, + "epoch": 0.42448979591836733, + "grad_norm": 0.6149294972419739, + "learning_rate": 2.897945205479452e-05, + "loss": 0.3333756923675537, + "mean_token_accuracy": 0.9134649366140366, + "num_tokens": 5989083.0, + "step": 3120 + }, + { + "entropy": 0.33037797883152964, + "epoch": 0.42585034013605444, + "grad_norm": 0.6826731562614441, + "learning_rate": 2.891095890410959e-05, + "loss": 0.33439276218414304, + "mean_token_accuracy": 0.9117500305175781, + "num_tokens": 6008071.0, + "step": 3130 + }, + { + "entropy": 0.3115034744143486, + "epoch": 0.4272108843537415, + "grad_norm": 0.44018131494522095, + "learning_rate": 2.8842465753424658e-05, + "loss": 0.3135596990585327, + "mean_token_accuracy": 0.9178120344877243, + "num_tokens": 6028939.0, + "step": 3140 + }, + { + "entropy": 0.3361651122570038, + "epoch": 0.42857142857142855, + "grad_norm": 0.3561447858810425, + "learning_rate": 2.877397260273973e-05, + "loss": 0.3284465312957764, + "mean_token_accuracy": 0.912507513165474, + "num_tokens": 6050975.0, + "step": 3150 + }, + { + "entropy": 0.3167419731616974, + "epoch": 0.42993197278911566, + "grad_norm": 0.4988236725330353, + "learning_rate": 2.8705479452054797e-05, + "loss": 0.31540229320526125, + "mean_token_accuracy": 0.91805699467659, + "num_tokens": 6068811.0, + "step": 3160 + }, + { + "entropy": 0.35025607869029046, + "epoch": 0.4312925170068027, + "grad_norm": 0.3634585440158844, + "learning_rate": 2.8636986301369866e-05, + "loss": 0.34669277667999265, + "mean_token_accuracy": 0.908183416724205, + "num_tokens": 6089617.0, + "step": 3170 + }, + { + "entropy": 0.33410345017910004, + "epoch": 0.4326530612244898, + "grad_norm": 0.380386084318161, + "learning_rate": 2.8568493150684932e-05, + "loss": 0.35002827644348145, + "mean_token_accuracy": 0.9116801351308823, + "num_tokens": 6108293.0, + "step": 3180 + }, + { + "entropy": 0.3474400773644447, + "epoch": 0.4340136054421769, + "grad_norm": 0.41803106665611267, + "learning_rate": 2.8499999999999998e-05, + "loss": 0.35688033103942873, + "mean_token_accuracy": 0.9099474012851715, + "num_tokens": 6127624.0, + "step": 3190 + }, + { + "entropy": 0.3581741526722908, + "epoch": 0.43537414965986393, + "grad_norm": 0.3337022364139557, + "learning_rate": 2.843150684931507e-05, + "loss": 0.3569644451141357, + "mean_token_accuracy": 0.9068025410175323, + "num_tokens": 6147219.0, + "step": 3200 + }, + { + "entropy": 0.366722059994936, + "epoch": 0.43673469387755104, + "grad_norm": 0.47171923518180847, + "learning_rate": 2.8363013698630137e-05, + "loss": 0.3685397386550903, + "mean_token_accuracy": 0.9036656260490418, + "num_tokens": 6167669.0, + "step": 3210 + }, + { + "entropy": 0.32237810268998146, + "epoch": 0.4380952380952381, + "grad_norm": 0.41389840841293335, + "learning_rate": 2.8294520547945207e-05, + "loss": 0.31839354038238527, + "mean_token_accuracy": 0.9167311549186706, + "num_tokens": 6187141.0, + "step": 3220 + }, + { + "entropy": 0.3237307697534561, + "epoch": 0.43945578231292515, + "grad_norm": 0.5178200006484985, + "learning_rate": 2.8226027397260273e-05, + "loss": 0.32291412353515625, + "mean_token_accuracy": 0.9083501189947129, + "num_tokens": 6209252.0, + "step": 3230 + }, + { + "entropy": 0.34426340013742446, + "epoch": 0.44081632653061226, + "grad_norm": 0.4553462862968445, + "learning_rate": 2.8157534246575346e-05, + "loss": 0.3504526615142822, + "mean_token_accuracy": 0.9100097268819809, + "num_tokens": 6228894.0, + "step": 3240 + }, + { + "entropy": 0.28253506049513816, + "epoch": 0.4421768707482993, + "grad_norm": 0.37686920166015625, + "learning_rate": 2.808904109589041e-05, + "loss": 0.2929633617401123, + "mean_token_accuracy": 0.9215697795152664, + "num_tokens": 6249587.0, + "step": 3250 + }, + { + "epoch": 0.4421768707482993, + "eval_entropy": 0.3371030978113413, + "eval_loss": 0.3342117369174957, + "eval_mean_token_accuracy": 0.9119487730662028, + "eval_num_tokens": 6249587.0, + "eval_runtime": 39.7403, + "eval_samples_per_second": 15.098, + "eval_steps_per_second": 7.549, + "step": 3250 + }, + { + "entropy": 0.3359912931919098, + "epoch": 0.4435374149659864, + "grad_norm": 0.5517682433128357, + "learning_rate": 2.8020547945205484e-05, + "loss": 0.3350799322128296, + "mean_token_accuracy": 0.9072889268398285, + "num_tokens": 6270421.0, + "step": 3260 + }, + { + "entropy": 0.38644204661250114, + "epoch": 0.4448979591836735, + "grad_norm": 0.38367074728012085, + "learning_rate": 2.795205479452055e-05, + "loss": 0.40497770309448244, + "mean_token_accuracy": 0.8989514976739883, + "num_tokens": 6289750.0, + "step": 3270 + }, + { + "entropy": 0.36485532075166704, + "epoch": 0.44625850340136053, + "grad_norm": 0.4921395182609558, + "learning_rate": 2.7883561643835617e-05, + "loss": 0.3587671995162964, + "mean_token_accuracy": 0.9088137924671174, + "num_tokens": 6309476.0, + "step": 3280 + }, + { + "entropy": 0.32781576067209245, + "epoch": 0.44761904761904764, + "grad_norm": 0.405824214220047, + "learning_rate": 2.7815068493150686e-05, + "loss": 0.3323342323303223, + "mean_token_accuracy": 0.9141393929719925, + "num_tokens": 6330359.0, + "step": 3290 + }, + { + "entropy": 0.35046872347593305, + "epoch": 0.4489795918367347, + "grad_norm": 0.4356522858142853, + "learning_rate": 2.7746575342465752e-05, + "loss": 0.3580676555633545, + "mean_token_accuracy": 0.9075710386037826, + "num_tokens": 6350395.0, + "step": 3300 + }, + { + "entropy": 0.3306349150836468, + "epoch": 0.45034013605442175, + "grad_norm": 0.3762452304363251, + "learning_rate": 2.7678082191780825e-05, + "loss": 0.3267848968505859, + "mean_token_accuracy": 0.9178031712770462, + "num_tokens": 6368777.0, + "step": 3310 + }, + { + "entropy": 0.29314617067575455, + "epoch": 0.45170068027210886, + "grad_norm": 0.4260740876197815, + "learning_rate": 2.760958904109589e-05, + "loss": 0.2808928728103638, + "mean_token_accuracy": 0.9242286056280136, + "num_tokens": 6386705.0, + "step": 3320 + }, + { + "entropy": 0.33772081658244135, + "epoch": 0.4530612244897959, + "grad_norm": 0.4309479594230652, + "learning_rate": 2.754109589041096e-05, + "loss": 0.3572484731674194, + "mean_token_accuracy": 0.9095708549022674, + "num_tokens": 6405597.0, + "step": 3330 + }, + { + "entropy": 0.3147235192358494, + "epoch": 0.454421768707483, + "grad_norm": 0.6065618395805359, + "learning_rate": 2.7472602739726027e-05, + "loss": 0.3092878580093384, + "mean_token_accuracy": 0.9218635380268096, + "num_tokens": 6425006.0, + "step": 3340 + }, + { + "entropy": 0.35482636019587516, + "epoch": 0.4557823129251701, + "grad_norm": 0.40962573885917664, + "learning_rate": 2.74041095890411e-05, + "loss": 0.3521085023880005, + "mean_token_accuracy": 0.9050173759460449, + "num_tokens": 6444964.0, + "step": 3350 + }, + { + "entropy": 0.3443373300135136, + "epoch": 0.45714285714285713, + "grad_norm": 0.47444677352905273, + "learning_rate": 2.7335616438356166e-05, + "loss": 0.3417380332946777, + "mean_token_accuracy": 0.9109756410121918, + "num_tokens": 6464883.0, + "step": 3360 + }, + { + "entropy": 0.30666022598743437, + "epoch": 0.45850340136054424, + "grad_norm": 0.3451339900493622, + "learning_rate": 2.7267123287671235e-05, + "loss": 0.308093786239624, + "mean_token_accuracy": 0.9187242925167084, + "num_tokens": 6484170.0, + "step": 3370 + }, + { + "entropy": 0.32256651744246484, + "epoch": 0.4598639455782313, + "grad_norm": 0.6191990971565247, + "learning_rate": 2.71986301369863e-05, + "loss": 0.3281312704086304, + "mean_token_accuracy": 0.915521040558815, + "num_tokens": 6502657.0, + "step": 3380 + }, + { + "entropy": 0.3734627470374107, + "epoch": 0.46122448979591835, + "grad_norm": 0.44824740290641785, + "learning_rate": 2.7130136986301367e-05, + "loss": 0.39291067123413087, + "mean_token_accuracy": 0.9002336442470551, + "num_tokens": 6522335.0, + "step": 3390 + }, + { + "entropy": 0.36504360362887384, + "epoch": 0.46258503401360546, + "grad_norm": 0.44734102487564087, + "learning_rate": 2.706164383561644e-05, + "loss": 0.366800332069397, + "mean_token_accuracy": 0.9116403192281723, + "num_tokens": 6541259.0, + "step": 3400 + }, + { + "entropy": 0.33948337733745576, + "epoch": 0.4639455782312925, + "grad_norm": 0.4975481331348419, + "learning_rate": 2.6993150684931506e-05, + "loss": 0.3351884126663208, + "mean_token_accuracy": 0.911971241235733, + "num_tokens": 6562033.0, + "step": 3410 + }, + { + "entropy": 0.36278425380587576, + "epoch": 0.46530612244897956, + "grad_norm": 0.44590020179748535, + "learning_rate": 2.692465753424658e-05, + "loss": 0.3616467475891113, + "mean_token_accuracy": 0.9112220734357834, + "num_tokens": 6579861.0, + "step": 3420 + }, + { + "entropy": 0.296990454941988, + "epoch": 0.4666666666666667, + "grad_norm": 0.35668641328811646, + "learning_rate": 2.6856164383561645e-05, + "loss": 0.28709557056427004, + "mean_token_accuracy": 0.923720845580101, + "num_tokens": 6597963.0, + "step": 3430 + }, + { + "entropy": 0.30575042217969894, + "epoch": 0.46802721088435373, + "grad_norm": 0.39647671580314636, + "learning_rate": 2.6787671232876715e-05, + "loss": 0.3078245401382446, + "mean_token_accuracy": 0.9169597178697586, + "num_tokens": 6616696.0, + "step": 3440 + }, + { + "entropy": 0.3101175561547279, + "epoch": 0.46938775510204084, + "grad_norm": 0.5445443987846375, + "learning_rate": 2.671917808219178e-05, + "loss": 0.3175575494766235, + "mean_token_accuracy": 0.9126063853502273, + "num_tokens": 6637062.0, + "step": 3450 + }, + { + "entropy": 0.34780861288309095, + "epoch": 0.4707482993197279, + "grad_norm": 0.3975132405757904, + "learning_rate": 2.6650684931506854e-05, + "loss": 0.35774109363555906, + "mean_token_accuracy": 0.9050011664628983, + "num_tokens": 6659231.0, + "step": 3460 + }, + { + "entropy": 0.32717221304774285, + "epoch": 0.47210884353741495, + "grad_norm": 0.4195864796638489, + "learning_rate": 2.658219178082192e-05, + "loss": 0.3109439373016357, + "mean_token_accuracy": 0.9161674290895462, + "num_tokens": 6679690.0, + "step": 3470 + }, + { + "entropy": 0.3339711882174015, + "epoch": 0.47346938775510206, + "grad_norm": 0.35985931754112244, + "learning_rate": 2.6513698630136986e-05, + "loss": 0.33592188358306885, + "mean_token_accuracy": 0.9106827944517135, + "num_tokens": 6698470.0, + "step": 3480 + }, + { + "entropy": 0.3498953923583031, + "epoch": 0.4748299319727891, + "grad_norm": 0.3127373158931732, + "learning_rate": 2.6445205479452055e-05, + "loss": 0.36162471771240234, + "mean_token_accuracy": 0.9053647011518479, + "num_tokens": 6719482.0, + "step": 3490 + }, + { + "entropy": 0.28822562620043757, + "epoch": 0.47619047619047616, + "grad_norm": 0.4204389452934265, + "learning_rate": 2.637671232876712e-05, + "loss": 0.29782910346984864, + "mean_token_accuracy": 0.9186246633529663, + "num_tokens": 6740195.0, + "step": 3500 + }, + { + "epoch": 0.47619047619047616, + "eval_entropy": 0.3430745062977076, + "eval_loss": 0.33298492431640625, + "eval_mean_token_accuracy": 0.9121425338586171, + "eval_num_tokens": 6740195.0, + "eval_runtime": 39.8932, + "eval_samples_per_second": 15.04, + "eval_steps_per_second": 7.52, + "step": 3500 + }, + { + "entropy": 0.32787102460861206, + "epoch": 0.4775510204081633, + "grad_norm": 0.5970384478569031, + "learning_rate": 2.6308219178082194e-05, + "loss": 0.31883673667907714, + "mean_token_accuracy": 0.9141757398843765, + "num_tokens": 6758388.0, + "step": 3510 + }, + { + "entropy": 0.36540004089474676, + "epoch": 0.47891156462585033, + "grad_norm": 0.5385341644287109, + "learning_rate": 2.623972602739726e-05, + "loss": 0.3638452053070068, + "mean_token_accuracy": 0.9118665605783463, + "num_tokens": 6776273.0, + "step": 3520 + }, + { + "entropy": 0.3373605191707611, + "epoch": 0.48027210884353744, + "grad_norm": 0.38150331377983093, + "learning_rate": 2.6171232876712333e-05, + "loss": 0.3207834243774414, + "mean_token_accuracy": 0.9155988216400146, + "num_tokens": 6793132.0, + "step": 3530 + }, + { + "entropy": 0.34099703952670096, + "epoch": 0.4816326530612245, + "grad_norm": 0.5244598984718323, + "learning_rate": 2.61027397260274e-05, + "loss": 0.33450777530670167, + "mean_token_accuracy": 0.9131689667701721, + "num_tokens": 6812749.0, + "step": 3540 + }, + { + "entropy": 0.43126574754714964, + "epoch": 0.48299319727891155, + "grad_norm": 0.4791218638420105, + "learning_rate": 2.603424657534247e-05, + "loss": 0.43479280471801757, + "mean_token_accuracy": 0.8856066018342972, + "num_tokens": 6833553.0, + "step": 3550 + }, + { + "entropy": 0.3489451795816422, + "epoch": 0.48435374149659866, + "grad_norm": 0.6042513251304626, + "learning_rate": 2.5965753424657535e-05, + "loss": 0.3617150068283081, + "mean_token_accuracy": 0.904024675488472, + "num_tokens": 6851609.0, + "step": 3560 + }, + { + "entropy": 0.3345202341675758, + "epoch": 0.4857142857142857, + "grad_norm": 0.47959575057029724, + "learning_rate": 2.5897260273972608e-05, + "loss": 0.34854423999786377, + "mean_token_accuracy": 0.9128339350223541, + "num_tokens": 6869710.0, + "step": 3570 + }, + { + "entropy": 0.2840237572789192, + "epoch": 0.48707482993197276, + "grad_norm": 0.542008638381958, + "learning_rate": 2.5828767123287674e-05, + "loss": 0.28817789554595946, + "mean_token_accuracy": 0.9232237786054611, + "num_tokens": 6889265.0, + "step": 3580 + }, + { + "entropy": 0.37114314883947375, + "epoch": 0.4884353741496599, + "grad_norm": 0.42148521542549133, + "learning_rate": 2.576027397260274e-05, + "loss": 0.3705109119415283, + "mean_token_accuracy": 0.9055974930524826, + "num_tokens": 6907724.0, + "step": 3590 + }, + { + "entropy": 0.3097227029502392, + "epoch": 0.4897959183673469, + "grad_norm": 0.5400570631027222, + "learning_rate": 2.569178082191781e-05, + "loss": 0.3039916276931763, + "mean_token_accuracy": 0.915545716881752, + "num_tokens": 6927218.0, + "step": 3600 + }, + { + "entropy": 0.311353924870491, + "epoch": 0.49115646258503404, + "grad_norm": 0.36434206366539, + "learning_rate": 2.5623287671232875e-05, + "loss": 0.3071993589401245, + "mean_token_accuracy": 0.9152117878198623, + "num_tokens": 6948124.0, + "step": 3610 + }, + { + "entropy": 0.34956820532679556, + "epoch": 0.4925170068027211, + "grad_norm": 0.3756413161754608, + "learning_rate": 2.5554794520547948e-05, + "loss": 0.345737624168396, + "mean_token_accuracy": 0.9104509115219116, + "num_tokens": 6967337.0, + "step": 3620 + }, + { + "entropy": 0.28347715772688387, + "epoch": 0.49387755102040815, + "grad_norm": 0.3341163694858551, + "learning_rate": 2.5486301369863014e-05, + "loss": 0.28453564643859863, + "mean_token_accuracy": 0.9223557978868484, + "num_tokens": 6985719.0, + "step": 3630 + }, + { + "entropy": 0.3215967521071434, + "epoch": 0.49523809523809526, + "grad_norm": 0.4879300892353058, + "learning_rate": 2.5417808219178084e-05, + "loss": 0.3172642230987549, + "mean_token_accuracy": 0.9150819301605224, + "num_tokens": 7003145.0, + "step": 3640 + }, + { + "entropy": 0.3079038575291634, + "epoch": 0.4965986394557823, + "grad_norm": 0.3948201537132263, + "learning_rate": 2.534931506849315e-05, + "loss": 0.32240614891052244, + "mean_token_accuracy": 0.9127428829669952, + "num_tokens": 7023038.0, + "step": 3650 + }, + { + "entropy": 0.38890808299183843, + "epoch": 0.49795918367346936, + "grad_norm": 0.40552157163619995, + "learning_rate": 2.5280821917808223e-05, + "loss": 0.39409432411193845, + "mean_token_accuracy": 0.897028648853302, + "num_tokens": 7046752.0, + "step": 3660 + }, + { + "entropy": 0.3068940281867981, + "epoch": 0.4993197278911565, + "grad_norm": 0.5080393552780151, + "learning_rate": 2.521232876712329e-05, + "loss": 0.2945326566696167, + "mean_token_accuracy": 0.9193342208862305, + "num_tokens": 7066803.0, + "step": 3670 + }, + { + "entropy": 0.30003788247704505, + "epoch": 0.5006802721088436, + "grad_norm": 0.4544333517551422, + "learning_rate": 2.5143835616438355e-05, + "loss": 0.30088942050933837, + "mean_token_accuracy": 0.919922587275505, + "num_tokens": 7086548.0, + "step": 3680 + }, + { + "entropy": 0.3212012261152267, + "epoch": 0.5020408163265306, + "grad_norm": 0.5139970183372498, + "learning_rate": 2.5075342465753428e-05, + "loss": 0.3209099054336548, + "mean_token_accuracy": 0.914613538980484, + "num_tokens": 7103932.0, + "step": 3690 + }, + { + "entropy": 0.3605083726346493, + "epoch": 0.5034013605442177, + "grad_norm": 0.5004434585571289, + "learning_rate": 2.5006849315068494e-05, + "loss": 0.3503889799118042, + "mean_token_accuracy": 0.910282626748085, + "num_tokens": 7122604.0, + "step": 3700 + }, + { + "entropy": 0.32034995295107366, + "epoch": 0.5047619047619047, + "grad_norm": 0.4455152451992035, + "learning_rate": 2.4938356164383563e-05, + "loss": 0.3144600629806519, + "mean_token_accuracy": 0.913508927822113, + "num_tokens": 7141520.0, + "step": 3710 + }, + { + "entropy": 0.32677931264042853, + "epoch": 0.5061224489795918, + "grad_norm": 0.5444691181182861, + "learning_rate": 2.4869863013698633e-05, + "loss": 0.33878235816955565, + "mean_token_accuracy": 0.9117316961288452, + "num_tokens": 7162081.0, + "step": 3720 + }, + { + "entropy": 0.35325164198875425, + "epoch": 0.507482993197279, + "grad_norm": 0.504077136516571, + "learning_rate": 2.48013698630137e-05, + "loss": 0.3583226680755615, + "mean_token_accuracy": 0.9071122765541076, + "num_tokens": 7181824.0, + "step": 3730 + }, + { + "entropy": 0.3118510887026787, + "epoch": 0.508843537414966, + "grad_norm": 0.45883047580718994, + "learning_rate": 2.473287671232877e-05, + "loss": 0.30337512493133545, + "mean_token_accuracy": 0.9171320110559463, + "num_tokens": 7200624.0, + "step": 3740 + }, + { + "entropy": 0.329176065325737, + "epoch": 0.5102040816326531, + "grad_norm": 0.48111435770988464, + "learning_rate": 2.4664383561643838e-05, + "loss": 0.33903625011444094, + "mean_token_accuracy": 0.9144390225410461, + "num_tokens": 7217963.0, + "step": 3750 + }, + { + "epoch": 0.5102040816326531, + "eval_entropy": 0.32912359858552614, + "eval_loss": 0.3315099775791168, + "eval_mean_token_accuracy": 0.9126162391901016, + "eval_num_tokens": 7217963.0, + "eval_runtime": 39.8303, + "eval_samples_per_second": 15.064, + "eval_steps_per_second": 7.532, + "step": 3750 + }, + { + "entropy": 0.3421541929244995, + "epoch": 0.5115646258503401, + "grad_norm": 0.4506266415119171, + "learning_rate": 2.4595890410958904e-05, + "loss": 0.34347808361053467, + "mean_token_accuracy": 0.9077278554439545, + "num_tokens": 7237883.0, + "step": 3760 + }, + { + "entropy": 0.3590623676776886, + "epoch": 0.5129251700680272, + "grad_norm": 0.38278287649154663, + "learning_rate": 2.4527397260273973e-05, + "loss": 0.3690760850906372, + "mean_token_accuracy": 0.9089499413967133, + "num_tokens": 7257138.0, + "step": 3770 + }, + { + "entropy": 0.36101208850741384, + "epoch": 0.5142857142857142, + "grad_norm": 0.32584571838378906, + "learning_rate": 2.4458904109589043e-05, + "loss": 0.3672176837921143, + "mean_token_accuracy": 0.9057380646467209, + "num_tokens": 7277589.0, + "step": 3780 + }, + { + "entropy": 0.2965987451374531, + "epoch": 0.5156462585034014, + "grad_norm": 0.37990301847457886, + "learning_rate": 2.4390410958904112e-05, + "loss": 0.294060206413269, + "mean_token_accuracy": 0.920094347000122, + "num_tokens": 7295419.0, + "step": 3790 + }, + { + "entropy": 0.3669845648109913, + "epoch": 0.5170068027210885, + "grad_norm": 0.395302414894104, + "learning_rate": 2.432191780821918e-05, + "loss": 0.3642202138900757, + "mean_token_accuracy": 0.9083656400442124, + "num_tokens": 7313005.0, + "step": 3800 + }, + { + "entropy": 0.3839815385639668, + "epoch": 0.5183673469387755, + "grad_norm": 0.4179222881793976, + "learning_rate": 2.4253424657534248e-05, + "loss": 0.38060669898986815, + "mean_token_accuracy": 0.9010459512472153, + "num_tokens": 7334087.0, + "step": 3810 + }, + { + "entropy": 0.33595051877200605, + "epoch": 0.5197278911564626, + "grad_norm": 0.35726672410964966, + "learning_rate": 2.4184931506849314e-05, + "loss": 0.3160902976989746, + "mean_token_accuracy": 0.9160772681236267, + "num_tokens": 7353243.0, + "step": 3820 + }, + { + "entropy": 0.3190218761563301, + "epoch": 0.5210884353741496, + "grad_norm": 0.39096274971961975, + "learning_rate": 2.4116438356164384e-05, + "loss": 0.3128466844558716, + "mean_token_accuracy": 0.9177798658609391, + "num_tokens": 7372683.0, + "step": 3830 + }, + { + "entropy": 0.28777577802538873, + "epoch": 0.5224489795918368, + "grad_norm": 0.4006626605987549, + "learning_rate": 2.4047945205479453e-05, + "loss": 0.2932948350906372, + "mean_token_accuracy": 0.9220825433731079, + "num_tokens": 7391156.0, + "step": 3840 + }, + { + "entropy": 0.3199619509279728, + "epoch": 0.5238095238095238, + "grad_norm": 0.35508039593696594, + "learning_rate": 2.3979452054794522e-05, + "loss": 0.3138143539428711, + "mean_token_accuracy": 0.9143597453832626, + "num_tokens": 7412725.0, + "step": 3850 + }, + { + "entropy": 0.35134848281741143, + "epoch": 0.5251700680272109, + "grad_norm": 0.406851202249527, + "learning_rate": 2.391095890410959e-05, + "loss": 0.35069732666015624, + "mean_token_accuracy": 0.910768261551857, + "num_tokens": 7431365.0, + "step": 3860 + }, + { + "entropy": 0.3226579576730728, + "epoch": 0.5265306122448979, + "grad_norm": 0.41831570863723755, + "learning_rate": 2.3842465753424658e-05, + "loss": 0.325051474571228, + "mean_token_accuracy": 0.9144408613443374, + "num_tokens": 7451520.0, + "step": 3870 + }, + { + "entropy": 0.27824380323290826, + "epoch": 0.527891156462585, + "grad_norm": 0.45743539929389954, + "learning_rate": 2.3773972602739727e-05, + "loss": 0.27892796993255614, + "mean_token_accuracy": 0.9252461075782776, + "num_tokens": 7470779.0, + "step": 3880 + }, + { + "entropy": 0.31983816549181937, + "epoch": 0.5292517006802722, + "grad_norm": 0.41483598947525024, + "learning_rate": 2.3705479452054797e-05, + "loss": 0.3180886507034302, + "mean_token_accuracy": 0.9165385901927948, + "num_tokens": 7489545.0, + "step": 3890 + }, + { + "entropy": 0.3343293689191341, + "epoch": 0.5306122448979592, + "grad_norm": 0.5109151601791382, + "learning_rate": 2.3636986301369866e-05, + "loss": 0.34193923473358157, + "mean_token_accuracy": 0.9141032576560975, + "num_tokens": 7507380.0, + "step": 3900 + }, + { + "entropy": 0.3208622790873051, + "epoch": 0.5319727891156463, + "grad_norm": 0.4916702210903168, + "learning_rate": 2.3568493150684933e-05, + "loss": 0.3239308834075928, + "mean_token_accuracy": 0.914379420876503, + "num_tokens": 7526466.0, + "step": 3910 + }, + { + "entropy": 0.3093146879225969, + "epoch": 0.5333333333333333, + "grad_norm": 0.32689499855041504, + "learning_rate": 2.35e-05, + "loss": 0.3163978815078735, + "mean_token_accuracy": 0.9175358593463898, + "num_tokens": 7547417.0, + "step": 3920 + }, + { + "entropy": 0.3612152311950922, + "epoch": 0.5346938775510204, + "grad_norm": 0.44085893034935, + "learning_rate": 2.3431506849315068e-05, + "loss": 0.3802115678787231, + "mean_token_accuracy": 0.902223750948906, + "num_tokens": 7568296.0, + "step": 3930 + }, + { + "entropy": 0.3383258245885372, + "epoch": 0.5360544217687074, + "grad_norm": 0.5608168244361877, + "learning_rate": 2.3363013698630138e-05, + "loss": 0.3415839433670044, + "mean_token_accuracy": 0.9112567931413651, + "num_tokens": 7587752.0, + "step": 3940 + }, + { + "entropy": 0.3647989623248577, + "epoch": 0.5374149659863946, + "grad_norm": 0.4722004532814026, + "learning_rate": 2.3294520547945207e-05, + "loss": 0.3584833860397339, + "mean_token_accuracy": 0.9099332094192505, + "num_tokens": 7607739.0, + "step": 3950 + }, + { + "entropy": 0.3759518824517727, + "epoch": 0.5387755102040817, + "grad_norm": 0.6017438173294067, + "learning_rate": 2.3226027397260277e-05, + "loss": 0.3851759433746338, + "mean_token_accuracy": 0.902659210562706, + "num_tokens": 7625816.0, + "step": 3960 + }, + { + "entropy": 0.31676906198263166, + "epoch": 0.5401360544217687, + "grad_norm": 0.3612753450870514, + "learning_rate": 2.3157534246575343e-05, + "loss": 0.31891517639160155, + "mean_token_accuracy": 0.9155329942703248, + "num_tokens": 7646074.0, + "step": 3970 + }, + { + "entropy": 0.3787412360310555, + "epoch": 0.5414965986394558, + "grad_norm": 0.41047224402427673, + "learning_rate": 2.3089041095890412e-05, + "loss": 0.3822017192840576, + "mean_token_accuracy": 0.9003373116254807, + "num_tokens": 7664536.0, + "step": 3980 + }, + { + "entropy": 0.3361421704292297, + "epoch": 0.5428571428571428, + "grad_norm": 0.41327348351478577, + "learning_rate": 2.302054794520548e-05, + "loss": 0.3355587959289551, + "mean_token_accuracy": 0.911680155992508, + "num_tokens": 7682379.0, + "step": 3990 + }, + { + "entropy": 0.3021339297294617, + "epoch": 0.54421768707483, + "grad_norm": 0.36242055892944336, + "learning_rate": 2.295205479452055e-05, + "loss": 0.3072765111923218, + "mean_token_accuracy": 0.9183337926864624, + "num_tokens": 7701447.0, + "step": 4000 + }, + { + "epoch": 0.54421768707483, + "eval_entropy": 0.3300413696716229, + "eval_loss": 0.3298623263835907, + "eval_mean_token_accuracy": 0.9126938621203105, + "eval_num_tokens": 7701447.0, + "eval_runtime": 39.9379, + "eval_samples_per_second": 15.023, + "eval_steps_per_second": 7.512, + "step": 4000 + }, + { + "entropy": 0.30280276834964753, + "epoch": 0.545578231292517, + "grad_norm": 0.5451289415359497, + "learning_rate": 2.2883561643835617e-05, + "loss": 0.30348217487335205, + "mean_token_accuracy": 0.9159641265869141, + "num_tokens": 7723410.0, + "step": 4010 + }, + { + "entropy": 0.26205305345356467, + "epoch": 0.5469387755102041, + "grad_norm": 0.4319462776184082, + "learning_rate": 2.2815068493150683e-05, + "loss": 0.2545130252838135, + "mean_token_accuracy": 0.9308669567108154, + "num_tokens": 7743532.0, + "step": 4020 + }, + { + "entropy": 0.32944763377308844, + "epoch": 0.5482993197278911, + "grad_norm": 0.5175082087516785, + "learning_rate": 2.2746575342465753e-05, + "loss": 0.3209054946899414, + "mean_token_accuracy": 0.9140241533517838, + "num_tokens": 7764584.0, + "step": 4030 + }, + { + "entropy": 0.3470706418156624, + "epoch": 0.5496598639455782, + "grad_norm": 0.4611230790615082, + "learning_rate": 2.2678082191780822e-05, + "loss": 0.3442211151123047, + "mean_token_accuracy": 0.9116791754961013, + "num_tokens": 7783273.0, + "step": 4040 + }, + { + "entropy": 0.3467777229845524, + "epoch": 0.5510204081632653, + "grad_norm": 0.5174922943115234, + "learning_rate": 2.260958904109589e-05, + "loss": 0.3491501331329346, + "mean_token_accuracy": 0.9060128688812256, + "num_tokens": 7803308.0, + "step": 4050 + }, + { + "entropy": 0.3715716302394867, + "epoch": 0.5523809523809524, + "grad_norm": 0.5032371878623962, + "learning_rate": 2.254109589041096e-05, + "loss": 0.36512949466705324, + "mean_token_accuracy": 0.9032170832157135, + "num_tokens": 7823268.0, + "step": 4060 + }, + { + "entropy": 0.355232909321785, + "epoch": 0.5537414965986395, + "grad_norm": 0.3794972896575928, + "learning_rate": 2.2472602739726027e-05, + "loss": 0.3677065849304199, + "mean_token_accuracy": 0.9080590635538102, + "num_tokens": 7843048.0, + "step": 4070 + }, + { + "entropy": 0.3806071810424328, + "epoch": 0.5551020408163265, + "grad_norm": 0.398653507232666, + "learning_rate": 2.2404109589041097e-05, + "loss": 0.37912545204162595, + "mean_token_accuracy": 0.9007632732391357, + "num_tokens": 7862587.0, + "step": 4080 + }, + { + "entropy": 0.29905916675925254, + "epoch": 0.5564625850340136, + "grad_norm": 0.39064309000968933, + "learning_rate": 2.2335616438356166e-05, + "loss": 0.2870203971862793, + "mean_token_accuracy": 0.9261332839727402, + "num_tokens": 7880399.0, + "step": 4090 + }, + { + "entropy": 0.3375513479113579, + "epoch": 0.5578231292517006, + "grad_norm": 0.4097396433353424, + "learning_rate": 2.2267123287671236e-05, + "loss": 0.3322545289993286, + "mean_token_accuracy": 0.9148608386516571, + "num_tokens": 7901445.0, + "step": 4100 + }, + { + "entropy": 0.3285654172301292, + "epoch": 0.5591836734693878, + "grad_norm": 0.47398117184638977, + "learning_rate": 2.2198630136986305e-05, + "loss": 0.32115213871002196, + "mean_token_accuracy": 0.9161695718765259, + "num_tokens": 7919428.0, + "step": 4110 + }, + { + "entropy": 0.32259594202041625, + "epoch": 0.5605442176870749, + "grad_norm": 0.4475609362125397, + "learning_rate": 2.213013698630137e-05, + "loss": 0.32392292022705077, + "mean_token_accuracy": 0.9168900787830353, + "num_tokens": 7938761.0, + "step": 4120 + }, + { + "entropy": 0.32752382531762125, + "epoch": 0.5619047619047619, + "grad_norm": 0.41794857382774353, + "learning_rate": 2.2061643835616437e-05, + "loss": 0.3300853967666626, + "mean_token_accuracy": 0.911037164926529, + "num_tokens": 7957516.0, + "step": 4130 + }, + { + "entropy": 0.3652478508651257, + "epoch": 0.563265306122449, + "grad_norm": 0.4401056468486786, + "learning_rate": 2.1993150684931507e-05, + "loss": 0.3552700519561768, + "mean_token_accuracy": 0.9047274440526962, + "num_tokens": 7977608.0, + "step": 4140 + }, + { + "entropy": 0.343021310120821, + "epoch": 0.564625850340136, + "grad_norm": 0.46899110078811646, + "learning_rate": 2.1924657534246576e-05, + "loss": 0.34839396476745604, + "mean_token_accuracy": 0.9092651695013046, + "num_tokens": 7995943.0, + "step": 4150 + }, + { + "entropy": 0.3456800006330013, + "epoch": 0.5659863945578232, + "grad_norm": 0.5434921383857727, + "learning_rate": 2.1856164383561646e-05, + "loss": 0.3406283140182495, + "mean_token_accuracy": 0.9088764578104019, + "num_tokens": 8016350.0, + "step": 4160 + }, + { + "entropy": 0.3137536272406578, + "epoch": 0.5673469387755102, + "grad_norm": 0.5325484871864319, + "learning_rate": 2.1787671232876715e-05, + "loss": 0.3117046594619751, + "mean_token_accuracy": 0.9154979318380356, + "num_tokens": 8037048.0, + "step": 4170 + }, + { + "entropy": 0.3241760052740574, + "epoch": 0.5687074829931973, + "grad_norm": 0.3219553232192993, + "learning_rate": 2.171917808219178e-05, + "loss": 0.319587254524231, + "mean_token_accuracy": 0.9146761208772659, + "num_tokens": 8056300.0, + "step": 4180 + }, + { + "entropy": 0.2647711656987667, + "epoch": 0.5700680272108843, + "grad_norm": 0.5715179443359375, + "learning_rate": 2.165068493150685e-05, + "loss": 0.2673149585723877, + "mean_token_accuracy": 0.9255240142345429, + "num_tokens": 8074144.0, + "step": 4190 + }, + { + "entropy": 0.3415665104985237, + "epoch": 0.5714285714285714, + "grad_norm": 0.4954402446746826, + "learning_rate": 2.158219178082192e-05, + "loss": 0.3447671175003052, + "mean_token_accuracy": 0.9117257714271545, + "num_tokens": 8091291.0, + "step": 4200 + }, + { + "entropy": 0.30447668135166167, + "epoch": 0.5727891156462585, + "grad_norm": 0.3845031261444092, + "learning_rate": 2.151369863013699e-05, + "loss": 0.29765496253967283, + "mean_token_accuracy": 0.9206657618284225, + "num_tokens": 8109439.0, + "step": 4210 + }, + { + "entropy": 0.3095638811588287, + "epoch": 0.5741496598639456, + "grad_norm": 0.3968900740146637, + "learning_rate": 2.1445205479452056e-05, + "loss": 0.322363543510437, + "mean_token_accuracy": 0.9155390083789825, + "num_tokens": 8131053.0, + "step": 4220 + }, + { + "entropy": 0.33604917004704477, + "epoch": 0.5755102040816327, + "grad_norm": 0.45397162437438965, + "learning_rate": 2.1376712328767122e-05, + "loss": 0.33268921375274657, + "mean_token_accuracy": 0.9129331290721894, + "num_tokens": 8152572.0, + "step": 4230 + }, + { + "entropy": 0.3526121251285076, + "epoch": 0.5768707482993197, + "grad_norm": 0.3945638835430145, + "learning_rate": 2.130821917808219e-05, + "loss": 0.35298843383789064, + "mean_token_accuracy": 0.9088378429412842, + "num_tokens": 8171370.0, + "step": 4240 + }, + { + "entropy": 0.3636915013194084, + "epoch": 0.5782312925170068, + "grad_norm": 0.5967618823051453, + "learning_rate": 2.123972602739726e-05, + "loss": 0.37089223861694337, + "mean_token_accuracy": 0.9077885746955872, + "num_tokens": 8188389.0, + "step": 4250 + }, + { + "epoch": 0.5782312925170068, + "eval_entropy": 0.32578996996084847, + "eval_loss": 0.329336941242218, + "eval_mean_token_accuracy": 0.9131122646729152, + "eval_num_tokens": 8188389.0, + "eval_runtime": 38.822, + "eval_samples_per_second": 15.455, + "eval_steps_per_second": 7.728, + "step": 4250 + }, + { + "entropy": 0.31215301901102066, + "epoch": 0.5795918367346938, + "grad_norm": 0.4824981689453125, + "learning_rate": 2.117123287671233e-05, + "loss": 0.3266132831573486, + "mean_token_accuracy": 0.9147640526294708, + "num_tokens": 8206409.0, + "step": 4260 + }, + { + "entropy": 0.3209221847355366, + "epoch": 0.580952380952381, + "grad_norm": 0.6410835981369019, + "learning_rate": 2.11027397260274e-05, + "loss": 0.31994919776916503, + "mean_token_accuracy": 0.9160535246133804, + "num_tokens": 8226250.0, + "step": 4270 + }, + { + "entropy": 0.3772594600915909, + "epoch": 0.582312925170068, + "grad_norm": 0.4383021891117096, + "learning_rate": 2.1034246575342466e-05, + "loss": 0.3900163650512695, + "mean_token_accuracy": 0.9026447653770446, + "num_tokens": 8245881.0, + "step": 4280 + }, + { + "entropy": 0.30700850337743757, + "epoch": 0.5836734693877551, + "grad_norm": 0.4085005521774292, + "learning_rate": 2.0965753424657535e-05, + "loss": 0.30626041889190675, + "mean_token_accuracy": 0.9162427425384522, + "num_tokens": 8264812.0, + "step": 4290 + }, + { + "entropy": 0.3398926958441734, + "epoch": 0.5850340136054422, + "grad_norm": 0.5330114960670471, + "learning_rate": 2.0897260273972605e-05, + "loss": 0.34441685676574707, + "mean_token_accuracy": 0.9106619924306869, + "num_tokens": 8283010.0, + "step": 4300 + }, + { + "entropy": 0.35671548917889595, + "epoch": 0.5863945578231292, + "grad_norm": 0.45322227478027344, + "learning_rate": 2.0828767123287674e-05, + "loss": 0.3564306259155273, + "mean_token_accuracy": 0.9076927453279495, + "num_tokens": 8300835.0, + "step": 4310 + }, + { + "entropy": 0.37111207768321036, + "epoch": 0.5877551020408164, + "grad_norm": 0.4854358732700348, + "learning_rate": 2.076027397260274e-05, + "loss": 0.3827904939651489, + "mean_token_accuracy": 0.9036923080682755, + "num_tokens": 8320214.0, + "step": 4320 + }, + { + "entropy": 0.36895843744277956, + "epoch": 0.5891156462585034, + "grad_norm": 0.4703625738620758, + "learning_rate": 2.069178082191781e-05, + "loss": 0.3752447128295898, + "mean_token_accuracy": 0.9071332007646561, + "num_tokens": 8339775.0, + "step": 4330 + }, + { + "entropy": 0.39478201493620874, + "epoch": 0.5904761904761905, + "grad_norm": 0.4411693513393402, + "learning_rate": 2.0623287671232876e-05, + "loss": 0.3966845989227295, + "mean_token_accuracy": 0.8968111395835876, + "num_tokens": 8360604.0, + "step": 4340 + }, + { + "entropy": 0.41347152292728423, + "epoch": 0.5918367346938775, + "grad_norm": 0.47045063972473145, + "learning_rate": 2.0554794520547945e-05, + "loss": 0.41939425468444824, + "mean_token_accuracy": 0.8988784611225128, + "num_tokens": 8379152.0, + "step": 4350 + }, + { + "entropy": 0.28764098733663557, + "epoch": 0.5931972789115646, + "grad_norm": 0.4305931329727173, + "learning_rate": 2.0486301369863015e-05, + "loss": 0.2825281858444214, + "mean_token_accuracy": 0.9262870222330093, + "num_tokens": 8397421.0, + "step": 4360 + }, + { + "entropy": 0.3454031616449356, + "epoch": 0.5945578231292517, + "grad_norm": 0.5220937132835388, + "learning_rate": 2.0417808219178084e-05, + "loss": 0.3588573694229126, + "mean_token_accuracy": 0.9089827597141266, + "num_tokens": 8416839.0, + "step": 4370 + }, + { + "entropy": 0.3660218968987465, + "epoch": 0.5959183673469388, + "grad_norm": 0.8208481073379517, + "learning_rate": 2.0349315068493154e-05, + "loss": 0.36882529258728025, + "mean_token_accuracy": 0.9018866419792175, + "num_tokens": 8437932.0, + "step": 4380 + }, + { + "entropy": 0.32640354707837105, + "epoch": 0.5972789115646259, + "grad_norm": 0.7729905843734741, + "learning_rate": 2.028082191780822e-05, + "loss": 0.32541344165802, + "mean_token_accuracy": 0.9163854837417602, + "num_tokens": 8456033.0, + "step": 4390 + }, + { + "entropy": 0.31475903652608395, + "epoch": 0.5986394557823129, + "grad_norm": 0.49552851915359497, + "learning_rate": 2.021232876712329e-05, + "loss": 0.30246944427490235, + "mean_token_accuracy": 0.918095150589943, + "num_tokens": 8474315.0, + "step": 4400 + }, + { + "entropy": 0.3114713206887245, + "epoch": 0.6, + "grad_norm": 0.7524731755256653, + "learning_rate": 2.014383561643836e-05, + "loss": 0.306212329864502, + "mean_token_accuracy": 0.9181839942932128, + "num_tokens": 8493812.0, + "step": 4410 + }, + { + "entropy": 0.3414305582642555, + "epoch": 0.601360544217687, + "grad_norm": 0.4476776421070099, + "learning_rate": 2.0075342465753425e-05, + "loss": 0.3371569156646729, + "mean_token_accuracy": 0.9159269124269486, + "num_tokens": 8512284.0, + "step": 4420 + }, + { + "entropy": 0.2988259807229042, + "epoch": 0.6027210884353742, + "grad_norm": 0.5182522535324097, + "learning_rate": 2.0006849315068494e-05, + "loss": 0.31771996021270754, + "mean_token_accuracy": 0.9163232058286667, + "num_tokens": 8531033.0, + "step": 4430 + }, + { + "entropy": 0.3422327384352684, + "epoch": 0.6040816326530613, + "grad_norm": 0.4643579125404358, + "learning_rate": 1.993835616438356e-05, + "loss": 0.3435725212097168, + "mean_token_accuracy": 0.9077637434005738, + "num_tokens": 8550689.0, + "step": 4440 + }, + { + "entropy": 0.3316516526043415, + "epoch": 0.6054421768707483, + "grad_norm": 0.5027662515640259, + "learning_rate": 1.986986301369863e-05, + "loss": 0.3284109592437744, + "mean_token_accuracy": 0.9112151861190796, + "num_tokens": 8571488.0, + "step": 4450 + }, + { + "entropy": 0.368307001888752, + "epoch": 0.6068027210884354, + "grad_norm": 0.5623902678489685, + "learning_rate": 1.98013698630137e-05, + "loss": 0.36309108734130857, + "mean_token_accuracy": 0.9045984745025635, + "num_tokens": 8591756.0, + "step": 4460 + }, + { + "entropy": 0.31291708797216417, + "epoch": 0.6081632653061224, + "grad_norm": 0.4109684228897095, + "learning_rate": 1.973287671232877e-05, + "loss": 0.32405858039855956, + "mean_token_accuracy": 0.9153440564870834, + "num_tokens": 8610911.0, + "step": 4470 + }, + { + "entropy": 0.3248316764831543, + "epoch": 0.6095238095238096, + "grad_norm": 0.3668299913406372, + "learning_rate": 1.966438356164384e-05, + "loss": 0.3182776212692261, + "mean_token_accuracy": 0.9126608431339264, + "num_tokens": 8629955.0, + "step": 4480 + }, + { + "entropy": 0.3336681894958019, + "epoch": 0.6108843537414966, + "grad_norm": 0.5103767514228821, + "learning_rate": 1.9595890410958904e-05, + "loss": 0.3319364547729492, + "mean_token_accuracy": 0.9127811759710311, + "num_tokens": 8647894.0, + "step": 4490 + }, + { + "entropy": 0.31860669925808904, + "epoch": 0.6122448979591837, + "grad_norm": 0.4623521566390991, + "learning_rate": 1.9527397260273974e-05, + "loss": 0.3230354070663452, + "mean_token_accuracy": 0.9166040539741516, + "num_tokens": 8667198.0, + "step": 4500 + }, + { + "epoch": 0.6122448979591837, + "eval_entropy": 0.3310739654302597, + "eval_loss": 0.3289305567741394, + "eval_mean_token_accuracy": 0.9130673670768737, + "eval_num_tokens": 8667198.0, + "eval_runtime": 38.7549, + "eval_samples_per_second": 15.482, + "eval_steps_per_second": 7.741, + "step": 4500 + }, + { + "entropy": 0.3124507375061512, + "epoch": 0.6136054421768707, + "grad_norm": 0.385375052690506, + "learning_rate": 1.945890410958904e-05, + "loss": 0.3219736337661743, + "mean_token_accuracy": 0.9118276238441467, + "num_tokens": 8690256.0, + "step": 4510 + }, + { + "entropy": 0.3162266179919243, + "epoch": 0.6149659863945578, + "grad_norm": 0.3542030155658722, + "learning_rate": 1.939041095890411e-05, + "loss": 0.3027017116546631, + "mean_token_accuracy": 0.9203230768442154, + "num_tokens": 8708888.0, + "step": 4520 + }, + { + "entropy": 0.3298695728182793, + "epoch": 0.6163265306122448, + "grad_norm": 0.47942066192626953, + "learning_rate": 1.932191780821918e-05, + "loss": 0.33888137340545654, + "mean_token_accuracy": 0.9136987268924713, + "num_tokens": 8727333.0, + "step": 4530 + }, + { + "entropy": 0.2821711003780365, + "epoch": 0.617687074829932, + "grad_norm": 0.48402971029281616, + "learning_rate": 1.925342465753425e-05, + "loss": 0.2859959125518799, + "mean_token_accuracy": 0.9232243895530701, + "num_tokens": 8747157.0, + "step": 4540 + }, + { + "entropy": 0.2933730438351631, + "epoch": 0.6190476190476191, + "grad_norm": 0.5369724035263062, + "learning_rate": 1.9184931506849314e-05, + "loss": 0.28847010135650636, + "mean_token_accuracy": 0.9202695101499557, + "num_tokens": 8765416.0, + "step": 4550 + }, + { + "entropy": 0.30423177406191826, + "epoch": 0.6204081632653061, + "grad_norm": 0.3822837173938751, + "learning_rate": 1.9116438356164384e-05, + "loss": 0.3026056528091431, + "mean_token_accuracy": 0.9191194862127304, + "num_tokens": 8782981.0, + "step": 4560 + }, + { + "entropy": 0.35051786080002784, + "epoch": 0.6217687074829932, + "grad_norm": 0.403596431016922, + "learning_rate": 1.9047945205479453e-05, + "loss": 0.34621405601501465, + "mean_token_accuracy": 0.911699402332306, + "num_tokens": 8802808.0, + "step": 4570 + }, + { + "entropy": 0.30863510593771937, + "epoch": 0.6231292517006802, + "grad_norm": 0.38001856207847595, + "learning_rate": 1.8979452054794523e-05, + "loss": 0.3103233575820923, + "mean_token_accuracy": 0.9180112898349762, + "num_tokens": 8822863.0, + "step": 4580 + }, + { + "entropy": 0.2945192024111748, + "epoch": 0.6244897959183674, + "grad_norm": 0.5920354127883911, + "learning_rate": 1.8910958904109592e-05, + "loss": 0.29104783535003664, + "mean_token_accuracy": 0.9209440588951111, + "num_tokens": 8843360.0, + "step": 4590 + }, + { + "entropy": 0.3656765803694725, + "epoch": 0.6258503401360545, + "grad_norm": 0.4458155035972595, + "learning_rate": 1.884246575342466e-05, + "loss": 0.37093372344970704, + "mean_token_accuracy": 0.904807910323143, + "num_tokens": 8863752.0, + "step": 4600 + }, + { + "entropy": 0.3120325908064842, + "epoch": 0.6272108843537415, + "grad_norm": 0.45716285705566406, + "learning_rate": 1.8773972602739725e-05, + "loss": 0.3167175054550171, + "mean_token_accuracy": 0.9172279864549637, + "num_tokens": 8883642.0, + "step": 4610 + }, + { + "entropy": 0.32725323736667633, + "epoch": 0.6285714285714286, + "grad_norm": 0.5018922686576843, + "learning_rate": 1.8705479452054794e-05, + "loss": 0.3386115074157715, + "mean_token_accuracy": 0.9148703008890152, + "num_tokens": 8903406.0, + "step": 4620 + }, + { + "entropy": 0.3152094043791294, + "epoch": 0.6299319727891156, + "grad_norm": 0.49038684368133545, + "learning_rate": 1.8636986301369863e-05, + "loss": 0.31165380477905275, + "mean_token_accuracy": 0.9160630106925964, + "num_tokens": 8922781.0, + "step": 4630 + }, + { + "entropy": 0.2654181182384491, + "epoch": 0.6312925170068027, + "grad_norm": 0.499210000038147, + "learning_rate": 1.8568493150684933e-05, + "loss": 0.25337116718292235, + "mean_token_accuracy": 0.9268150001764297, + "num_tokens": 8940811.0, + "step": 4640 + }, + { + "entropy": 0.33546371310949324, + "epoch": 0.6326530612244898, + "grad_norm": 0.5591289401054382, + "learning_rate": 1.85e-05, + "loss": 0.3524590015411377, + "mean_token_accuracy": 0.9123670727014541, + "num_tokens": 8959421.0, + "step": 4650 + }, + { + "entropy": 0.30375867560505865, + "epoch": 0.6340136054421769, + "grad_norm": 0.657088041305542, + "learning_rate": 1.843150684931507e-05, + "loss": 0.2986541509628296, + "mean_token_accuracy": 0.9188253223896027, + "num_tokens": 8978608.0, + "step": 4660 + }, + { + "entropy": 0.29843793213367464, + "epoch": 0.6353741496598639, + "grad_norm": 0.5112680196762085, + "learning_rate": 1.8363013698630138e-05, + "loss": 0.2931118726730347, + "mean_token_accuracy": 0.9223059266805649, + "num_tokens": 8996614.0, + "step": 4670 + }, + { + "entropy": 0.33378486707806587, + "epoch": 0.636734693877551, + "grad_norm": 0.437621533870697, + "learning_rate": 1.8294520547945207e-05, + "loss": 0.32436633110046387, + "mean_token_accuracy": 0.9149109095335006, + "num_tokens": 9017250.0, + "step": 4680 + }, + { + "entropy": 0.2954431213438511, + "epoch": 0.638095238095238, + "grad_norm": 0.626248300075531, + "learning_rate": 1.8226027397260277e-05, + "loss": 0.3086385250091553, + "mean_token_accuracy": 0.9184481203556061, + "num_tokens": 9038262.0, + "step": 4690 + }, + { + "entropy": 0.29330349788069726, + "epoch": 0.6394557823129252, + "grad_norm": 0.5271269679069519, + "learning_rate": 1.8157534246575343e-05, + "loss": 0.2987428426742554, + "mean_token_accuracy": 0.9220951855182647, + "num_tokens": 9059039.0, + "step": 4700 + }, + { + "entropy": 0.3254406459629536, + "epoch": 0.6408163265306123, + "grad_norm": 0.38454508781433105, + "learning_rate": 1.808904109589041e-05, + "loss": 0.3387522459030151, + "mean_token_accuracy": 0.9124308109283448, + "num_tokens": 9077519.0, + "step": 4710 + }, + { + "entropy": 0.3163925364613533, + "epoch": 0.6421768707482993, + "grad_norm": 0.5404890775680542, + "learning_rate": 1.802054794520548e-05, + "loss": 0.3156402587890625, + "mean_token_accuracy": 0.914623373746872, + "num_tokens": 9097280.0, + "step": 4720 + }, + { + "entropy": 0.3105604760348797, + "epoch": 0.6435374149659864, + "grad_norm": 0.38867318630218506, + "learning_rate": 1.7952054794520548e-05, + "loss": 0.3073798656463623, + "mean_token_accuracy": 0.9193964332342148, + "num_tokens": 9116868.0, + "step": 4730 + }, + { + "entropy": 0.34601864591240883, + "epoch": 0.6448979591836734, + "grad_norm": 0.33544185757637024, + "learning_rate": 1.7883561643835618e-05, + "loss": 0.3442887783050537, + "mean_token_accuracy": 0.9118550598621369, + "num_tokens": 9135871.0, + "step": 4740 + }, + { + "entropy": 0.3289314046502113, + "epoch": 0.6462585034013606, + "grad_norm": 0.7844065427780151, + "learning_rate": 1.7815068493150687e-05, + "loss": 0.3229420900344849, + "mean_token_accuracy": 0.9158719092607498, + "num_tokens": 9155805.0, + "step": 4750 + }, + { + "epoch": 0.6462585034013606, + "eval_entropy": 0.33255102773507433, + "eval_loss": 0.32763543725013733, + "eval_mean_token_accuracy": 0.9132404702901841, + "eval_num_tokens": 9155805.0, + "eval_runtime": 39.9686, + "eval_samples_per_second": 15.012, + "eval_steps_per_second": 7.506, + "step": 4750 + }, + { + "entropy": 0.32009952403604985, + "epoch": 0.6476190476190476, + "grad_norm": 0.4704689085483551, + "learning_rate": 1.7746575342465753e-05, + "loss": 0.31637964248657224, + "mean_token_accuracy": 0.9144832044839859, + "num_tokens": 9177273.0, + "step": 4760 + }, + { + "entropy": 0.28996511846780776, + "epoch": 0.6489795918367347, + "grad_norm": 0.41101402044296265, + "learning_rate": 1.7678082191780823e-05, + "loss": 0.285528564453125, + "mean_token_accuracy": 0.92576684653759, + "num_tokens": 9197118.0, + "step": 4770 + }, + { + "entropy": 0.3418573744595051, + "epoch": 0.6503401360544218, + "grad_norm": 0.5408941507339478, + "learning_rate": 1.7609589041095892e-05, + "loss": 0.3393819808959961, + "mean_token_accuracy": 0.9118351727724076, + "num_tokens": 9216516.0, + "step": 4780 + }, + { + "entropy": 0.28226210810244085, + "epoch": 0.6517006802721088, + "grad_norm": 0.3848341703414917, + "learning_rate": 1.754109589041096e-05, + "loss": 0.2931527614593506, + "mean_token_accuracy": 0.9224736660718917, + "num_tokens": 9234639.0, + "step": 4790 + }, + { + "entropy": 0.2693475380539894, + "epoch": 0.6530612244897959, + "grad_norm": 0.6227667331695557, + "learning_rate": 1.747260273972603e-05, + "loss": 0.2764613389968872, + "mean_token_accuracy": 0.9238964438438415, + "num_tokens": 9254926.0, + "step": 4800 + }, + { + "entropy": 0.328694748878479, + "epoch": 0.654421768707483, + "grad_norm": 0.4423985183238983, + "learning_rate": 1.7404109589041097e-05, + "loss": 0.3174917459487915, + "mean_token_accuracy": 0.9097775369882584, + "num_tokens": 9276122.0, + "step": 4810 + }, + { + "entropy": 0.37669333517551423, + "epoch": 0.6557823129251701, + "grad_norm": 0.5443205237388611, + "learning_rate": 1.7335616438356163e-05, + "loss": 0.3813244581222534, + "mean_token_accuracy": 0.9008943438529968, + "num_tokens": 9295112.0, + "step": 4820 + }, + { + "entropy": 0.3570864848792553, + "epoch": 0.6571428571428571, + "grad_norm": 0.539834201335907, + "learning_rate": 1.7267123287671233e-05, + "loss": 0.36410791873931886, + "mean_token_accuracy": 0.9081277132034302, + "num_tokens": 9314161.0, + "step": 4830 + }, + { + "entropy": 0.31993206292390824, + "epoch": 0.6585034013605442, + "grad_norm": 0.3892541825771332, + "learning_rate": 1.7198630136986302e-05, + "loss": 0.29779579639434817, + "mean_token_accuracy": 0.9220435619354248, + "num_tokens": 9332900.0, + "step": 4840 + }, + { + "entropy": 0.2923553496599197, + "epoch": 0.6598639455782312, + "grad_norm": 0.4788413643836975, + "learning_rate": 1.713013698630137e-05, + "loss": 0.29410948753356936, + "mean_token_accuracy": 0.9204231530427933, + "num_tokens": 9351463.0, + "step": 4850 + }, + { + "entropy": 0.28082970082759856, + "epoch": 0.6612244897959184, + "grad_norm": 0.4797994792461395, + "learning_rate": 1.7061643835616438e-05, + "loss": 0.291374945640564, + "mean_token_accuracy": 0.9210595309734344, + "num_tokens": 9370433.0, + "step": 4860 + }, + { + "entropy": 0.3138731438666582, + "epoch": 0.6625850340136055, + "grad_norm": 0.4365018606185913, + "learning_rate": 1.6993150684931507e-05, + "loss": 0.32789273262023927, + "mean_token_accuracy": 0.9142170727252961, + "num_tokens": 9390262.0, + "step": 4870 + }, + { + "entropy": 0.29280120581388475, + "epoch": 0.6639455782312925, + "grad_norm": 0.5428627729415894, + "learning_rate": 1.6924657534246577e-05, + "loss": 0.27776288986206055, + "mean_token_accuracy": 0.9247405260801316, + "num_tokens": 9408584.0, + "step": 4880 + }, + { + "entropy": 0.32643493488430975, + "epoch": 0.6653061224489796, + "grad_norm": 0.5670334100723267, + "learning_rate": 1.6856164383561646e-05, + "loss": 0.3300167560577393, + "mean_token_accuracy": 0.9129970610141754, + "num_tokens": 9427236.0, + "step": 4890 + }, + { + "entropy": 0.3827863521873951, + "epoch": 0.6666666666666666, + "grad_norm": 0.41381803154945374, + "learning_rate": 1.6787671232876716e-05, + "loss": 0.40310182571411135, + "mean_token_accuracy": 0.899763336777687, + "num_tokens": 9444771.0, + "step": 4900 + }, + { + "entropy": 0.32538103610277175, + "epoch": 0.6680272108843538, + "grad_norm": 0.43814322352409363, + "learning_rate": 1.671917808219178e-05, + "loss": 0.33401734828948976, + "mean_token_accuracy": 0.9110633939504623, + "num_tokens": 9463864.0, + "step": 4910 + }, + { + "entropy": 0.24819178245961665, + "epoch": 0.6693877551020408, + "grad_norm": 0.484448105096817, + "learning_rate": 1.6650684931506848e-05, + "loss": 0.23955152034759522, + "mean_token_accuracy": 0.933735242486, + "num_tokens": 9481457.0, + "step": 4920 + }, + { + "entropy": 0.33870874494314196, + "epoch": 0.6707482993197279, + "grad_norm": 0.7475461363792419, + "learning_rate": 1.6582191780821917e-05, + "loss": 0.3345521926879883, + "mean_token_accuracy": 0.9089796751737594, + "num_tokens": 9499700.0, + "step": 4930 + }, + { + "entropy": 0.28814957924187184, + "epoch": 0.672108843537415, + "grad_norm": 0.6497180461883545, + "learning_rate": 1.6513698630136987e-05, + "loss": 0.29244041442871094, + "mean_token_accuracy": 0.9212163001298904, + "num_tokens": 9518473.0, + "step": 4940 + }, + { + "entropy": 0.3113638870418072, + "epoch": 0.673469387755102, + "grad_norm": 0.6817547082901001, + "learning_rate": 1.6445205479452056e-05, + "loss": 0.3035978555679321, + "mean_token_accuracy": 0.9179104954004288, + "num_tokens": 9537747.0, + "step": 4950 + }, + { + "entropy": 0.3260213859379292, + "epoch": 0.6748299319727891, + "grad_norm": 0.5817997455596924, + "learning_rate": 1.6376712328767126e-05, + "loss": 0.3352161407470703, + "mean_token_accuracy": 0.9155428946018219, + "num_tokens": 9556544.0, + "step": 4960 + }, + { + "entropy": 0.32320692837238313, + "epoch": 0.6761904761904762, + "grad_norm": 0.40952223539352417, + "learning_rate": 1.6308219178082192e-05, + "loss": 0.3208148717880249, + "mean_token_accuracy": 0.9158533960580826, + "num_tokens": 9574599.0, + "step": 4970 + }, + { + "entropy": 0.32987525388598443, + "epoch": 0.6775510204081633, + "grad_norm": 0.38160601258277893, + "learning_rate": 1.623972602739726e-05, + "loss": 0.3314782381057739, + "mean_token_accuracy": 0.9137853533029556, + "num_tokens": 9594134.0, + "step": 4980 + }, + { + "entropy": 0.3361847847700119, + "epoch": 0.6789115646258503, + "grad_norm": 0.5430313348770142, + "learning_rate": 1.617123287671233e-05, + "loss": 0.3365447998046875, + "mean_token_accuracy": 0.9137306600809098, + "num_tokens": 9612548.0, + "step": 4990 + }, + { + "entropy": 0.30371019840240476, + "epoch": 0.6802721088435374, + "grad_norm": 0.5549236536026001, + "learning_rate": 1.61027397260274e-05, + "loss": 0.30310888290405275, + "mean_token_accuracy": 0.9160631775856019, + "num_tokens": 9631640.0, + "step": 5000 + }, + { + "epoch": 0.6802721088435374, + "eval_entropy": 0.3331662664314111, + "eval_loss": 0.32677820324897766, + "eval_mean_token_accuracy": 0.9136480834086737, + "eval_num_tokens": 9631640.0, + "eval_runtime": 39.708, + "eval_samples_per_second": 15.11, + "eval_steps_per_second": 7.555, + "step": 5000 + }, + { + "entropy": 0.3210461221635342, + "epoch": 0.6816326530612244, + "grad_norm": 0.6082227230072021, + "learning_rate": 1.6034246575342466e-05, + "loss": 0.3225013256072998, + "mean_token_accuracy": 0.9164930671453476, + "num_tokens": 9650167.0, + "step": 5010 + }, + { + "entropy": 0.33073775470256805, + "epoch": 0.6829931972789116, + "grad_norm": 0.4650852084159851, + "learning_rate": 1.5965753424657536e-05, + "loss": 0.32491614818573, + "mean_token_accuracy": 0.9105731517076492, + "num_tokens": 9669888.0, + "step": 5020 + }, + { + "entropy": 0.2889856085181236, + "epoch": 0.6843537414965987, + "grad_norm": 0.4409579038619995, + "learning_rate": 1.5897260273972602e-05, + "loss": 0.2879915237426758, + "mean_token_accuracy": 0.9247685641050338, + "num_tokens": 9686802.0, + "step": 5030 + }, + { + "entropy": 0.30220348536968233, + "epoch": 0.6857142857142857, + "grad_norm": 0.4094906747341156, + "learning_rate": 1.582876712328767e-05, + "loss": 0.2927564144134521, + "mean_token_accuracy": 0.921038344502449, + "num_tokens": 9705587.0, + "step": 5040 + }, + { + "entropy": 0.30619596429169177, + "epoch": 0.6870748299319728, + "grad_norm": 0.44534412026405334, + "learning_rate": 1.576027397260274e-05, + "loss": 0.30527758598327637, + "mean_token_accuracy": 0.9183918595314026, + "num_tokens": 9723365.0, + "step": 5050 + }, + { + "entropy": 0.27247815728187563, + "epoch": 0.6884353741496598, + "grad_norm": 0.38370785117149353, + "learning_rate": 1.569178082191781e-05, + "loss": 0.28004071712493894, + "mean_token_accuracy": 0.9289819926023484, + "num_tokens": 9741522.0, + "step": 5060 + }, + { + "entropy": 0.3562318213284016, + "epoch": 0.689795918367347, + "grad_norm": 0.42891645431518555, + "learning_rate": 1.5623287671232876e-05, + "loss": 0.3679760456085205, + "mean_token_accuracy": 0.906402376294136, + "num_tokens": 9761886.0, + "step": 5070 + }, + { + "entropy": 0.30469153448939323, + "epoch": 0.691156462585034, + "grad_norm": 0.4258774518966675, + "learning_rate": 1.5554794520547946e-05, + "loss": 0.2953490734100342, + "mean_token_accuracy": 0.9204110503196716, + "num_tokens": 9782113.0, + "step": 5080 + }, + { + "entropy": 0.3751088730990887, + "epoch": 0.6925170068027211, + "grad_norm": 0.43709659576416016, + "learning_rate": 1.5486301369863015e-05, + "loss": 0.3805356740951538, + "mean_token_accuracy": 0.9013684421777726, + "num_tokens": 9801414.0, + "step": 5090 + }, + { + "entropy": 0.3188751734793186, + "epoch": 0.6938775510204082, + "grad_norm": 0.4579872488975525, + "learning_rate": 1.5417808219178085e-05, + "loss": 0.3172019958496094, + "mean_token_accuracy": 0.9144055992364883, + "num_tokens": 9821329.0, + "step": 5100 + }, + { + "entropy": 0.3659577779471874, + "epoch": 0.6952380952380952, + "grad_norm": 0.5475864410400391, + "learning_rate": 1.534931506849315e-05, + "loss": 0.35874128341674805, + "mean_token_accuracy": 0.9058806359767914, + "num_tokens": 9840983.0, + "step": 5110 + }, + { + "entropy": 0.3272984899580479, + "epoch": 0.6965986394557823, + "grad_norm": 0.5433106422424316, + "learning_rate": 1.528082191780822e-05, + "loss": 0.320189380645752, + "mean_token_accuracy": 0.9151940882205963, + "num_tokens": 9861435.0, + "step": 5120 + }, + { + "entropy": 0.3580063499510288, + "epoch": 0.6979591836734694, + "grad_norm": 0.37401801347732544, + "learning_rate": 1.5212328767123288e-05, + "loss": 0.37555441856384275, + "mean_token_accuracy": 0.904027909040451, + "num_tokens": 9881061.0, + "step": 5130 + }, + { + "entropy": 0.3050012022256851, + "epoch": 0.6993197278911565, + "grad_norm": 0.5558372139930725, + "learning_rate": 1.5143835616438356e-05, + "loss": 0.3010509252548218, + "mean_token_accuracy": 0.9206721186637878, + "num_tokens": 9898420.0, + "step": 5140 + }, + { + "entropy": 0.3440397623926401, + "epoch": 0.7006802721088435, + "grad_norm": 0.5558260083198547, + "learning_rate": 1.5075342465753425e-05, + "loss": 0.32512967586517333, + "mean_token_accuracy": 0.9161503165960312, + "num_tokens": 9916386.0, + "step": 5150 + }, + { + "entropy": 0.32052273452281954, + "epoch": 0.7020408163265306, + "grad_norm": 0.42511069774627686, + "learning_rate": 1.5006849315068495e-05, + "loss": 0.31507837772369385, + "mean_token_accuracy": 0.9184840559959412, + "num_tokens": 9935862.0, + "step": 5160 + }, + { + "entropy": 0.3130710139870644, + "epoch": 0.7034013605442176, + "grad_norm": 0.48433229327201843, + "learning_rate": 1.4938356164383563e-05, + "loss": 0.3007185459136963, + "mean_token_accuracy": 0.9166130065917969, + "num_tokens": 9956146.0, + "step": 5170 + }, + { + "entropy": 0.34180518612265587, + "epoch": 0.7047619047619048, + "grad_norm": 0.5500788688659668, + "learning_rate": 1.4869863013698632e-05, + "loss": 0.3480189323425293, + "mean_token_accuracy": 0.9084232956171036, + "num_tokens": 9975283.0, + "step": 5180 + }, + { + "entropy": 0.26845929883420466, + "epoch": 0.7061224489795919, + "grad_norm": 0.4044052064418793, + "learning_rate": 1.48013698630137e-05, + "loss": 0.2679526090621948, + "mean_token_accuracy": 0.9261024564504623, + "num_tokens": 9993710.0, + "step": 5190 + }, + { + "entropy": 0.3100937880575657, + "epoch": 0.7074829931972789, + "grad_norm": 0.3828263282775879, + "learning_rate": 1.473287671232877e-05, + "loss": 0.3154742956161499, + "mean_token_accuracy": 0.9174116760492325, + "num_tokens": 10013864.0, + "step": 5200 + }, + { + "entropy": 0.3618170410394669, + "epoch": 0.708843537414966, + "grad_norm": 0.40374597907066345, + "learning_rate": 1.4664383561643835e-05, + "loss": 0.36072003841400146, + "mean_token_accuracy": 0.9016565412282944, + "num_tokens": 10034656.0, + "step": 5210 + }, + { + "entropy": 0.3923501305282116, + "epoch": 0.710204081632653, + "grad_norm": 0.5628812313079834, + "learning_rate": 1.4595890410958903e-05, + "loss": 0.4183984279632568, + "mean_token_accuracy": 0.900890639424324, + "num_tokens": 10053481.0, + "step": 5220 + }, + { + "entropy": 0.3111284948885441, + "epoch": 0.7115646258503401, + "grad_norm": 0.5592334270477295, + "learning_rate": 1.4527397260273973e-05, + "loss": 0.3107800006866455, + "mean_token_accuracy": 0.9183153718709945, + "num_tokens": 10070985.0, + "step": 5230 + }, + { + "entropy": 0.264003212377429, + "epoch": 0.7129251700680272, + "grad_norm": 0.3325921595096588, + "learning_rate": 1.4458904109589042e-05, + "loss": 0.2693720817565918, + "mean_token_accuracy": 0.9262395352125168, + "num_tokens": 10092029.0, + "step": 5240 + }, + { + "entropy": 0.30503557771444323, + "epoch": 0.7142857142857143, + "grad_norm": 0.6577172875404358, + "learning_rate": 1.439041095890411e-05, + "loss": 0.31351213455200194, + "mean_token_accuracy": 0.9185102015733719, + "num_tokens": 10111599.0, + "step": 5250 + }, + { + "epoch": 0.7142857142857143, + "eval_entropy": 0.3265386664867401, + "eval_loss": 0.32589972019195557, + "eval_mean_token_accuracy": 0.913812648455302, + "eval_num_tokens": 10111599.0, + "eval_runtime": 40.0077, + "eval_samples_per_second": 14.997, + "eval_steps_per_second": 7.499, + "step": 5250 + }, + { + "entropy": 0.32656988948583604, + "epoch": 0.7156462585034014, + "grad_norm": 0.45740580558776855, + "learning_rate": 1.432191780821918e-05, + "loss": 0.3337229251861572, + "mean_token_accuracy": 0.913157707452774, + "num_tokens": 10130030.0, + "step": 5260 + }, + { + "entropy": 0.332012340426445, + "epoch": 0.7170068027210884, + "grad_norm": 0.4293174147605896, + "learning_rate": 1.4253424657534247e-05, + "loss": 0.33874466419219973, + "mean_token_accuracy": 0.9121835470199585, + "num_tokens": 10150708.0, + "step": 5270 + }, + { + "entropy": 0.3409136638045311, + "epoch": 0.7183673469387755, + "grad_norm": 0.6065399646759033, + "learning_rate": 1.4184931506849317e-05, + "loss": 0.31609444618225097, + "mean_token_accuracy": 0.9152237951755524, + "num_tokens": 10170436.0, + "step": 5280 + }, + { + "entropy": 0.3278017446398735, + "epoch": 0.7197278911564626, + "grad_norm": 0.5555611848831177, + "learning_rate": 1.4116438356164386e-05, + "loss": 0.3270564556121826, + "mean_token_accuracy": 0.9139165192842483, + "num_tokens": 10188702.0, + "step": 5290 + }, + { + "entropy": 0.29700450971722603, + "epoch": 0.7210884353741497, + "grad_norm": 0.5627049207687378, + "learning_rate": 1.4047945205479452e-05, + "loss": 0.29241275787353516, + "mean_token_accuracy": 0.9209558874368667, + "num_tokens": 10208040.0, + "step": 5300 + }, + { + "entropy": 0.30556924641132355, + "epoch": 0.7224489795918367, + "grad_norm": 0.4673140048980713, + "learning_rate": 1.397945205479452e-05, + "loss": 0.30704848766326903, + "mean_token_accuracy": 0.9185676604509354, + "num_tokens": 10225913.0, + "step": 5310 + }, + { + "entropy": 0.3303392879664898, + "epoch": 0.7238095238095238, + "grad_norm": 0.4551662504673004, + "learning_rate": 1.391095890410959e-05, + "loss": 0.34713623523712156, + "mean_token_accuracy": 0.9099254727363586, + "num_tokens": 10243438.0, + "step": 5320 + }, + { + "entropy": 0.3365324631333351, + "epoch": 0.7251700680272108, + "grad_norm": 0.4157435894012451, + "learning_rate": 1.3842465753424657e-05, + "loss": 0.3378427505493164, + "mean_token_accuracy": 0.909062597155571, + "num_tokens": 10262805.0, + "step": 5330 + }, + { + "entropy": 0.32411291301250456, + "epoch": 0.726530612244898, + "grad_norm": 0.38107967376708984, + "learning_rate": 1.3773972602739727e-05, + "loss": 0.3202343225479126, + "mean_token_accuracy": 0.9115140378475189, + "num_tokens": 10282508.0, + "step": 5340 + }, + { + "entropy": 0.3038345813751221, + "epoch": 0.7278911564625851, + "grad_norm": 0.4391093850135803, + "learning_rate": 1.3705479452054794e-05, + "loss": 0.29564452171325684, + "mean_token_accuracy": 0.9200590491294861, + "num_tokens": 10303702.0, + "step": 5350 + }, + { + "entropy": 0.30474315136671065, + "epoch": 0.7292517006802721, + "grad_norm": 0.3891599178314209, + "learning_rate": 1.3636986301369864e-05, + "loss": 0.3134526491165161, + "mean_token_accuracy": 0.920561146736145, + "num_tokens": 10321825.0, + "step": 5360 + }, + { + "entropy": 0.3209965944290161, + "epoch": 0.7306122448979592, + "grad_norm": 0.6909648180007935, + "learning_rate": 1.3568493150684933e-05, + "loss": 0.33262181282043457, + "mean_token_accuracy": 0.9147113114595413, + "num_tokens": 10343513.0, + "step": 5370 + }, + { + "entropy": 0.2940310426056385, + "epoch": 0.7319727891156462, + "grad_norm": 0.3770340085029602, + "learning_rate": 1.3500000000000001e-05, + "loss": 0.2746826887130737, + "mean_token_accuracy": 0.9232329875230789, + "num_tokens": 10363756.0, + "step": 5380 + }, + { + "entropy": 0.3722431115806103, + "epoch": 0.7333333333333333, + "grad_norm": 0.5671103596687317, + "learning_rate": 1.343150684931507e-05, + "loss": 0.3857078790664673, + "mean_token_accuracy": 0.9042104989290237, + "num_tokens": 10382027.0, + "step": 5390 + }, + { + "entropy": 0.3080087319016457, + "epoch": 0.7346938775510204, + "grad_norm": 0.4369518458843231, + "learning_rate": 1.3363013698630137e-05, + "loss": 0.3244903802871704, + "mean_token_accuracy": 0.9169433772563934, + "num_tokens": 10401036.0, + "step": 5400 + }, + { + "entropy": 0.30762851536273955, + "epoch": 0.7360544217687075, + "grad_norm": 0.42277780175209045, + "learning_rate": 1.3294520547945205e-05, + "loss": 0.3111214876174927, + "mean_token_accuracy": 0.920467808842659, + "num_tokens": 10420343.0, + "step": 5410 + }, + { + "entropy": 0.2915546834468842, + "epoch": 0.7374149659863946, + "grad_norm": 0.6937938928604126, + "learning_rate": 1.3226027397260274e-05, + "loss": 0.29332144260406495, + "mean_token_accuracy": 0.928145581483841, + "num_tokens": 10435914.0, + "step": 5420 + }, + { + "entropy": 0.3241829499602318, + "epoch": 0.7387755102040816, + "grad_norm": 0.28303202986717224, + "learning_rate": 1.3157534246575342e-05, + "loss": 0.3249546527862549, + "mean_token_accuracy": 0.9146370053291321, + "num_tokens": 10456518.0, + "step": 5430 + }, + { + "entropy": 0.27756612598896024, + "epoch": 0.7401360544217687, + "grad_norm": 0.4878402054309845, + "learning_rate": 1.3089041095890411e-05, + "loss": 0.281647253036499, + "mean_token_accuracy": 0.9224755674600601, + "num_tokens": 10477480.0, + "step": 5440 + }, + { + "entropy": 0.31852116584777834, + "epoch": 0.7414965986394558, + "grad_norm": 0.5137626528739929, + "learning_rate": 1.302054794520548e-05, + "loss": 0.31983242034912107, + "mean_token_accuracy": 0.9154018461704254, + "num_tokens": 10496416.0, + "step": 5450 + }, + { + "entropy": 0.3167009152472019, + "epoch": 0.7428571428571429, + "grad_norm": 0.6255688667297363, + "learning_rate": 1.2952054794520549e-05, + "loss": 0.3200571060180664, + "mean_token_accuracy": 0.9154588907957077, + "num_tokens": 10515961.0, + "step": 5460 + }, + { + "entropy": 0.375573131069541, + "epoch": 0.7442176870748299, + "grad_norm": 0.42178791761398315, + "learning_rate": 1.2883561643835618e-05, + "loss": 0.40090117454528806, + "mean_token_accuracy": 0.8996903270483017, + "num_tokens": 10534924.0, + "step": 5470 + }, + { + "entropy": 0.304948503151536, + "epoch": 0.745578231292517, + "grad_norm": 0.3416388928890228, + "learning_rate": 1.2815068493150686e-05, + "loss": 0.3036719799041748, + "mean_token_accuracy": 0.9170698165893555, + "num_tokens": 10556668.0, + "step": 5480 + }, + { + "entropy": 0.3498163715004921, + "epoch": 0.746938775510204, + "grad_norm": 0.4322749376296997, + "learning_rate": 1.2746575342465755e-05, + "loss": 0.35872642993927, + "mean_token_accuracy": 0.9104368448257446, + "num_tokens": 10574438.0, + "step": 5490 + }, + { + "entropy": 0.2817006319761276, + "epoch": 0.7482993197278912, + "grad_norm": 0.3572882413864136, + "learning_rate": 1.2678082191780821e-05, + "loss": 0.2768646001815796, + "mean_token_accuracy": 0.9242710262537003, + "num_tokens": 10595536.0, + "step": 5500 + }, + { + "epoch": 0.7482993197278912, + "eval_entropy": 0.3340467913697163, + "eval_loss": 0.3254561424255371, + "eval_mean_token_accuracy": 0.9137606120109558, + "eval_num_tokens": 10595536.0, + "eval_runtime": 39.9117, + "eval_samples_per_second": 15.033, + "eval_steps_per_second": 7.517, + "step": 5500 + }, + { + "entropy": 0.3420073136687279, + "epoch": 0.7496598639455783, + "grad_norm": 0.6343442797660828, + "learning_rate": 1.260958904109589e-05, + "loss": 0.3322641611099243, + "mean_token_accuracy": 0.9081849426031112, + "num_tokens": 10616452.0, + "step": 5510 + }, + { + "entropy": 0.3741004906594753, + "epoch": 0.7510204081632653, + "grad_norm": 0.3708195388317108, + "learning_rate": 1.2541095890410959e-05, + "loss": 0.370376443862915, + "mean_token_accuracy": 0.9020725101232528, + "num_tokens": 10636869.0, + "step": 5520 + }, + { + "entropy": 0.36120699867606165, + "epoch": 0.7523809523809524, + "grad_norm": 0.3880722224712372, + "learning_rate": 1.2472602739726028e-05, + "loss": 0.3640472173690796, + "mean_token_accuracy": 0.9033250749111176, + "num_tokens": 10659405.0, + "step": 5530 + }, + { + "entropy": 0.3709873460233212, + "epoch": 0.7537414965986394, + "grad_norm": 0.4630112946033478, + "learning_rate": 1.2404109589041096e-05, + "loss": 0.3700759172439575, + "mean_token_accuracy": 0.9041523218154908, + "num_tokens": 10677909.0, + "step": 5540 + }, + { + "entropy": 0.2900518998503685, + "epoch": 0.7551020408163265, + "grad_norm": 0.5871257781982422, + "learning_rate": 1.2335616438356165e-05, + "loss": 0.28741438388824464, + "mean_token_accuracy": 0.9231601834297181, + "num_tokens": 10696772.0, + "step": 5550 + }, + { + "entropy": 0.31697488874197005, + "epoch": 0.7564625850340136, + "grad_norm": 0.4777441620826721, + "learning_rate": 1.2267123287671233e-05, + "loss": 0.32103445529937746, + "mean_token_accuracy": 0.9166768312454223, + "num_tokens": 10716784.0, + "step": 5560 + }, + { + "entropy": 0.3552616074681282, + "epoch": 0.7578231292517007, + "grad_norm": 0.4223809838294983, + "learning_rate": 1.2198630136986303e-05, + "loss": 0.37296199798583984, + "mean_token_accuracy": 0.9043572753667831, + "num_tokens": 10735450.0, + "step": 5570 + }, + { + "entropy": 0.2962200723588467, + "epoch": 0.7591836734693878, + "grad_norm": 0.43479984998703003, + "learning_rate": 1.213013698630137e-05, + "loss": 0.2950768947601318, + "mean_token_accuracy": 0.91934255361557, + "num_tokens": 10755247.0, + "step": 5580 + }, + { + "entropy": 0.31132856346666815, + "epoch": 0.7605442176870748, + "grad_norm": 0.4492180645465851, + "learning_rate": 1.2061643835616438e-05, + "loss": 0.3084630012512207, + "mean_token_accuracy": 0.9199709206819534, + "num_tokens": 10774582.0, + "step": 5590 + }, + { + "entropy": 0.30692355558276174, + "epoch": 0.7619047619047619, + "grad_norm": 0.5301331281661987, + "learning_rate": 1.1993150684931508e-05, + "loss": 0.2932703971862793, + "mean_token_accuracy": 0.9213265508413315, + "num_tokens": 10793235.0, + "step": 5600 + }, + { + "entropy": 0.299712085723877, + "epoch": 0.763265306122449, + "grad_norm": 0.38069605827331543, + "learning_rate": 1.1924657534246577e-05, + "loss": 0.30311241149902346, + "mean_token_accuracy": 0.9191953390836716, + "num_tokens": 10813845.0, + "step": 5610 + }, + { + "entropy": 0.34512806236743926, + "epoch": 0.7646258503401361, + "grad_norm": 0.472634881734848, + "learning_rate": 1.1856164383561643e-05, + "loss": 0.3428913354873657, + "mean_token_accuracy": 0.9112067639827728, + "num_tokens": 10832987.0, + "step": 5620 + }, + { + "entropy": 0.3049409069120884, + "epoch": 0.7659863945578231, + "grad_norm": 0.4912843704223633, + "learning_rate": 1.1787671232876713e-05, + "loss": 0.3134711503982544, + "mean_token_accuracy": 0.9179065018892288, + "num_tokens": 10851395.0, + "step": 5630 + }, + { + "entropy": 0.31765844002366067, + "epoch": 0.7673469387755102, + "grad_norm": 0.5902607440948486, + "learning_rate": 1.1719178082191782e-05, + "loss": 0.3220794916152954, + "mean_token_accuracy": 0.9152461349964142, + "num_tokens": 10869970.0, + "step": 5640 + }, + { + "entropy": 0.32936450466513634, + "epoch": 0.7687074829931972, + "grad_norm": 0.48000672459602356, + "learning_rate": 1.165068493150685e-05, + "loss": 0.3320637226104736, + "mean_token_accuracy": 0.9145504355430603, + "num_tokens": 10888631.0, + "step": 5650 + }, + { + "entropy": 0.3303429067134857, + "epoch": 0.7700680272108843, + "grad_norm": 0.33110785484313965, + "learning_rate": 1.158219178082192e-05, + "loss": 0.32364022731781006, + "mean_token_accuracy": 0.9085778921842576, + "num_tokens": 10911541.0, + "step": 5660 + }, + { + "entropy": 0.3667861148715019, + "epoch": 0.7714285714285715, + "grad_norm": 0.41618260741233826, + "learning_rate": 1.1513698630136985e-05, + "loss": 0.36133155822753904, + "mean_token_accuracy": 0.9050101220607758, + "num_tokens": 10931770.0, + "step": 5670 + }, + { + "entropy": 0.34156960174441336, + "epoch": 0.7727891156462585, + "grad_norm": 0.4888611435890198, + "learning_rate": 1.1445205479452055e-05, + "loss": 0.35315518379211425, + "mean_token_accuracy": 0.9090872585773468, + "num_tokens": 10950198.0, + "step": 5680 + }, + { + "entropy": 0.3624446362257004, + "epoch": 0.7741496598639456, + "grad_norm": 0.6312777996063232, + "learning_rate": 1.1376712328767124e-05, + "loss": 0.3629330635070801, + "mean_token_accuracy": 0.9079570889472961, + "num_tokens": 10968770.0, + "step": 5690 + }, + { + "entropy": 0.30860179737210275, + "epoch": 0.7755102040816326, + "grad_norm": 0.6905585527420044, + "learning_rate": 1.1308219178082192e-05, + "loss": 0.31810925006866453, + "mean_token_accuracy": 0.916623106598854, + "num_tokens": 10986842.0, + "step": 5700 + }, + { + "entropy": 0.3661921538412571, + "epoch": 0.7768707482993197, + "grad_norm": 0.6159297823905945, + "learning_rate": 1.1239726027397262e-05, + "loss": 0.3769394397735596, + "mean_token_accuracy": 0.9014403194189071, + "num_tokens": 11008087.0, + "step": 5710 + }, + { + "entropy": 0.3508593410253525, + "epoch": 0.7782312925170068, + "grad_norm": 0.34525948762893677, + "learning_rate": 1.117123287671233e-05, + "loss": 0.33626294136047363, + "mean_token_accuracy": 0.9080008000135422, + "num_tokens": 11030274.0, + "step": 5720 + }, + { + "entropy": 0.36780777871608733, + "epoch": 0.7795918367346939, + "grad_norm": 0.5597181916236877, + "learning_rate": 1.1102739726027397e-05, + "loss": 0.38009593486785886, + "mean_token_accuracy": 0.9044328391551971, + "num_tokens": 11049724.0, + "step": 5730 + }, + { + "entropy": 0.29728768691420554, + "epoch": 0.780952380952381, + "grad_norm": 0.5715142488479614, + "learning_rate": 1.1034246575342467e-05, + "loss": 0.30108027458190917, + "mean_token_accuracy": 0.9220935791730881, + "num_tokens": 11066979.0, + "step": 5740 + }, + { + "entropy": 0.33334632143378257, + "epoch": 0.782312925170068, + "grad_norm": 0.464752733707428, + "learning_rate": 1.0965753424657534e-05, + "loss": 0.3300285339355469, + "mean_token_accuracy": 0.9103938549757004, + "num_tokens": 11087708.0, + "step": 5750 + }, + { + "epoch": 0.782312925170068, + "eval_entropy": 0.33076301847894984, + "eval_loss": 0.3248400092124939, + "eval_mean_token_accuracy": 0.9138080885012945, + "eval_num_tokens": 11087708.0, + "eval_runtime": 39.9646, + "eval_samples_per_second": 15.013, + "eval_steps_per_second": 7.507, + "step": 5750 + }, + { + "entropy": 0.3186205253005028, + "epoch": 0.7836734693877551, + "grad_norm": 0.42055535316467285, + "learning_rate": 1.0897260273972604e-05, + "loss": 0.3032658815383911, + "mean_token_accuracy": 0.9145529806613922, + "num_tokens": 11108724.0, + "step": 5760 + }, + { + "entropy": 0.3299889124929905, + "epoch": 0.7850340136054422, + "grad_norm": 0.42789825797080994, + "learning_rate": 1.0828767123287672e-05, + "loss": 0.34171323776245116, + "mean_token_accuracy": 0.9122124373912811, + "num_tokens": 11126528.0, + "step": 5770 + }, + { + "entropy": 0.33234588280320165, + "epoch": 0.7863945578231293, + "grad_norm": 0.5156826376914978, + "learning_rate": 1.076027397260274e-05, + "loss": 0.3394943714141846, + "mean_token_accuracy": 0.9124648332595825, + "num_tokens": 11144913.0, + "step": 5780 + }, + { + "entropy": 0.3698764108121395, + "epoch": 0.7877551020408163, + "grad_norm": 0.3107636272907257, + "learning_rate": 1.0691780821917809e-05, + "loss": 0.3694345235824585, + "mean_token_accuracy": 0.9042896002531051, + "num_tokens": 11167390.0, + "step": 5790 + }, + { + "entropy": 0.3120001643896103, + "epoch": 0.7891156462585034, + "grad_norm": 0.5672539472579956, + "learning_rate": 1.0623287671232877e-05, + "loss": 0.3034272909164429, + "mean_token_accuracy": 0.9210961252450943, + "num_tokens": 11186470.0, + "step": 5800 + }, + { + "entropy": 0.31296552121639254, + "epoch": 0.7904761904761904, + "grad_norm": 0.6534621119499207, + "learning_rate": 1.0554794520547946e-05, + "loss": 0.32099902629852295, + "mean_token_accuracy": 0.9163517713546753, + "num_tokens": 11204042.0, + "step": 5810 + }, + { + "entropy": 0.2991756394505501, + "epoch": 0.7918367346938775, + "grad_norm": 0.5420209169387817, + "learning_rate": 1.0486301369863014e-05, + "loss": 0.2895534038543701, + "mean_token_accuracy": 0.9218130499124527, + "num_tokens": 11222662.0, + "step": 5820 + }, + { + "entropy": 0.29351976960897447, + "epoch": 0.7931972789115647, + "grad_norm": 0.41789329051971436, + "learning_rate": 1.0417808219178082e-05, + "loss": 0.2951449155807495, + "mean_token_accuracy": 0.9238580882549285, + "num_tokens": 11240403.0, + "step": 5830 + }, + { + "entropy": 0.36796221882104874, + "epoch": 0.7945578231292517, + "grad_norm": 0.3995990753173828, + "learning_rate": 1.0349315068493151e-05, + "loss": 0.38138272762298586, + "mean_token_accuracy": 0.9061296254396438, + "num_tokens": 11259380.0, + "step": 5840 + }, + { + "entropy": 0.3072926219552755, + "epoch": 0.7959183673469388, + "grad_norm": 0.4705047011375427, + "learning_rate": 1.028082191780822e-05, + "loss": 0.3008893489837646, + "mean_token_accuracy": 0.9149341195821762, + "num_tokens": 11280378.0, + "step": 5850 + }, + { + "entropy": 0.2909562915563583, + "epoch": 0.7972789115646258, + "grad_norm": 0.44585728645324707, + "learning_rate": 1.0212328767123288e-05, + "loss": 0.28569257259368896, + "mean_token_accuracy": 0.9219981044530868, + "num_tokens": 11298956.0, + "step": 5860 + }, + { + "entropy": 0.30830206871032717, + "epoch": 0.7986394557823129, + "grad_norm": 0.6111286282539368, + "learning_rate": 1.0143835616438356e-05, + "loss": 0.3069111108779907, + "mean_token_accuracy": 0.920006063580513, + "num_tokens": 11318152.0, + "step": 5870 + }, + { + "entropy": 0.3211949087679386, + "epoch": 0.8, + "grad_norm": 0.5721073150634766, + "learning_rate": 1.0075342465753424e-05, + "loss": 0.3205984592437744, + "mean_token_accuracy": 0.9189320415258407, + "num_tokens": 11335961.0, + "step": 5880 + }, + { + "entropy": 0.3145246207714081, + "epoch": 0.8013605442176871, + "grad_norm": 0.4170847237110138, + "learning_rate": 1.0006849315068494e-05, + "loss": 0.3303114652633667, + "mean_token_accuracy": 0.9154134333133698, + "num_tokens": 11354734.0, + "step": 5890 + }, + { + "entropy": 0.2957589320838451, + "epoch": 0.8027210884353742, + "grad_norm": 0.5433431267738342, + "learning_rate": 9.938356164383563e-06, + "loss": 0.29770326614379883, + "mean_token_accuracy": 0.9195897787809372, + "num_tokens": 11373075.0, + "step": 5900 + }, + { + "entropy": 0.30681760758161547, + "epoch": 0.8040816326530612, + "grad_norm": 0.6477324366569519, + "learning_rate": 9.86986301369863e-06, + "loss": 0.31293673515319825, + "mean_token_accuracy": 0.9200047612190246, + "num_tokens": 11390279.0, + "step": 5910 + }, + { + "entropy": 0.3382511667907238, + "epoch": 0.8054421768707483, + "grad_norm": 0.5619089603424072, + "learning_rate": 9.801369863013699e-06, + "loss": 0.34138176441192625, + "mean_token_accuracy": 0.9094976276159287, + "num_tokens": 11409791.0, + "step": 5920 + }, + { + "entropy": 0.2938502006232738, + "epoch": 0.8068027210884354, + "grad_norm": 0.3731178045272827, + "learning_rate": 9.732876712328768e-06, + "loss": 0.2872425317764282, + "mean_token_accuracy": 0.9224807500839234, + "num_tokens": 11429401.0, + "step": 5930 + }, + { + "entropy": 0.33451291769742963, + "epoch": 0.8081632653061225, + "grad_norm": 0.43806591629981995, + "learning_rate": 9.664383561643836e-06, + "loss": 0.33567047119140625, + "mean_token_accuracy": 0.9116082489490509, + "num_tokens": 11449058.0, + "step": 5940 + }, + { + "entropy": 0.30286668688058854, + "epoch": 0.8095238095238095, + "grad_norm": 0.587124228477478, + "learning_rate": 9.595890410958905e-06, + "loss": 0.31200466156005857, + "mean_token_accuracy": 0.9177083373069763, + "num_tokens": 11470294.0, + "step": 5950 + }, + { + "entropy": 0.3303552411496639, + "epoch": 0.8108843537414966, + "grad_norm": 0.6207278966903687, + "learning_rate": 9.527397260273973e-06, + "loss": 0.33328967094421386, + "mean_token_accuracy": 0.9125414162874221, + "num_tokens": 11489357.0, + "step": 5960 + }, + { + "entropy": 0.31682623624801637, + "epoch": 0.8122448979591836, + "grad_norm": 0.44043925404548645, + "learning_rate": 9.458904109589041e-06, + "loss": 0.31499733924865725, + "mean_token_accuracy": 0.9134839475154877, + "num_tokens": 11508233.0, + "step": 5970 + }, + { + "entropy": 0.3149236857891083, + "epoch": 0.8136054421768707, + "grad_norm": 0.36899179220199585, + "learning_rate": 9.39041095890411e-06, + "loss": 0.3117794990539551, + "mean_token_accuracy": 0.9213671326637268, + "num_tokens": 11525907.0, + "step": 5980 + }, + { + "entropy": 0.3045039355754852, + "epoch": 0.8149659863945579, + "grad_norm": 0.49508801102638245, + "learning_rate": 9.321917808219178e-06, + "loss": 0.2972161531448364, + "mean_token_accuracy": 0.9218720376491547, + "num_tokens": 11545454.0, + "step": 5990 + }, + { + "entropy": 0.33072281926870345, + "epoch": 0.8163265306122449, + "grad_norm": 0.3706882894039154, + "learning_rate": 9.253424657534248e-06, + "loss": 0.339734411239624, + "mean_token_accuracy": 0.9110476523637772, + "num_tokens": 11565392.0, + "step": 6000 + }, + { + "epoch": 0.8163265306122449, + "eval_entropy": 0.33086989077428974, + "eval_loss": 0.32422059774398804, + "eval_mean_token_accuracy": 0.913934031923612, + "eval_num_tokens": 11565392.0, + "eval_runtime": 38.7107, + "eval_samples_per_second": 15.5, + "eval_steps_per_second": 7.75, + "step": 6000 + }, + { + "entropy": 0.298756917193532, + "epoch": 0.817687074829932, + "grad_norm": 0.42621785402297974, + "learning_rate": 9.184931506849315e-06, + "loss": 0.29668822288513186, + "mean_token_accuracy": 0.9230580598115921, + "num_tokens": 11583377.0, + "step": 6010 + }, + { + "entropy": 0.3184999957680702, + "epoch": 0.819047619047619, + "grad_norm": 0.5035575032234192, + "learning_rate": 9.116438356164383e-06, + "loss": 0.3027308702468872, + "mean_token_accuracy": 0.9170263260602951, + "num_tokens": 11603965.0, + "step": 6020 + }, + { + "entropy": 0.3802847608923912, + "epoch": 0.8204081632653061, + "grad_norm": 0.49796271324157715, + "learning_rate": 9.047945205479453e-06, + "loss": 0.38724322319030763, + "mean_token_accuracy": 0.9009683400392532, + "num_tokens": 11624271.0, + "step": 6030 + }, + { + "entropy": 0.27982044592499733, + "epoch": 0.8217687074829932, + "grad_norm": 0.5557689666748047, + "learning_rate": 8.97945205479452e-06, + "loss": 0.2826409816741943, + "mean_token_accuracy": 0.9224202632904053, + "num_tokens": 11644456.0, + "step": 6040 + }, + { + "entropy": 0.3160627827048302, + "epoch": 0.8231292517006803, + "grad_norm": 0.576414167881012, + "learning_rate": 8.91095890410959e-06, + "loss": 0.31594884395599365, + "mean_token_accuracy": 0.9167442142963409, + "num_tokens": 11664317.0, + "step": 6050 + }, + { + "entropy": 0.27179846093058585, + "epoch": 0.8244897959183674, + "grad_norm": 0.5361799597740173, + "learning_rate": 8.84246575342466e-06, + "loss": 0.2700121641159058, + "mean_token_accuracy": 0.9278384625911713, + "num_tokens": 11682377.0, + "step": 6060 + }, + { + "entropy": 0.30858047008514405, + "epoch": 0.8258503401360544, + "grad_norm": 0.4765695333480835, + "learning_rate": 8.773972602739725e-06, + "loss": 0.30114083290100097, + "mean_token_accuracy": 0.9185465186834335, + "num_tokens": 11700972.0, + "step": 6070 + }, + { + "entropy": 0.3052784204483032, + "epoch": 0.8272108843537415, + "grad_norm": 0.5827853679656982, + "learning_rate": 8.705479452054795e-06, + "loss": 0.30794899463653563, + "mean_token_accuracy": 0.917677515745163, + "num_tokens": 11721322.0, + "step": 6080 + }, + { + "entropy": 0.31110915169119835, + "epoch": 0.8285714285714286, + "grad_norm": 0.5326927900314331, + "learning_rate": 8.636986301369863e-06, + "loss": 0.32460036277771, + "mean_token_accuracy": 0.9198139071464538, + "num_tokens": 11738982.0, + "step": 6090 + }, + { + "entropy": 0.33925400823354723, + "epoch": 0.8299319727891157, + "grad_norm": 0.36450883746147156, + "learning_rate": 8.568493150684932e-06, + "loss": 0.3317252635955811, + "mean_token_accuracy": 0.9089361041784286, + "num_tokens": 11760916.0, + "step": 6100 + }, + { + "entropy": 0.3158437326550484, + "epoch": 0.8312925170068027, + "grad_norm": 0.491068959236145, + "learning_rate": 8.500000000000002e-06, + "loss": 0.3143324851989746, + "mean_token_accuracy": 0.9161739617586135, + "num_tokens": 11779949.0, + "step": 6110 + }, + { + "entropy": 0.3439742915332317, + "epoch": 0.8326530612244898, + "grad_norm": 0.39802658557891846, + "learning_rate": 8.431506849315068e-06, + "loss": 0.3491824626922607, + "mean_token_accuracy": 0.9078180462121963, + "num_tokens": 11799970.0, + "step": 6120 + }, + { + "entropy": 0.2836092166602612, + "epoch": 0.8340136054421768, + "grad_norm": 0.4259534478187561, + "learning_rate": 8.363013698630137e-06, + "loss": 0.2912003040313721, + "mean_token_accuracy": 0.9252801060676574, + "num_tokens": 11820426.0, + "step": 6130 + }, + { + "entropy": 0.3005498096346855, + "epoch": 0.8353741496598639, + "grad_norm": 0.4778730273246765, + "learning_rate": 8.294520547945207e-06, + "loss": 0.28653755187988283, + "mean_token_accuracy": 0.9214383721351623, + "num_tokens": 11839890.0, + "step": 6140 + }, + { + "entropy": 0.3486380882561207, + "epoch": 0.8367346938775511, + "grad_norm": 0.6083812117576599, + "learning_rate": 8.226027397260274e-06, + "loss": 0.35432674884796145, + "mean_token_accuracy": 0.9074101686477661, + "num_tokens": 11859990.0, + "step": 6150 + }, + { + "entropy": 0.32456274405121804, + "epoch": 0.8380952380952381, + "grad_norm": 0.4606610834598541, + "learning_rate": 8.157534246575344e-06, + "loss": 0.3331348419189453, + "mean_token_accuracy": 0.9130833357572555, + "num_tokens": 11878717.0, + "step": 6160 + }, + { + "entropy": 0.32492469996213913, + "epoch": 0.8394557823129252, + "grad_norm": 0.4846050441265106, + "learning_rate": 8.089041095890412e-06, + "loss": 0.3428283929824829, + "mean_token_accuracy": 0.9127777367830276, + "num_tokens": 11897621.0, + "step": 6170 + }, + { + "entropy": 0.3137147091329098, + "epoch": 0.8408163265306122, + "grad_norm": 0.45196622610092163, + "learning_rate": 8.02054794520548e-06, + "loss": 0.3091363191604614, + "mean_token_accuracy": 0.9139469265937805, + "num_tokens": 11916535.0, + "step": 6180 + }, + { + "entropy": 0.29806752055883406, + "epoch": 0.8421768707482993, + "grad_norm": 0.454557865858078, + "learning_rate": 7.952054794520549e-06, + "loss": 0.29362752437591555, + "mean_token_accuracy": 0.9199837118387222, + "num_tokens": 11935369.0, + "step": 6190 + }, + { + "entropy": 0.28466799929738046, + "epoch": 0.8435374149659864, + "grad_norm": 0.5338443517684937, + "learning_rate": 7.883561643835617e-06, + "loss": 0.29237349033355714, + "mean_token_accuracy": 0.9232655674219131, + "num_tokens": 11953135.0, + "step": 6200 + }, + { + "entropy": 0.3033223427832127, + "epoch": 0.8448979591836735, + "grad_norm": 0.6123531460762024, + "learning_rate": 7.815068493150686e-06, + "loss": 0.3017022371292114, + "mean_token_accuracy": 0.920993834733963, + "num_tokens": 11970670.0, + "step": 6210 + }, + { + "entropy": 0.36309069730341437, + "epoch": 0.8462585034013606, + "grad_norm": 0.6074350476264954, + "learning_rate": 7.746575342465754e-06, + "loss": 0.3690043926239014, + "mean_token_accuracy": 0.907643273472786, + "num_tokens": 11988089.0, + "step": 6220 + }, + { + "entropy": 0.32631669789552686, + "epoch": 0.8476190476190476, + "grad_norm": 0.5981218814849854, + "learning_rate": 7.678082191780822e-06, + "loss": 0.337539529800415, + "mean_token_accuracy": 0.9157464295625687, + "num_tokens": 12005832.0, + "step": 6230 + }, + { + "entropy": 0.34066905826330185, + "epoch": 0.8489795918367347, + "grad_norm": 0.626277506351471, + "learning_rate": 7.609589041095891e-06, + "loss": 0.3323520660400391, + "mean_token_accuracy": 0.9154704749584198, + "num_tokens": 12022879.0, + "step": 6240 + }, + { + "entropy": 0.3810716487467289, + "epoch": 0.8503401360544217, + "grad_norm": 0.4984925091266632, + "learning_rate": 7.54109589041096e-06, + "loss": 0.4056093215942383, + "mean_token_accuracy": 0.9004001200199128, + "num_tokens": 12042173.0, + "step": 6250 + }, + { + "epoch": 0.8503401360544217, + "eval_entropy": 0.32555937600632506, + "eval_loss": 0.32379451394081116, + "eval_mean_token_accuracy": 0.9141173479954402, + "eval_num_tokens": 12042173.0, + "eval_runtime": 38.9464, + "eval_samples_per_second": 15.406, + "eval_steps_per_second": 7.703, + "step": 6250 + }, + { + "entropy": 0.30119421854615214, + "epoch": 0.8517006802721089, + "grad_norm": 0.45719167590141296, + "learning_rate": 7.4726027397260285e-06, + "loss": 0.2935132265090942, + "mean_token_accuracy": 0.9244018375873566, + "num_tokens": 12059658.0, + "step": 6260 + }, + { + "entropy": 0.3010863531380892, + "epoch": 0.8530612244897959, + "grad_norm": 0.5109034776687622, + "learning_rate": 7.404109589041095e-06, + "loss": 0.31348700523376466, + "mean_token_accuracy": 0.9156623929738998, + "num_tokens": 12078793.0, + "step": 6270 + }, + { + "entropy": 0.30613968297839167, + "epoch": 0.854421768707483, + "grad_norm": 0.5922409892082214, + "learning_rate": 7.335616438356165e-06, + "loss": 0.31641838550567625, + "mean_token_accuracy": 0.9163318037986755, + "num_tokens": 12098446.0, + "step": 6280 + }, + { + "entropy": 0.3765171080827713, + "epoch": 0.85578231292517, + "grad_norm": 0.5758050084114075, + "learning_rate": 7.2671232876712335e-06, + "loss": 0.37129898071289064, + "mean_token_accuracy": 0.9047569125890732, + "num_tokens": 12118576.0, + "step": 6290 + }, + { + "entropy": 0.3128926806151867, + "epoch": 0.8571428571428571, + "grad_norm": 0.3453630208969116, + "learning_rate": 7.198630136986302e-06, + "loss": 0.31790430545806886, + "mean_token_accuracy": 0.9154722660779953, + "num_tokens": 12137064.0, + "step": 6300 + }, + { + "entropy": 0.34110006168484686, + "epoch": 0.8585034013605443, + "grad_norm": 0.7402700185775757, + "learning_rate": 7.130136986301371e-06, + "loss": 0.3416825532913208, + "mean_token_accuracy": 0.910080087184906, + "num_tokens": 12157405.0, + "step": 6310 + }, + { + "entropy": 0.3634137712419033, + "epoch": 0.8598639455782313, + "grad_norm": 0.6401071548461914, + "learning_rate": 7.0616438356164386e-06, + "loss": 0.36912858486175537, + "mean_token_accuracy": 0.9012720108032226, + "num_tokens": 12176452.0, + "step": 6320 + }, + { + "entropy": 0.3082569859921932, + "epoch": 0.8612244897959184, + "grad_norm": 0.5355600118637085, + "learning_rate": 6.993150684931507e-06, + "loss": 0.29816470146179197, + "mean_token_accuracy": 0.9203158468008041, + "num_tokens": 12194612.0, + "step": 6330 + }, + { + "entropy": 0.3393115036189556, + "epoch": 0.8625850340136054, + "grad_norm": 0.38070395588874817, + "learning_rate": 6.924657534246576e-06, + "loss": 0.32619943618774416, + "mean_token_accuracy": 0.9085036665201187, + "num_tokens": 12217956.0, + "step": 6340 + }, + { + "entropy": 0.3250208295881748, + "epoch": 0.8639455782312925, + "grad_norm": 0.7538926005363464, + "learning_rate": 6.8561643835616444e-06, + "loss": 0.33664188385009763, + "mean_token_accuracy": 0.9119222640991211, + "num_tokens": 12236318.0, + "step": 6350 + }, + { + "entropy": 0.2986321195960045, + "epoch": 0.8653061224489796, + "grad_norm": 0.4130662679672241, + "learning_rate": 6.787671232876712e-06, + "loss": 0.28825740814208983, + "mean_token_accuracy": 0.9217880070209503, + "num_tokens": 12255826.0, + "step": 6360 + }, + { + "entropy": 0.2841433063149452, + "epoch": 0.8666666666666667, + "grad_norm": 0.6472411155700684, + "learning_rate": 6.719178082191781e-06, + "loss": 0.27176086902618407, + "mean_token_accuracy": 0.9227411568164825, + "num_tokens": 12276465.0, + "step": 6370 + }, + { + "entropy": 0.34614059180021284, + "epoch": 0.8680272108843538, + "grad_norm": 0.4979070723056793, + "learning_rate": 6.6506849315068495e-06, + "loss": 0.3492851495742798, + "mean_token_accuracy": 0.9061255156993866, + "num_tokens": 12295950.0, + "step": 6380 + }, + { + "entropy": 0.3491402305662632, + "epoch": 0.8693877551020408, + "grad_norm": 0.5168848633766174, + "learning_rate": 6.582191780821918e-06, + "loss": 0.356025767326355, + "mean_token_accuracy": 0.9081326395273208, + "num_tokens": 12315074.0, + "step": 6390 + }, + { + "entropy": 0.3183485813438892, + "epoch": 0.8707482993197279, + "grad_norm": 0.3737778961658478, + "learning_rate": 6.513698630136987e-06, + "loss": 0.3137796401977539, + "mean_token_accuracy": 0.9168394416570663, + "num_tokens": 12333113.0, + "step": 6400 + }, + { + "entropy": 0.3234575852751732, + "epoch": 0.8721088435374149, + "grad_norm": 0.4080883860588074, + "learning_rate": 6.4452054794520545e-06, + "loss": 0.3283559799194336, + "mean_token_accuracy": 0.910961264371872, + "num_tokens": 12354119.0, + "step": 6410 + }, + { + "entropy": 0.33070197626948356, + "epoch": 0.8734693877551021, + "grad_norm": 0.6628795862197876, + "learning_rate": 6.376712328767123e-06, + "loss": 0.33696324825286866, + "mean_token_accuracy": 0.9128307223320007, + "num_tokens": 12372396.0, + "step": 6420 + }, + { + "entropy": 0.296940540522337, + "epoch": 0.8748299319727891, + "grad_norm": 0.5327218174934387, + "learning_rate": 6.308219178082192e-06, + "loss": 0.29673740863800047, + "mean_token_accuracy": 0.9188735485076904, + "num_tokens": 12391926.0, + "step": 6430 + }, + { + "entropy": 0.37082950621843336, + "epoch": 0.8761904761904762, + "grad_norm": 0.5576393008232117, + "learning_rate": 6.23972602739726e-06, + "loss": 0.37212495803833007, + "mean_token_accuracy": 0.903967159986496, + "num_tokens": 12411397.0, + "step": 6440 + }, + { + "entropy": 0.292638061195612, + "epoch": 0.8775510204081632, + "grad_norm": 0.35143810510635376, + "learning_rate": 6.171232876712329e-06, + "loss": 0.2931572675704956, + "mean_token_accuracy": 0.9227148920297623, + "num_tokens": 12432121.0, + "step": 6450 + }, + { + "entropy": 0.3156722396612167, + "epoch": 0.8789115646258503, + "grad_norm": 0.774771511554718, + "learning_rate": 6.102739726027398e-06, + "loss": 0.31828532218933103, + "mean_token_accuracy": 0.9142497092485428, + "num_tokens": 12451601.0, + "step": 6460 + }, + { + "entropy": 0.292581620067358, + "epoch": 0.8802721088435375, + "grad_norm": 0.4632379710674286, + "learning_rate": 6.034246575342466e-06, + "loss": 0.28803262710571287, + "mean_token_accuracy": 0.9243631362915039, + "num_tokens": 12470105.0, + "step": 6470 + }, + { + "entropy": 0.3136554330587387, + "epoch": 0.8816326530612245, + "grad_norm": 0.5806280374526978, + "learning_rate": 5.965753424657534e-06, + "loss": 0.31753370761871336, + "mean_token_accuracy": 0.9147692441940307, + "num_tokens": 12488804.0, + "step": 6480 + }, + { + "entropy": 0.33160665780305865, + "epoch": 0.8829931972789116, + "grad_norm": 0.47363021969795227, + "learning_rate": 5.8972602739726035e-06, + "loss": 0.3279127836227417, + "mean_token_accuracy": 0.9173679798841476, + "num_tokens": 12507279.0, + "step": 6490 + }, + { + "entropy": 0.30978685468435285, + "epoch": 0.8843537414965986, + "grad_norm": 0.3723524212837219, + "learning_rate": 5.828767123287671e-06, + "loss": 0.311632776260376, + "mean_token_accuracy": 0.9137750864028931, + "num_tokens": 12527509.0, + "step": 6500 + }, + { + "epoch": 0.8843537414965986, + "eval_entropy": 0.3286215581248204, + "eval_loss": 0.32335928082466125, + "eval_mean_token_accuracy": 0.9142575333515803, + "eval_num_tokens": 12527509.0, + "eval_runtime": 38.7448, + "eval_samples_per_second": 15.486, + "eval_steps_per_second": 7.743, + "step": 6500 + }, + { + "entropy": 0.35880630612373354, + "epoch": 0.8857142857142857, + "grad_norm": 0.514137864112854, + "learning_rate": 5.76027397260274e-06, + "loss": 0.3609093427658081, + "mean_token_accuracy": 0.908568149805069, + "num_tokens": 12546024.0, + "step": 6510 + }, + { + "entropy": 0.2908146843314171, + "epoch": 0.8870748299319728, + "grad_norm": 0.4764300286769867, + "learning_rate": 5.6917808219178085e-06, + "loss": 0.29313054084777834, + "mean_token_accuracy": 0.9210966646671295, + "num_tokens": 12564957.0, + "step": 6520 + }, + { + "entropy": 0.3947236731648445, + "epoch": 0.8884353741496599, + "grad_norm": 0.564361572265625, + "learning_rate": 5.623287671232877e-06, + "loss": 0.3897683620452881, + "mean_token_accuracy": 0.9005183577537537, + "num_tokens": 12584511.0, + "step": 6530 + }, + { + "entropy": 0.35087050646543505, + "epoch": 0.889795918367347, + "grad_norm": 0.39907175302505493, + "learning_rate": 5.554794520547946e-06, + "loss": 0.35727748870849607, + "mean_token_accuracy": 0.9082259654998779, + "num_tokens": 12603539.0, + "step": 6540 + }, + { + "entropy": 0.30452171340584755, + "epoch": 0.891156462585034, + "grad_norm": 0.40334832668304443, + "learning_rate": 5.486301369863014e-06, + "loss": 0.31457273960113524, + "mean_token_accuracy": 0.9177052706480027, + "num_tokens": 12622546.0, + "step": 6550 + }, + { + "entropy": 0.34483264610171316, + "epoch": 0.8925170068027211, + "grad_norm": 0.5541892051696777, + "learning_rate": 5.417808219178082e-06, + "loss": 0.35019664764404296, + "mean_token_accuracy": 0.9109074980020523, + "num_tokens": 12641267.0, + "step": 6560 + }, + { + "entropy": 0.3339083693921566, + "epoch": 0.8938775510204081, + "grad_norm": 0.4268859624862671, + "learning_rate": 5.349315068493151e-06, + "loss": 0.33824708461761477, + "mean_token_accuracy": 0.9065180271863937, + "num_tokens": 12662141.0, + "step": 6570 + }, + { + "entropy": 0.32709172666072844, + "epoch": 0.8952380952380953, + "grad_norm": 0.4115796387195587, + "learning_rate": 5.2808219178082195e-06, + "loss": 0.33449928760528563, + "mean_token_accuracy": 0.9107886403799057, + "num_tokens": 12683863.0, + "step": 6580 + }, + { + "entropy": 0.359021432697773, + "epoch": 0.8965986394557823, + "grad_norm": 0.5937194228172302, + "learning_rate": 5.212328767123288e-06, + "loss": 0.34922354221343993, + "mean_token_accuracy": 0.9072045892477035, + "num_tokens": 12703543.0, + "step": 6590 + }, + { + "entropy": 0.39199258685112, + "epoch": 0.8979591836734694, + "grad_norm": 0.6500011086463928, + "learning_rate": 5.143835616438356e-06, + "loss": 0.3893763542175293, + "mean_token_accuracy": 0.9033649802207947, + "num_tokens": 12721280.0, + "step": 6600 + }, + { + "entropy": 0.2838050812482834, + "epoch": 0.8993197278911564, + "grad_norm": 0.4997044503688812, + "learning_rate": 5.075342465753425e-06, + "loss": 0.28191652297973635, + "mean_token_accuracy": 0.9257587313652038, + "num_tokens": 12738942.0, + "step": 6610 + }, + { + "entropy": 0.3208158388733864, + "epoch": 0.9006802721088435, + "grad_norm": 0.463008850812912, + "learning_rate": 5.006849315068493e-06, + "loss": 0.3163119316101074, + "mean_token_accuracy": 0.9138482451438904, + "num_tokens": 12758737.0, + "step": 6620 + }, + { + "entropy": 0.32699171006679534, + "epoch": 0.9020408163265307, + "grad_norm": 0.46865853667259216, + "learning_rate": 4.938356164383562e-06, + "loss": 0.3282167434692383, + "mean_token_accuracy": 0.917575141787529, + "num_tokens": 12776236.0, + "step": 6630 + }, + { + "entropy": 0.33992503136396407, + "epoch": 0.9034013605442177, + "grad_norm": 0.622612714767456, + "learning_rate": 4.86986301369863e-06, + "loss": 0.3454226732254028, + "mean_token_accuracy": 0.9105749785900116, + "num_tokens": 12793838.0, + "step": 6640 + }, + { + "entropy": 0.29390531592071056, + "epoch": 0.9047619047619048, + "grad_norm": 0.44798150658607483, + "learning_rate": 4.801369863013699e-06, + "loss": 0.27416980266571045, + "mean_token_accuracy": 0.9256226003170014, + "num_tokens": 12813036.0, + "step": 6650 + }, + { + "entropy": 0.350774671882391, + "epoch": 0.9061224489795918, + "grad_norm": 0.38738077878952026, + "learning_rate": 4.732876712328768e-06, + "loss": 0.33705477714538573, + "mean_token_accuracy": 0.9067309200763702, + "num_tokens": 12834574.0, + "step": 6660 + }, + { + "entropy": 0.3244813568890095, + "epoch": 0.9074829931972789, + "grad_norm": 0.4696221649646759, + "learning_rate": 4.664383561643835e-06, + "loss": 0.31435041427612304, + "mean_token_accuracy": 0.9171089708805085, + "num_tokens": 12852605.0, + "step": 6670 + }, + { + "entropy": 0.305752881616354, + "epoch": 0.908843537414966, + "grad_norm": 0.4763096868991852, + "learning_rate": 4.595890410958904e-06, + "loss": 0.3067082643508911, + "mean_token_accuracy": 0.918914520740509, + "num_tokens": 12870795.0, + "step": 6680 + }, + { + "entropy": 0.31308693066239357, + "epoch": 0.9102040816326531, + "grad_norm": 0.5077400803565979, + "learning_rate": 4.5273972602739735e-06, + "loss": 0.30920541286468506, + "mean_token_accuracy": 0.9194491058588028, + "num_tokens": 12890313.0, + "step": 6690 + }, + { + "entropy": 0.2930800631642342, + "epoch": 0.9115646258503401, + "grad_norm": 0.5321216583251953, + "learning_rate": 4.458904109589041e-06, + "loss": 0.2862373352050781, + "mean_token_accuracy": 0.9227749168872833, + "num_tokens": 12909441.0, + "step": 6700 + }, + { + "entropy": 0.32441584542393687, + "epoch": 0.9129251700680272, + "grad_norm": 0.5052834153175354, + "learning_rate": 4.39041095890411e-06, + "loss": 0.33454344272613523, + "mean_token_accuracy": 0.9128939062356949, + "num_tokens": 12928259.0, + "step": 6710 + }, + { + "entropy": 0.3400576122105122, + "epoch": 0.9142857142857143, + "grad_norm": 0.42697256803512573, + "learning_rate": 4.321917808219178e-06, + "loss": 0.3487342357635498, + "mean_token_accuracy": 0.9104131639003754, + "num_tokens": 12946832.0, + "step": 6720 + }, + { + "entropy": 0.3238037794828415, + "epoch": 0.9156462585034013, + "grad_norm": 0.4467569887638092, + "learning_rate": 4.253424657534247e-06, + "loss": 0.3275768518447876, + "mean_token_accuracy": 0.9114355087280274, + "num_tokens": 12967923.0, + "step": 6730 + }, + { + "entropy": 0.3033832423388958, + "epoch": 0.9170068027210885, + "grad_norm": 0.5587620139122009, + "learning_rate": 4.184931506849316e-06, + "loss": 0.3123866319656372, + "mean_token_accuracy": 0.9181416839361191, + "num_tokens": 12986379.0, + "step": 6740 + }, + { + "entropy": 0.32793152034282685, + "epoch": 0.9183673469387755, + "grad_norm": 0.4971228241920471, + "learning_rate": 4.116438356164384e-06, + "loss": 0.32430260181427, + "mean_token_accuracy": 0.9148997128009796, + "num_tokens": 13006250.0, + "step": 6750 + }, + { + "epoch": 0.9183673469387755, + "eval_entropy": 0.32498093172907827, + "eval_loss": 0.3230104446411133, + "eval_mean_token_accuracy": 0.9144533753395081, + "eval_num_tokens": 13006250.0, + "eval_runtime": 38.9222, + "eval_samples_per_second": 15.415, + "eval_steps_per_second": 7.708, + "step": 6750 + }, + { + "entropy": 0.35129692405462265, + "epoch": 0.9197278911564626, + "grad_norm": 0.450941264629364, + "learning_rate": 4.047945205479452e-06, + "loss": 0.36153857707977294, + "mean_token_accuracy": 0.9063918352127075, + "num_tokens": 13026583.0, + "step": 6760 + }, + { + "entropy": 0.3283290073275566, + "epoch": 0.9210884353741496, + "grad_norm": 0.5283523797988892, + "learning_rate": 3.979452054794521e-06, + "loss": 0.3139669179916382, + "mean_token_accuracy": 0.9148370325565338, + "num_tokens": 13047823.0, + "step": 6770 + }, + { + "entropy": 0.33190669044852256, + "epoch": 0.9224489795918367, + "grad_norm": 0.361507773399353, + "learning_rate": 3.9109589041095895e-06, + "loss": 0.34453206062316893, + "mean_token_accuracy": 0.9123291075229645, + "num_tokens": 13065483.0, + "step": 6780 + }, + { + "entropy": 0.3849423386156559, + "epoch": 0.9238095238095239, + "grad_norm": 0.5551365613937378, + "learning_rate": 3.842465753424658e-06, + "loss": 0.3853173017501831, + "mean_token_accuracy": 0.9038680106401443, + "num_tokens": 13084658.0, + "step": 6790 + }, + { + "entropy": 0.3309860430657864, + "epoch": 0.9251700680272109, + "grad_norm": 0.5244467258453369, + "learning_rate": 3.7739726027397263e-06, + "loss": 0.32932875156402586, + "mean_token_accuracy": 0.909374451637268, + "num_tokens": 13105539.0, + "step": 6800 + }, + { + "entropy": 0.26563679054379463, + "epoch": 0.926530612244898, + "grad_norm": 0.4336610436439514, + "learning_rate": 3.705479452054795e-06, + "loss": 0.25863826274871826, + "mean_token_accuracy": 0.9274649411439896, + "num_tokens": 13124934.0, + "step": 6810 + }, + { + "entropy": 0.3203303225338459, + "epoch": 0.927891156462585, + "grad_norm": 0.39582449197769165, + "learning_rate": 3.636986301369863e-06, + "loss": 0.32346789836883544, + "mean_token_accuracy": 0.9153842777013779, + "num_tokens": 13143409.0, + "step": 6820 + }, + { + "entropy": 0.33626803755760193, + "epoch": 0.9292517006802721, + "grad_norm": 0.6105754375457764, + "learning_rate": 3.5684931506849317e-06, + "loss": 0.3388401508331299, + "mean_token_accuracy": 0.9112157255411149, + "num_tokens": 13163858.0, + "step": 6830 + }, + { + "entropy": 0.3165531374514103, + "epoch": 0.9306122448979591, + "grad_norm": 0.6469919681549072, + "learning_rate": 3.5000000000000004e-06, + "loss": 0.3066595792770386, + "mean_token_accuracy": 0.9175763040781021, + "num_tokens": 13183825.0, + "step": 6840 + }, + { + "entropy": 0.38997668847441674, + "epoch": 0.9319727891156463, + "grad_norm": 0.49790993332862854, + "learning_rate": 3.4315068493150686e-06, + "loss": 0.40053672790527345, + "mean_token_accuracy": 0.8986356943845749, + "num_tokens": 13203094.0, + "step": 6850 + }, + { + "entropy": 0.2587420903146267, + "epoch": 0.9333333333333333, + "grad_norm": 0.4957631230354309, + "learning_rate": 3.363013698630137e-06, + "loss": 0.25925211906433104, + "mean_token_accuracy": 0.9277909904718399, + "num_tokens": 13224483.0, + "step": 6860 + }, + { + "entropy": 0.2768689256161451, + "epoch": 0.9346938775510204, + "grad_norm": 0.398699551820755, + "learning_rate": 3.2945205479452054e-06, + "loss": 0.27082579135894774, + "mean_token_accuracy": 0.9245598196983338, + "num_tokens": 13243719.0, + "step": 6870 + }, + { + "entropy": 0.2864796228706837, + "epoch": 0.9360544217687075, + "grad_norm": 0.4619617164134979, + "learning_rate": 3.226027397260274e-06, + "loss": 0.28009073734283446, + "mean_token_accuracy": 0.9254051715135574, + "num_tokens": 13263387.0, + "step": 6880 + }, + { + "entropy": 0.2629375524818897, + "epoch": 0.9374149659863945, + "grad_norm": 0.5026074647903442, + "learning_rate": 3.1575342465753422e-06, + "loss": 0.2672743558883667, + "mean_token_accuracy": 0.9294937402009964, + "num_tokens": 13280810.0, + "step": 6890 + }, + { + "entropy": 0.3043747998774052, + "epoch": 0.9387755102040817, + "grad_norm": 0.6281470656394958, + "learning_rate": 3.089041095890411e-06, + "loss": 0.3025246858596802, + "mean_token_accuracy": 0.9180625200271606, + "num_tokens": 13299231.0, + "step": 6900 + }, + { + "entropy": 0.33138156682252884, + "epoch": 0.9401360544217687, + "grad_norm": 0.5016866326332092, + "learning_rate": 3.0205479452054795e-06, + "loss": 0.34398717880249025, + "mean_token_accuracy": 0.911366480588913, + "num_tokens": 13319077.0, + "step": 6910 + }, + { + "entropy": 0.324043782800436, + "epoch": 0.9414965986394558, + "grad_norm": 0.3665086328983307, + "learning_rate": 2.952054794520548e-06, + "loss": 0.3297460079193115, + "mean_token_accuracy": 0.9116989314556122, + "num_tokens": 13339163.0, + "step": 6920 + }, + { + "entropy": 0.31357899233698844, + "epoch": 0.9428571428571428, + "grad_norm": 0.4561605155467987, + "learning_rate": 2.8835616438356167e-06, + "loss": 0.3067751169204712, + "mean_token_accuracy": 0.9200424224138259, + "num_tokens": 13357061.0, + "step": 6930 + }, + { + "entropy": 0.33805915862321856, + "epoch": 0.9442176870748299, + "grad_norm": 0.3674219846725464, + "learning_rate": 2.815068493150685e-06, + "loss": 0.3418180704116821, + "mean_token_accuracy": 0.908822026848793, + "num_tokens": 13377792.0, + "step": 6940 + }, + { + "entropy": 0.3485445693135262, + "epoch": 0.9455782312925171, + "grad_norm": 0.3852236568927765, + "learning_rate": 2.7465753424657536e-06, + "loss": 0.3584400177001953, + "mean_token_accuracy": 0.9075007975101471, + "num_tokens": 13398215.0, + "step": 6950 + }, + { + "entropy": 0.30585007295012473, + "epoch": 0.9469387755102041, + "grad_norm": 0.5300541520118713, + "learning_rate": 2.6780821917808218e-06, + "loss": 0.3072829246520996, + "mean_token_accuracy": 0.9181619912385941, + "num_tokens": 13418511.0, + "step": 6960 + }, + { + "entropy": 0.3123452767729759, + "epoch": 0.9482993197278912, + "grad_norm": 0.5543433427810669, + "learning_rate": 2.6095890410958904e-06, + "loss": 0.3134234666824341, + "mean_token_accuracy": 0.9177923172712326, + "num_tokens": 13438071.0, + "step": 6970 + }, + { + "entropy": 0.3253453135490417, + "epoch": 0.9496598639455782, + "grad_norm": 0.6263024806976318, + "learning_rate": 2.541095890410959e-06, + "loss": 0.3312222480773926, + "mean_token_accuracy": 0.9184915989637374, + "num_tokens": 13455223.0, + "step": 6980 + }, + { + "entropy": 0.30098590180277823, + "epoch": 0.9510204081632653, + "grad_norm": 0.41422611474990845, + "learning_rate": 2.4726027397260277e-06, + "loss": 0.29193758964538574, + "mean_token_accuracy": 0.9190629869699478, + "num_tokens": 13476518.0, + "step": 6990 + }, + { + "entropy": 0.2495936904102564, + "epoch": 0.9523809523809523, + "grad_norm": 0.5148512125015259, + "learning_rate": 2.404109589041096e-06, + "loss": 0.25398826599121094, + "mean_token_accuracy": 0.931014546751976, + "num_tokens": 13495349.0, + "step": 7000 + }, + { + "epoch": 0.9523809523809523, + "eval_entropy": 0.32407169657448925, + "eval_loss": 0.3227205276489258, + "eval_mean_token_accuracy": 0.9144436609745026, + "eval_num_tokens": 13495349.0, + "eval_runtime": 40.5516, + "eval_samples_per_second": 14.796, + "eval_steps_per_second": 7.398, + "step": 7000 + }, + { + "entropy": 0.39397928565740586, + "epoch": 0.9537414965986395, + "grad_norm": 0.49788719415664673, + "learning_rate": 2.3356164383561645e-06, + "loss": 0.3860008955001831, + "mean_token_accuracy": 0.897650420665741, + "num_tokens": 13514703.0, + "step": 7010 + }, + { + "entropy": 0.3235211636871099, + "epoch": 0.9551020408163265, + "grad_norm": 0.7469091415405273, + "learning_rate": 2.2671232876712327e-06, + "loss": 0.33283085823059083, + "mean_token_accuracy": 0.9115709811449051, + "num_tokens": 13534485.0, + "step": 7020 + }, + { + "entropy": 0.28874940276145933, + "epoch": 0.9564625850340136, + "grad_norm": 0.3857863247394562, + "learning_rate": 2.1986301369863017e-06, + "loss": 0.2785567045211792, + "mean_token_accuracy": 0.9208014994859696, + "num_tokens": 13554148.0, + "step": 7030 + }, + { + "entropy": 0.3946863390505314, + "epoch": 0.9578231292517007, + "grad_norm": 0.5933367609977722, + "learning_rate": 2.13013698630137e-06, + "loss": 0.41825008392333984, + "mean_token_accuracy": 0.8936791986227035, + "num_tokens": 13575347.0, + "step": 7040 + }, + { + "entropy": 0.31831209883093836, + "epoch": 0.9591836734693877, + "grad_norm": 0.5279243588447571, + "learning_rate": 2.0616438356164386e-06, + "loss": 0.3086137056350708, + "mean_token_accuracy": 0.9153400510549545, + "num_tokens": 13594286.0, + "step": 7050 + }, + { + "entropy": 0.3001719556748867, + "epoch": 0.9605442176870749, + "grad_norm": 0.5925727486610413, + "learning_rate": 1.9931506849315068e-06, + "loss": 0.29816312789916993, + "mean_token_accuracy": 0.9233756989240647, + "num_tokens": 13612101.0, + "step": 7060 + }, + { + "entropy": 0.2860943116247654, + "epoch": 0.9619047619047619, + "grad_norm": 0.4324999749660492, + "learning_rate": 1.9246575342465754e-06, + "loss": 0.28548643589019773, + "mean_token_accuracy": 0.9229339271783829, + "num_tokens": 13632308.0, + "step": 7070 + }, + { + "entropy": 0.2696798574179411, + "epoch": 0.963265306122449, + "grad_norm": 0.47976070642471313, + "learning_rate": 1.856164383561644e-06, + "loss": 0.2697856903076172, + "mean_token_accuracy": 0.9258852988481522, + "num_tokens": 13652430.0, + "step": 7080 + }, + { + "entropy": 0.3074658364057541, + "epoch": 0.964625850340136, + "grad_norm": 0.5789731740951538, + "learning_rate": 1.7876712328767124e-06, + "loss": 0.30829975605010984, + "mean_token_accuracy": 0.9198275089263916, + "num_tokens": 13670932.0, + "step": 7090 + }, + { + "entropy": 0.2700748883187771, + "epoch": 0.9659863945578231, + "grad_norm": 0.5600951313972473, + "learning_rate": 1.7191780821917809e-06, + "loss": 0.2561317205429077, + "mean_token_accuracy": 0.9302997469902039, + "num_tokens": 13689050.0, + "step": 7100 + }, + { + "entropy": 0.25459411814808847, + "epoch": 0.9673469387755103, + "grad_norm": 0.4180262088775635, + "learning_rate": 1.6506849315068493e-06, + "loss": 0.26971304416656494, + "mean_token_accuracy": 0.9278194218873977, + "num_tokens": 13707330.0, + "step": 7110 + }, + { + "entropy": 0.3380099833011627, + "epoch": 0.9687074829931973, + "grad_norm": 0.3550525903701782, + "learning_rate": 1.5821917808219177e-06, + "loss": 0.3356317043304443, + "mean_token_accuracy": 0.9100026935338974, + "num_tokens": 13729267.0, + "step": 7120 + }, + { + "entropy": 0.30718842446804046, + "epoch": 0.9700680272108844, + "grad_norm": 0.3944030702114105, + "learning_rate": 1.5136986301369863e-06, + "loss": 0.3045744180679321, + "mean_token_accuracy": 0.9190286636352539, + "num_tokens": 13749923.0, + "step": 7130 + }, + { + "entropy": 0.2982238419353962, + "epoch": 0.9714285714285714, + "grad_norm": 0.5965185165405273, + "learning_rate": 1.4452054794520547e-06, + "loss": 0.2974499225616455, + "mean_token_accuracy": 0.9249072134494781, + "num_tokens": 13766285.0, + "step": 7140 + }, + { + "entropy": 0.30790726840496063, + "epoch": 0.9727891156462585, + "grad_norm": 0.47756844758987427, + "learning_rate": 1.3767123287671234e-06, + "loss": 0.3204644203186035, + "mean_token_accuracy": 0.9175480782985688, + "num_tokens": 13785433.0, + "step": 7150 + }, + { + "entropy": 0.34362565353512764, + "epoch": 0.9741496598639455, + "grad_norm": 0.4054695963859558, + "learning_rate": 1.3082191780821918e-06, + "loss": 0.3412818431854248, + "mean_token_accuracy": 0.9061507374048233, + "num_tokens": 13807369.0, + "step": 7160 + }, + { + "entropy": 0.2839637853205204, + "epoch": 0.9755102040816327, + "grad_norm": 0.5588259100914001, + "learning_rate": 1.2397260273972604e-06, + "loss": 0.27560198307037354, + "mean_token_accuracy": 0.9249170988798141, + "num_tokens": 13827276.0, + "step": 7170 + }, + { + "entropy": 0.343147137016058, + "epoch": 0.9768707482993197, + "grad_norm": 0.8266026973724365, + "learning_rate": 1.1712328767123288e-06, + "loss": 0.36100358963012696, + "mean_token_accuracy": 0.9057711988687516, + "num_tokens": 13845606.0, + "step": 7180 + }, + { + "entropy": 0.35489280000329015, + "epoch": 0.9782312925170068, + "grad_norm": 0.4694916605949402, + "learning_rate": 1.1027397260273972e-06, + "loss": 0.3540391445159912, + "mean_token_accuracy": 0.913532018661499, + "num_tokens": 13865551.0, + "step": 7190 + }, + { + "entropy": 0.33037103042006494, + "epoch": 0.9795918367346939, + "grad_norm": 0.5188440680503845, + "learning_rate": 1.0342465753424659e-06, + "loss": 0.3330535888671875, + "mean_token_accuracy": 0.9100096374750137, + "num_tokens": 13884552.0, + "step": 7200 + }, + { + "entropy": 0.3238568499684334, + "epoch": 0.9809523809523809, + "grad_norm": 0.5132967233657837, + "learning_rate": 9.657534246575343e-07, + "loss": 0.3343978404998779, + "mean_token_accuracy": 0.9120694696903229, + "num_tokens": 13905871.0, + "step": 7210 + }, + { + "entropy": 0.2768064573407173, + "epoch": 0.9823129251700681, + "grad_norm": 0.5923624038696289, + "learning_rate": 8.972602739726027e-07, + "loss": 0.2772003412246704, + "mean_token_accuracy": 0.9270251214504241, + "num_tokens": 13924266.0, + "step": 7220 + }, + { + "entropy": 0.36717668995261193, + "epoch": 0.9836734693877551, + "grad_norm": 0.47369247674942017, + "learning_rate": 8.287671232876713e-07, + "loss": 0.3650016784667969, + "mean_token_accuracy": 0.9068687915802002, + "num_tokens": 13944354.0, + "step": 7230 + }, + { + "entropy": 0.286948836222291, + "epoch": 0.9850340136054422, + "grad_norm": 0.5255698561668396, + "learning_rate": 7.602739726027397e-07, + "loss": 0.2726752519607544, + "mean_token_accuracy": 0.925079733133316, + "num_tokens": 13963359.0, + "step": 7240 + }, + { + "entropy": 0.3442313067615032, + "epoch": 0.9863945578231292, + "grad_norm": 0.47418609261512756, + "learning_rate": 6.917808219178082e-07, + "loss": 0.35138564109802245, + "mean_token_accuracy": 0.9129589021205902, + "num_tokens": 13981101.0, + "step": 7250 + }, + { + "epoch": 0.9863945578231292, + "eval_entropy": 0.32534752316772936, + "eval_loss": 0.32265961170196533, + "eval_mean_token_accuracy": 0.9144763757785161, + "eval_num_tokens": 13981101.0, + "eval_runtime": 40.4232, + "eval_samples_per_second": 14.843, + "eval_steps_per_second": 7.421, + "step": 7250 + }, + { + "entropy": 0.33081375658512113, + "epoch": 0.9877551020408163, + "grad_norm": 0.6706321835517883, + "learning_rate": 6.232876712328768e-07, + "loss": 0.3306723594665527, + "mean_token_accuracy": 0.913168066740036, + "num_tokens": 13999739.0, + "step": 7260 + }, + { + "entropy": 0.33509607464075086, + "epoch": 0.9891156462585035, + "grad_norm": 0.7398495674133301, + "learning_rate": 5.547945205479453e-07, + "loss": 0.3389270305633545, + "mean_token_accuracy": 0.9147013843059539, + "num_tokens": 14019473.0, + "step": 7270 + }, + { + "entropy": 0.3206873118877411, + "epoch": 0.9904761904761905, + "grad_norm": 0.41634514927864075, + "learning_rate": 4.863013698630137e-07, + "loss": 0.31725330352783204, + "mean_token_accuracy": 0.9171919494867324, + "num_tokens": 14037026.0, + "step": 7280 + }, + { + "entropy": 0.32098953425884247, + "epoch": 0.9918367346938776, + "grad_norm": 0.528227686882019, + "learning_rate": 4.1780821917808217e-07, + "loss": 0.32311558723449707, + "mean_token_accuracy": 0.9178615361452103, + "num_tokens": 14054921.0, + "step": 7290 + }, + { + "entropy": 0.34365782923996446, + "epoch": 0.9931972789115646, + "grad_norm": 0.5224635601043701, + "learning_rate": 3.493150684931507e-07, + "loss": 0.34767937660217285, + "mean_token_accuracy": 0.9085775941610337, + "num_tokens": 14074708.0, + "step": 7300 + }, + { + "entropy": 0.32567395493388174, + "epoch": 0.9945578231292517, + "grad_norm": 0.5263256430625916, + "learning_rate": 2.8082191780821916e-07, + "loss": 0.3315287113189697, + "mean_token_accuracy": 0.9172344833612442, + "num_tokens": 14092128.0, + "step": 7310 + }, + { + "entropy": 0.3712735839188099, + "epoch": 0.9959183673469387, + "grad_norm": 0.43343350291252136, + "learning_rate": 2.1232876712328768e-07, + "loss": 0.3691534519195557, + "mean_token_accuracy": 0.9037802994251252, + "num_tokens": 14111135.0, + "step": 7320 + }, + { + "entropy": 0.3800099574029446, + "epoch": 0.9972789115646259, + "grad_norm": 0.5186244249343872, + "learning_rate": 1.4383561643835617e-07, + "loss": 0.3669553518295288, + "mean_token_accuracy": 0.9025765776634216, + "num_tokens": 14132324.0, + "step": 7330 + }, + { + "entropy": 0.33196054100990297, + "epoch": 0.998639455782313, + "grad_norm": 0.48980236053466797, + "learning_rate": 7.534246575342467e-08, + "loss": 0.328293514251709, + "mean_token_accuracy": 0.9147080510854722, + "num_tokens": 14150730.0, + "step": 7340 + }, + { + "entropy": 0.3340927064418793, + "epoch": 1.0, + "grad_norm": 0.5299854278564453, + "learning_rate": 6.8493150684931506e-09, + "loss": 0.331965708732605, + "mean_token_accuracy": 0.9151187986135483, + "num_tokens": 14169990.0, + "step": 7350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.32546927154064176, + "eval_loss": 0.32264190912246704, + "eval_mean_token_accuracy": 0.9144927906990051, + "eval_num_tokens": 14169990.0, + "eval_runtime": 40.2503, + "eval_samples_per_second": 14.907, + "eval_steps_per_second": 7.453, + "step": 7350 + } + ], + "logging_steps": 10, + "max_steps": 7350, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 250, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.8228349046524314e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-7350/training_args.bin b/checkpoint-7350/training_args.bin new file mode 100644 index 0000000..7178ddc --- /dev/null +++ b/checkpoint-7350/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b54c85194e9ec9f646206be51009ed347eb345ba25a878f68633e3a3a2bb102 +size 5841 diff --git a/config.json b/config.json new file mode 100644 index 0000000..a782da3 --- /dev/null +++ b/config.json @@ -0,0 +1,111 @@ +{ + "architectures": [ + "SmolLM3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 128012, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 11008, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 65536, + "max_window_layers": 28, + "mlp_bias": false, + "model_type": "smollm3", + "no_rope_layer_interval": 4, + "no_rope_layers": [ + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0 + ], + "num_attention_heads": 16, + "num_hidden_layers": 36, + "num_key_value_heads": 4, + "pad_token_id": 128012, + "pretraining_tp": 1, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 5000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.17.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..955e48d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,10 @@ +{ + "do_sample": true, + "eos_token_id": [ + 128012 + ], + "pad_token_id": 128012, + "temperature": 0.6, + "top_p": 0.95, + "transformers_version": "5.17.0" +} diff --git a/model-00001-of-00002.safetensors b/model-00001-of-00002.safetensors new file mode 100644 index 0000000..39b58f6 --- /dev/null +++ b/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f27e354ce7ef1374a61920870983aa717945d65e25b3b6bb7ef832d4bad4c8e6 +size 3962776208 diff --git a/model-00002-of-00002.safetensors b/model-00002-of-00002.safetensors new file mode 100644 index 0000000..ef89324 --- /dev/null +++ b/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a0bc645476cbbcab4899327bae0180b3fa6f8fe6c852d3e7df86a35e7a85e8 +size 2187458808 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..0afd8e1 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ad24c47ec0ec791789a448f8618b9af948d328336e5e788008555f119ecfbc6 +size 6150235096 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..299e07a --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,334 @@ +{ + "metadata": { + "total_parameters": 3075098624, + "total_size": 6150197248 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..074fa66 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b6a500b662a34eb3f0374db856ba4ad7de4c81040571d78dc0d357238930005 +size 17208819 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..d953c6c --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": true, + "eos_token": "<|im_end|>", + "fast": false, + "is_local": false, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|im_end|>", + "tokenizer_class": "PreTrainedTokenizerFast" +} \ No newline at end of file