From e7f998cf6de2f50cb50051a06e9dff4f540c3899 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 20 Jul 2026 05:53:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: cs-552-2026-the-transformers/general_knowledge_model Source: Original Platform --- .gitattributes | 37 + README.md | 58 + chat_template.jinja | 94 + checkpoint-1250/chat_template.jinja | 89 + checkpoint-1250/config.json | 63 + checkpoint-1250/generation_config.json | 12 + checkpoint-1250/model.safetensors | 3 + checkpoint-1250/optimizer.pt | 3 + checkpoint-1250/rng_state.pth | 3 + checkpoint-1250/scheduler.pt | 3 + checkpoint-1250/tokenizer.json | 3 + checkpoint-1250/tokenizer_config.json | 30 + checkpoint-1250/trainer_state.json | 3409 +++++++++++ checkpoint-1250/training_args.bin | 3 + checkpoint-2375/chat_template.jinja | 94 + checkpoint-2375/config.json | 63 + checkpoint-2375/generation_config.json | 12 + checkpoint-2375/model.safetensors | 3 + checkpoint-2375/optimizer.pt | 3 + checkpoint-2375/rng_state.pth | 3 + checkpoint-2375/scheduler.pt | 3 + checkpoint-2375/tokenizer.json | 3 + checkpoint-2375/tokenizer_config.json | 30 + checkpoint-2375/trainer_state.json | 2404 ++++++++ checkpoint-2375/training_args.bin | 3 + checkpoint-2551/chat_template.jinja | 89 + checkpoint-2551/config.json | 63 + checkpoint-2551/generation_config.json | 12 + checkpoint-2551/model.safetensors | 3 + checkpoint-2551/optimizer.pt | 3 + checkpoint-2551/rng_state.pth | 3 + checkpoint-2551/scheduler.pt | 3 + checkpoint-2551/tokenizer.json | 3 + checkpoint-2551/tokenizer_config.json | 30 + checkpoint-2551/trainer_state.json | 2584 +++++++++ checkpoint-2551/training_args.bin | 3 + checkpoint-3125/chat_template.jinja | 89 + checkpoint-3125/config.json | 63 + checkpoint-3125/generation_config.json | 12 + checkpoint-3125/model.safetensors | 3 + checkpoint-3125/optimizer.pt | 3 + checkpoint-3125/rng_state.pth | 3 + checkpoint-3125/scheduler.pt | 3 + checkpoint-3125/tokenizer.json | 3 + checkpoint-3125/tokenizer_config.json | 30 + checkpoint-3125/trainer_state.json | 3154 +++++++++++ checkpoint-3125/training_args.bin | 3 + checkpoint-4375/chat_template.jinja | 94 + checkpoint-4375/config.json | 63 + checkpoint-4375/generation_config.json | 12 + checkpoint-4375/model.safetensors | 3 + checkpoint-4375/optimizer.pt | 3 + checkpoint-4375/rng_state.pth | 3 + checkpoint-4375/scheduler.pt | 3 + checkpoint-4375/tokenizer.json | 3 + checkpoint-4375/tokenizer_config.json | 30 + checkpoint-4375/trainer_state.json | 4404 +++++++++++++++ checkpoint-4375/training_args.bin | 3 + checkpoint-6241/chat_template.jinja | 90 + checkpoint-6241/config.json | 63 + checkpoint-6241/generation_config.json | 12 + checkpoint-6241/model.safetensors | 3 + checkpoint-6241/optimizer.pt | 3 + checkpoint-6241/rng_state.pth | 3 + checkpoint-6241/scheduler.pt | 3 + checkpoint-6241/tokenizer.json | 3 + checkpoint-6241/tokenizer_config.json | 30 + checkpoint-6241/trainer_state.json | 6274 +++++++++++++++++++++ checkpoint-6241/training_args.bin | 3 + checkpoint-7136/chat_template.jinja | 13 + checkpoint-7136/config.json | 63 + checkpoint-7136/generation_config.json | 12 + checkpoint-7136/model.safetensors | 3 + checkpoint-7136/optimizer.pt | 3 + checkpoint-7136/rng_state.pth | 3 + checkpoint-7136/scheduler.pt | 3 + checkpoint-7136/tokenizer.json | 3 + checkpoint-7136/tokenizer_config.json | 30 + checkpoint-7136/trainer_state.json | 7164 ++++++++++++++++++++++++ checkpoint-7136/training_args.bin | 3 + checkpoint-896/chat_template.jinja | 89 + checkpoint-896/config.json | 63 + checkpoint-896/generation_config.json | 12 + checkpoint-896/model.safetensors | 3 + checkpoint-896/optimizer.pt | 3 + checkpoint-896/rng_state.pth | 3 + checkpoint-896/scheduler.pt | 3 + checkpoint-896/tokenizer.json | 3 + checkpoint-896/tokenizer_config.json | 30 + checkpoint-896/trainer_state.json | 924 +++ checkpoint-896/training_args.bin | 3 + config.json | 63 + data_manifest.json | 84 + generation_config.json | 12 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + training_args.bin | 3 + 98 files changed, 32335 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-1250/chat_template.jinja create mode 100644 checkpoint-1250/config.json create mode 100644 checkpoint-1250/generation_config.json create mode 100644 checkpoint-1250/model.safetensors create mode 100644 checkpoint-1250/optimizer.pt create mode 100644 checkpoint-1250/rng_state.pth create mode 100644 checkpoint-1250/scheduler.pt create mode 100644 checkpoint-1250/tokenizer.json create mode 100644 checkpoint-1250/tokenizer_config.json create mode 100644 checkpoint-1250/trainer_state.json create mode 100644 checkpoint-1250/training_args.bin create mode 100644 checkpoint-2375/chat_template.jinja create mode 100644 checkpoint-2375/config.json create mode 100644 checkpoint-2375/generation_config.json create mode 100644 checkpoint-2375/model.safetensors create mode 100644 checkpoint-2375/optimizer.pt create mode 100644 checkpoint-2375/rng_state.pth create mode 100644 checkpoint-2375/scheduler.pt create mode 100644 checkpoint-2375/tokenizer.json create mode 100644 checkpoint-2375/tokenizer_config.json create mode 100644 checkpoint-2375/trainer_state.json create mode 100644 checkpoint-2375/training_args.bin create mode 100644 checkpoint-2551/chat_template.jinja create mode 100644 checkpoint-2551/config.json create mode 100644 checkpoint-2551/generation_config.json create mode 100644 checkpoint-2551/model.safetensors create mode 100644 checkpoint-2551/optimizer.pt create mode 100644 checkpoint-2551/rng_state.pth create mode 100644 checkpoint-2551/scheduler.pt create mode 100644 checkpoint-2551/tokenizer.json create mode 100644 checkpoint-2551/tokenizer_config.json create mode 100644 checkpoint-2551/trainer_state.json create mode 100644 checkpoint-2551/training_args.bin create mode 100644 checkpoint-3125/chat_template.jinja create mode 100644 checkpoint-3125/config.json create mode 100644 checkpoint-3125/generation_config.json create mode 100644 checkpoint-3125/model.safetensors create mode 100644 checkpoint-3125/optimizer.pt create mode 100644 checkpoint-3125/rng_state.pth create mode 100644 checkpoint-3125/scheduler.pt create mode 100644 checkpoint-3125/tokenizer.json create mode 100644 checkpoint-3125/tokenizer_config.json create mode 100644 checkpoint-3125/trainer_state.json create mode 100644 checkpoint-3125/training_args.bin create mode 100644 checkpoint-4375/chat_template.jinja create mode 100644 checkpoint-4375/config.json create mode 100644 checkpoint-4375/generation_config.json create mode 100644 checkpoint-4375/model.safetensors create mode 100644 checkpoint-4375/optimizer.pt create mode 100644 checkpoint-4375/rng_state.pth create mode 100644 checkpoint-4375/scheduler.pt create mode 100644 checkpoint-4375/tokenizer.json create mode 100644 checkpoint-4375/tokenizer_config.json create mode 100644 checkpoint-4375/trainer_state.json create mode 100644 checkpoint-4375/training_args.bin create mode 100644 checkpoint-6241/chat_template.jinja create mode 100644 checkpoint-6241/config.json create mode 100644 checkpoint-6241/generation_config.json create mode 100644 checkpoint-6241/model.safetensors create mode 100644 checkpoint-6241/optimizer.pt create mode 100644 checkpoint-6241/rng_state.pth create mode 100644 checkpoint-6241/scheduler.pt create mode 100644 checkpoint-6241/tokenizer.json create mode 100644 checkpoint-6241/tokenizer_config.json create mode 100644 checkpoint-6241/trainer_state.json create mode 100644 checkpoint-6241/training_args.bin create mode 100644 checkpoint-7136/chat_template.jinja create mode 100644 checkpoint-7136/config.json create mode 100644 checkpoint-7136/generation_config.json create mode 100644 checkpoint-7136/model.safetensors create mode 100644 checkpoint-7136/optimizer.pt create mode 100644 checkpoint-7136/rng_state.pth create mode 100644 checkpoint-7136/scheduler.pt create mode 100644 checkpoint-7136/tokenizer.json create mode 100644 checkpoint-7136/tokenizer_config.json create mode 100644 checkpoint-7136/trainer_state.json create mode 100644 checkpoint-7136/training_args.bin create mode 100644 checkpoint-896/chat_template.jinja create mode 100644 checkpoint-896/config.json create mode 100644 checkpoint-896/generation_config.json create mode 100644 checkpoint-896/model.safetensors create mode 100644 checkpoint-896/optimizer.pt create mode 100644 checkpoint-896/rng_state.pth create mode 100644 checkpoint-896/scheduler.pt create mode 100644 checkpoint-896/tokenizer.json create mode 100644 checkpoint-896/tokenizer_config.json create mode 100644 checkpoint-896/trainer_state.json create mode 100644 checkpoint-896/training_args.bin create mode 100644 config.json create mode 100644 data_manifest.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..253152b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-2551/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..23d399e --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-1.7B +library_name: transformers +model_name: gk_thinking_v6_knowledge_base +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for gk_thinking_v6_knowledge_base + +This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 1.3.0 +- Transformers: 5.7.0 +- Pytorch: 2.10.0+cu128 +- Datasets: 4.8.5 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..a43db74 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,94 @@ +{%- set enable_thinking = true %} +{%- set default_system_message = "You are a knowledgeable assistant. For multiple-choice questions, reason step by step, then write only the letter of the correct answer inside \\boxed{}." %} +{%- if messages[0]['role'] != 'system' %} + {%- set messages = [{'role': 'system', 'content': default_system_message}] + messages %} +{%- endif %} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1250/chat_template.jinja b/checkpoint-1250/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-1250/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1250/config.json b/checkpoint-1250/config.json new file mode 100644 index 0000000..32a91b7 --- /dev/null +++ b/checkpoint-1250/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-1250/generation_config.json b/checkpoint-1250/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-1250/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-1250/model.safetensors b/checkpoint-1250/model.safetensors new file mode 100644 index 0000000..6d46a93 --- /dev/null +++ b/checkpoint-1250/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181262ae4ad047edf5c9426a2fd6b57ea981ec362767448f5811552336154680 +size 6882335328 diff --git a/checkpoint-1250/optimizer.pt b/checkpoint-1250/optimizer.pt new file mode 100644 index 0000000..4bb650e --- /dev/null +++ b/checkpoint-1250/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b12a71007dd725e387d6317522f008a161e3f9486adf46c6b985bfa0669beef +size 13764874811 diff --git a/checkpoint-1250/rng_state.pth b/checkpoint-1250/rng_state.pth new file mode 100644 index 0000000..d837423 --- /dev/null +++ b/checkpoint-1250/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8c1991954b8bd4c7ad3fd59b9380ccac8fd9f38770839efa9738b23b2c7143f +size 14645 diff --git a/checkpoint-1250/scheduler.pt b/checkpoint-1250/scheduler.pt new file mode 100644 index 0000000..9376d19 --- /dev/null +++ b/checkpoint-1250/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b36fb6f8aa4469ef93abf7025e853e21894673ca0b03036567aa61037282b5b1 +size 1465 diff --git a/checkpoint-1250/tokenizer.json b/checkpoint-1250/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-1250/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-1250/tokenizer_config.json b/checkpoint-1250/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/checkpoint-1250/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1250/trainer_state.json b/checkpoint-1250/trainer_state.json new file mode 100644 index 0000000..84b61d4 --- /dev/null +++ b/checkpoint-1250/trainer_state.json @@ -0,0 +1,3409 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 1250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 257.1, + "completions/max_terminated_length": 158.3, + "completions/mean_length": 42.925, + "completions/mean_terminated_length": 36.65708351135254, + "completions/min_length": 14.1, + "completions/min_terminated_length": 14.1, + "entropy": 0.7689311370253563, + "epoch": 0.008, + "frac_reward_zero_std": 0.35, + "grad_norm": 6.757325172424316, + "learning_rate": 4.964e-07, + "loss": -0.03516485691070557, + "num_tokens": 14928.0, + "reward": 0.7131250023841857, + "reward_std": 0.4031145960092545, + "rewards/reward_fn/mean": 0.7131250023841857, + "rewards/reward_fn/std": 0.40311461091041567, + "step": 10, + "step_time": 12.329081743443385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 124.0, + "completions/max_terminated_length": 124.0, + "completions/mean_length": 28.95, + "completions/mean_terminated_length": 28.95, + "completions/min_length": 15.3, + "completions/min_terminated_length": 15.3, + "entropy": 0.74221798684448, + "epoch": 0.016, + "frac_reward_zero_std": 0.4, + "grad_norm": 24.46213722229004, + "learning_rate": 4.923999999999999e-07, + "loss": -0.03677875101566315, + "num_tokens": 27908.0, + "reward": 0.8331249713897705, + "reward_std": 0.4601421281695366, + "rewards/reward_fn/mean": 0.8331249713897705, + "rewards/reward_fn/std": 0.4601421505212784, + "step": 20, + "step_time": 6.713664122438058 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 312.5, + "completions/max_terminated_length": 312.5, + "completions/mean_length": 45.70625, + "completions/mean_terminated_length": 45.70625, + "completions/min_length": 15.9, + "completions/min_terminated_length": 15.9, + "entropy": 0.7767099749296904, + "epoch": 0.024, + "frac_reward_zero_std": 0.475, + "grad_norm": 8.362913131713867, + "learning_rate": 4.884e-07, + "loss": -0.06059606671333313, + "num_tokens": 43689.0, + "reward": 0.8893749833106994, + "reward_std": 0.3755300402641296, + "rewards/reward_fn/mean": 0.8893749833106994, + "rewards/reward_fn/std": 0.3755300521850586, + "step": 30, + "step_time": 14.446745052048936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.3, + "completions/max_terminated_length": 133.3, + "completions/mean_length": 29.8375, + "completions/mean_terminated_length": 29.8375, + "completions/min_length": 16.8, + "completions/min_terminated_length": 16.8, + "entropy": 0.7361419148743152, + "epoch": 0.032, + "frac_reward_zero_std": 0.55, + "grad_norm": 12.163936614990234, + "learning_rate": 4.844e-07, + "loss": -0.03543267250061035, + "num_tokens": 56847.0, + "reward": 0.8531249761581421, + "reward_std": 0.35396517962217333, + "rewards/reward_fn/mean": 0.8531249761581421, + "rewards/reward_fn/std": 0.3539652034640312, + "step": 40, + "step_time": 7.049018428754062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 108.2, + "completions/max_terminated_length": 108.2, + "completions/mean_length": 27.10625, + "completions/mean_terminated_length": 27.10625, + "completions/min_length": 16.7, + "completions/min_terminated_length": 16.7, + "entropy": 0.5995874460786581, + "epoch": 0.04, + "frac_reward_zero_std": 0.525, + "grad_norm": 8.617820739746094, + "learning_rate": 4.804e-07, + "loss": 0.062236183881759645, + "num_tokens": 69272.0, + "reward": 0.9143749713897705, + "reward_std": 0.35480276346206663, + "rewards/reward_fn/mean": 0.9143749713897705, + "rewards/reward_fn/std": 0.35480278730392456, + "step": 50, + "step_time": 6.082800254039467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 53.2, + "completions/max_terminated_length": 53.2, + "completions/mean_length": 23.50625, + "completions/mean_terminated_length": 23.50625, + "completions/min_length": 17.5, + "completions/min_terminated_length": 17.5, + "entropy": 0.6239847084507346, + "epoch": 0.048, + "frac_reward_zero_std": 0.7, + "grad_norm": 8.013570785522461, + "learning_rate": 4.7639999999999995e-07, + "loss": -0.007780641317367554, + "num_tokens": 81493.0, + "reward": 0.8724999845027923, + "reward_std": 0.23893336951732635, + "rewards/reward_fn/mean": 0.8724999845027923, + "rewards/reward_fn/std": 0.23893338441848755, + "step": 60, + "step_time": 3.830377937294543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 47.5, + "completions/max_terminated_length": 47.5, + "completions/mean_length": 21.3125, + "completions/mean_terminated_length": 21.3125, + "completions/min_length": 17.1, + "completions/min_terminated_length": 17.1, + "entropy": 0.526786202006042, + "epoch": 0.056, + "frac_reward_zero_std": 0.8, + "grad_norm": 6.611401081085205, + "learning_rate": 4.7239999999999997e-07, + "loss": 0.0007052101194858551, + "num_tokens": 92903.0, + "reward": 0.954999977350235, + "reward_std": 0.30052519142627715, + "rewards/reward_fn/mean": 0.954999977350235, + "rewards/reward_fn/std": 0.30052521228790285, + "step": 70, + "step_time": 3.6160760662984104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 201.9, + "completions/max_terminated_length": 112.8, + "completions/mean_length": 34.225, + "completions/mean_terminated_length": 28.01541690826416, + "completions/min_length": 17.3, + "completions/min_terminated_length": 17.3, + "entropy": 0.6730542730540037, + "epoch": 0.064, + "frac_reward_zero_std": 0.725, + "grad_norm": 8.44511890411377, + "learning_rate": 4.684e-07, + "loss": 0.19833827018737793, + "num_tokens": 106795.0, + "reward": 0.8831249833106994, + "reward_std": 0.2491457238793373, + "rewards/reward_fn/mean": 0.8831249833106994, + "rewards/reward_fn/std": 0.24914574027061462, + "step": 80, + "step_time": 10.088793818978592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 171.4, + "completions/max_terminated_length": 171.4, + "completions/mean_length": 30.21875, + "completions/mean_terminated_length": 30.21875, + "completions/min_length": 17.7, + "completions/min_terminated_length": 17.7, + "entropy": 0.5667739075608551, + "epoch": 0.072, + "frac_reward_zero_std": 0.9, + "grad_norm": 0.0, + "learning_rate": 4.6439999999999995e-07, + "loss": 0.002179677784442902, + "num_tokens": 119930.0, + "reward": 0.9687499701976776, + "reward_std": 0.2984331727027893, + "rewards/reward_fn/mean": 0.9687499701976776, + "rewards/reward_fn/std": 0.2984331905841827, + "step": 90, + "step_time": 8.863318855362014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 74.8, + "completions/max_terminated_length": 74.8, + "completions/mean_length": 23.96875, + "completions/mean_terminated_length": 23.96875, + "completions/min_length": 17.8, + "completions/min_terminated_length": 17.8, + "entropy": 0.494637239864096, + "epoch": 0.08, + "frac_reward_zero_std": 0.925, + "grad_norm": 0.0, + "learning_rate": 4.6039999999999997e-07, + "loss": 0.001673000119626522, + "num_tokens": 131773.0, + "reward": 0.971874988079071, + "reward_std": 0.1934887498617172, + "rewards/reward_fn/mean": 0.971874988079071, + "rewards/reward_fn/std": 0.19348875880241395, + "step": 100, + "step_time": 4.718112504808232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 46.2, + "completions/max_terminated_length": 46.2, + "completions/mean_length": 21.71875, + "completions/mean_terminated_length": 21.71875, + "completions/min_length": 17.2, + "completions/min_terminated_length": 17.2, + "entropy": 0.44686332195997236, + "epoch": 0.088, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 4.5639999999999993e-07, + "loss": 0.007064198702573776, + "num_tokens": 143140.0, + "reward": 0.9249999701976777, + "reward_std": 0.2545803815126419, + "rewards/reward_fn/mean": 0.9249999701976777, + "rewards/reward_fn/std": 0.2545804023742676, + "step": 110, + "step_time": 3.6042728299740703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 25.2, + "completions/max_terminated_length": 25.2, + "completions/mean_length": 19.40625, + "completions/mean_terminated_length": 19.40625, + "completions/min_length": 17.1, + "completions/min_terminated_length": 17.1, + "entropy": 0.3366036908584647, + "epoch": 0.096, + "frac_reward_zero_std": 0.775, + "grad_norm": 13.464982986450195, + "learning_rate": 4.524e-07, + "loss": -0.0006624836474657059, + "num_tokens": 154305.0, + "reward": 1.0399999618530273, + "reward_std": 0.34495194256305695, + "rewards/reward_fn/mean": 1.0399999618530273, + "rewards/reward_fn/std": 0.34495197534561156, + "step": 120, + "step_time": 2.747153246589005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.3, + "completions/max_terminated_length": 29.3, + "completions/mean_length": 19.65625, + "completions/mean_terminated_length": 19.65625, + "completions/min_length": 17.2, + "completions/min_terminated_length": 17.2, + "entropy": 0.36671050266595556, + "epoch": 0.104, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 4.484e-07, + "loss": 0.017103588581085204, + "num_tokens": 165738.0, + "reward": 0.9574999690055848, + "reward_std": 0.3057817339897156, + "rewards/reward_fn/mean": 0.9574999690055848, + "rewards/reward_fn/std": 0.305781751871109, + "step": 130, + "step_time": 2.903817686345428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.8, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 19.84375, + "completions/mean_terminated_length": 19.84375, + "completions/min_length": 17.4, + "completions/min_terminated_length": 17.4, + "entropy": 0.35206709057092667, + "epoch": 0.112, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 4.444e-07, + "loss": 0.0023837201297283173, + "num_tokens": 177529.0, + "reward": 1.003124976158142, + "reward_std": 0.27038749754428865, + "rewards/reward_fn/mean": 1.003124976158142, + "rewards/reward_fn/std": 0.27038750648498533, + "step": 140, + "step_time": 2.9909598857630044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 33.7, + "completions/max_terminated_length": 33.7, + "completions/mean_length": 20.18125, + "completions/mean_terminated_length": 20.18125, + "completions/min_length": 17.4, + "completions/min_terminated_length": 17.4, + "entropy": 0.34009722527116537, + "epoch": 0.12, + "frac_reward_zero_std": 0.825, + "grad_norm": 4.034805774688721, + "learning_rate": 4.404e-07, + "loss": 0.005604463815689087, + "num_tokens": 188954.0, + "reward": 0.9499999761581421, + "reward_std": 0.25686181485652926, + "rewards/reward_fn/mean": 0.9499999761581421, + "rewards/reward_fn/std": 0.25686182677745817, + "step": 150, + "step_time": 3.0971988524775953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 64.1, + "completions/max_terminated_length": 64.1, + "completions/mean_length": 22.725, + "completions/mean_terminated_length": 22.725, + "completions/min_length": 17.1, + "completions/min_terminated_length": 17.1, + "entropy": 0.3341391346533783, + "epoch": 0.128, + "frac_reward_zero_std": 0.75, + "grad_norm": 3.904665946960449, + "learning_rate": 4.364e-07, + "loss": 0.0010127602145075798, + "num_tokens": 200770.0, + "reward": 1.0562499642372132, + "reward_std": 0.3720459073781967, + "rewards/reward_fn/mean": 1.0562499642372132, + "rewards/reward_fn/std": 0.37204593122005464, + "step": 160, + "step_time": 4.2844222981948406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 60.1, + "completions/max_terminated_length": 60.1, + "completions/mean_length": 21.50625, + "completions/mean_terminated_length": 21.50625, + "completions/min_length": 17.4, + "completions/min_terminated_length": 17.4, + "entropy": 0.3025246943347156, + "epoch": 0.136, + "frac_reward_zero_std": 0.8, + "grad_norm": 5.239154815673828, + "learning_rate": 4.324e-07, + "loss": -0.0667100191116333, + "num_tokens": 212767.0, + "reward": 0.9624999642372132, + "reward_std": 0.2936569094657898, + "rewards/reward_fn/mean": 0.9624999642372132, + "rewards/reward_fn/std": 0.2936569362878799, + "step": 170, + "step_time": 4.151584721263498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.7, + "completions/max_terminated_length": 189.7, + "completions/mean_length": 32.33125, + "completions/mean_terminated_length": 32.33125, + "completions/min_length": 17.6, + "completions/min_terminated_length": 17.6, + "entropy": 0.38091158121824265, + "epoch": 0.144, + "frac_reward_zero_std": 0.825, + "grad_norm": 8.872406005859375, + "learning_rate": 4.284e-07, + "loss": -0.008929825574159621, + "num_tokens": 225856.0, + "reward": 1.0249999642372132, + "reward_std": 0.3340115398168564, + "rewards/reward_fn/mean": 1.0249999642372132, + "rewards/reward_fn/std": 0.33401156663894654, + "step": 180, + "step_time": 9.366582131106407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.7, + "completions/max_terminated_length": 355.7, + "completions/mean_length": 68.725, + "completions/mean_terminated_length": 68.725, + "completions/min_length": 17.5, + "completions/min_terminated_length": 17.5, + "entropy": 0.5331769159063697, + "epoch": 0.152, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 4.2439999999999996e-07, + "loss": 0.05694311261177063, + "num_tokens": 245108.0, + "reward": 0.9968749701976776, + "reward_std": 0.30177369713783264, + "rewards/reward_fn/mean": 0.9968749701976776, + "rewards/reward_fn/std": 0.3017737179994583, + "step": 190, + "step_time": 16.25606108647771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.7, + "completions/max_terminated_length": 231.7, + "completions/mean_length": 39.475, + "completions/mean_terminated_length": 39.475, + "completions/min_length": 17.6, + "completions/min_terminated_length": 17.6, + "entropy": 0.32137000167858787, + "epoch": 0.16, + "frac_reward_zero_std": 0.675, + "grad_norm": 6.173330307006836, + "learning_rate": 4.204e-07, + "loss": -0.10696818828582763, + "num_tokens": 259512.0, + "reward": 1.012499952316284, + "reward_std": 0.3910213738679886, + "rewards/reward_fn/mean": 1.012499952316284, + "rewards/reward_fn/std": 0.39102140367031096, + "step": 200, + "step_time": 11.044270927784964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 314.0, + "completions/max_terminated_length": 314.0, + "completions/mean_length": 40.30625, + "completions/mean_terminated_length": 40.30625, + "completions/min_length": 17.1, + "completions/min_terminated_length": 17.1, + "entropy": 0.3382976199500263, + "epoch": 0.168, + "frac_reward_zero_std": 0.775, + "grad_norm": 12.670361518859863, + "learning_rate": 4.164e-07, + "loss": 0.05056280493736267, + "num_tokens": 274317.0, + "reward": 1.0124999582767487, + "reward_std": 0.37820068299770354, + "rewards/reward_fn/mean": 1.0124999582767487, + "rewards/reward_fn/std": 0.37820071876049044, + "step": 210, + "step_time": 14.444551136810333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 94.8, + "completions/max_terminated_length": 94.8, + "completions/mean_length": 24.9375, + "completions/mean_terminated_length": 24.9375, + "completions/min_length": 17.5, + "completions/min_terminated_length": 17.5, + "entropy": 0.2844417320564389, + "epoch": 0.176, + "frac_reward_zero_std": 0.75, + "grad_norm": 11.695199966430664, + "learning_rate": 4.1239999999999996e-07, + "loss": 0.012198887765407562, + "num_tokens": 286943.0, + "reward": 0.9874999761581421, + "reward_std": 0.2448488086462021, + "rewards/reward_fn/mean": 0.9874999761581421, + "rewards/reward_fn/std": 0.24484881460666658, + "step": 220, + "step_time": 5.559161439398304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 103.9, + "completions/max_terminated_length": 103.9, + "completions/mean_length": 29.1125, + "completions/mean_terminated_length": 29.1125, + "completions/min_length": 16.6, + "completions/min_terminated_length": 16.6, + "entropy": 0.31354843971785157, + "epoch": 0.184, + "frac_reward_zero_std": 0.775, + "grad_norm": 7.898475646972656, + "learning_rate": 4.084e-07, + "loss": -0.01266075372695923, + "num_tokens": 299829.0, + "reward": 0.9837499558925629, + "reward_std": 0.35773794949054716, + "rewards/reward_fn/mean": 0.9837499558925629, + "rewards/reward_fn/std": 0.3577379733324051, + "step": 230, + "step_time": 5.904518230678514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 70.4, + "completions/max_terminated_length": 70.4, + "completions/mean_length": 24.5625, + "completions/mean_terminated_length": 24.5625, + "completions/min_length": 17.6, + "completions/min_terminated_length": 17.6, + "entropy": 0.24243622907670215, + "epoch": 0.192, + "frac_reward_zero_std": 0.775, + "grad_norm": 0.0, + "learning_rate": 4.0439999999999994e-07, + "loss": -0.022087261080741882, + "num_tokens": 312139.0, + "reward": 1.081249964237213, + "reward_std": 0.3463846206665039, + "rewards/reward_fn/mean": 1.081249964237213, + "rewards/reward_fn/std": 0.3463846266269684, + "step": 240, + "step_time": 4.509387341840193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 85.0, + "completions/max_terminated_length": 85.0, + "completions/mean_length": 30.96875, + "completions/mean_terminated_length": 30.96875, + "completions/min_length": 17.7, + "completions/min_terminated_length": 17.7, + "entropy": 0.2872183081228286, + "epoch": 0.2, + "frac_reward_zero_std": 0.875, + "grad_norm": 0.0, + "learning_rate": 4.0039999999999996e-07, + "loss": 0.0037321031093597414, + "num_tokens": 325758.0, + "reward": 0.9562499642372131, + "reward_std": 0.31391805708408355, + "rewards/reward_fn/mean": 0.9562499642372131, + "rewards/reward_fn/std": 0.3139180839061737, + "step": 250, + "step_time": 5.160763737838716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 127.1, + "completions/max_terminated_length": 127.1, + "completions/mean_length": 29.21875, + "completions/mean_terminated_length": 29.21875, + "completions/min_length": 17.4, + "completions/min_terminated_length": 17.4, + "entropy": 0.23362355224089698, + "epoch": 0.208, + "frac_reward_zero_std": 0.85, + "grad_norm": 6.962454795837402, + "learning_rate": 3.964e-07, + "loss": 0.028215166926383973, + "num_tokens": 338481.0, + "reward": 1.0999999582767486, + "reward_std": 0.42081114947795867, + "rewards/reward_fn/mean": 1.0999999582767486, + "rewards/reward_fn/std": 0.4208111733198166, + "step": 260, + "step_time": 6.7953305871225895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.8, + "completions/max_terminated_length": 116.8, + "completions/mean_length": 30.49375, + "completions/mean_terminated_length": 30.49375, + "completions/min_length": 17.5, + "completions/min_terminated_length": 17.5, + "entropy": 0.2725491218268871, + "epoch": 0.216, + "frac_reward_zero_std": 0.725, + "grad_norm": 5.130438327789307, + "learning_rate": 3.924e-07, + "loss": -0.05285842418670654, + "num_tokens": 351340.0, + "reward": 0.9593749642372131, + "reward_std": 0.30626748204231263, + "rewards/reward_fn/mean": 0.9593749642372131, + "rewards/reward_fn/std": 0.3062675029039383, + "step": 270, + "step_time": 6.392075706832111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.2, + "completions/max_terminated_length": 231.2, + "completions/mean_length": 38.7625, + "completions/mean_terminated_length": 38.7625, + "completions/min_length": 17.2, + "completions/min_terminated_length": 17.2, + "entropy": 0.28721734539140015, + "epoch": 0.224, + "frac_reward_zero_std": 0.825, + "grad_norm": 0.0, + "learning_rate": 3.884e-07, + "loss": -0.011222265660762787, + "num_tokens": 365774.0, + "reward": 0.9218749761581421, + "reward_std": 0.25241841971874235, + "rewards/reward_fn/mean": 0.9218749761581421, + "rewards/reward_fn/std": 0.2524184435606003, + "step": 280, + "step_time": 11.114369830535725 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.9, + "completions/max_terminated_length": 255.9, + "completions/mean_length": 41.78125, + "completions/mean_terminated_length": 41.78125, + "completions/min_length": 17.4, + "completions/min_terminated_length": 17.4, + "entropy": 0.28078931191121226, + "epoch": 0.232, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 3.8440000000000003e-07, + "loss": -0.07034780383110047, + "num_tokens": 380923.0, + "reward": 0.9999999582767487, + "reward_std": 0.35158316493034364, + "rewards/reward_fn/mean": 0.9999999582767487, + "rewards/reward_fn/std": 0.3515831857919693, + "step": 290, + "step_time": 11.978949176566676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.5, + "completions/max_terminated_length": 331.5, + "completions/mean_length": 78.25625, + "completions/mean_terminated_length": 78.25625, + "completions/min_length": 17.4, + "completions/min_terminated_length": 17.4, + "entropy": 0.34983569611795245, + "epoch": 0.24, + "frac_reward_zero_std": 0.8, + "grad_norm": 6.399377822875977, + "learning_rate": 3.804e-07, + "loss": -0.0407560795545578, + "num_tokens": 401460.0, + "reward": 0.9374999701976776, + "reward_std": 0.25913873612880706, + "rewards/reward_fn/mean": 0.9374999701976776, + "rewards/reward_fn/std": 0.2591387540102005, + "step": 300, + "step_time": 15.129521024413407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 461.6, + "completions/max_terminated_length": 461.6, + "completions/mean_length": 151.96875, + "completions/mean_terminated_length": 151.96875, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "entropy": 0.568750799074769, + "epoch": 0.248, + "frac_reward_zero_std": 0.75, + "grad_norm": 1.8816224336624146, + "learning_rate": 3.764e-07, + "loss": 0.01563715487718582, + "num_tokens": 434015.0, + "reward": 0.9343749523162842, + "reward_std": 0.3420647859573364, + "rewards/reward_fn/mean": 0.9343749523162842, + "rewards/reward_fn/std": 0.3420648217201233, + "step": 310, + "step_time": 21.686344171082602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 465.6, + "completions/max_terminated_length": 465.6, + "completions/mean_length": 160.825, + "completions/mean_terminated_length": 160.825, + "completions/min_length": 17.6, + "completions/min_terminated_length": 17.6, + "entropy": 0.5512657368555665, + "epoch": 0.256, + "frac_reward_zero_std": 0.75, + "grad_norm": 1.9943861961364746, + "learning_rate": 3.7239999999999997e-07, + "loss": 0.02993807792663574, + "num_tokens": 468327.0, + "reward": 1.0312499523162841, + "reward_std": 0.4049929678440094, + "rewards/reward_fn/mean": 1.0312499523162841, + "rewards/reward_fn/std": 0.404993000626564, + "step": 320, + "step_time": 20.60081666558981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 409.4, + "completions/max_terminated_length": 409.4, + "completions/mean_length": 139.36875, + "completions/mean_terminated_length": 139.36875, + "completions/min_length": 17.9, + "completions/min_terminated_length": 17.9, + "entropy": 0.4664949773345143, + "epoch": 0.264, + "frac_reward_zero_std": 0.8, + "grad_norm": 3.3355984687805176, + "learning_rate": 3.684e-07, + "loss": -0.015160781145095826, + "num_tokens": 498934.0, + "reward": 1.1312499642372131, + "reward_std": 0.3833997189998627, + "rewards/reward_fn/mean": 1.1312499642372131, + "rewards/reward_fn/std": 0.38339973986148834, + "step": 330, + "step_time": 18.231197547214105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 521.8, + "completions/max_terminated_length": 471.9, + "completions/mean_length": 219.0875, + "completions/mean_terminated_length": 209.3812515258789, + "completions/min_length": 26.7, + "completions/min_terminated_length": 26.7, + "entropy": 0.5834914448671042, + "epoch": 0.272, + "frac_reward_zero_std": 0.625, + "grad_norm": 2.5507986545562744, + "learning_rate": 3.644e-07, + "loss": 0.030872175097465517, + "num_tokens": 541844.0, + "reward": 1.0062499642372131, + "reward_std": 0.35398963987827303, + "rewards/reward_fn/mean": 1.0062499642372131, + "rewards/reward_fn/std": 0.35398967415094373, + "step": 340, + "step_time": 23.158983804937453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.9, + "completions/max_terminated_length": 589.9, + "completions/mean_length": 213.15, + "completions/mean_terminated_length": 213.15, + "completions/min_length": 19.7, + "completions/min_terminated_length": 19.7, + "entropy": 0.5913283064961433, + "epoch": 0.28, + "frac_reward_zero_std": 0.725, + "grad_norm": 0.0, + "learning_rate": 3.6039999999999997e-07, + "loss": 0.01863671690225601, + "num_tokens": 584100.0, + "reward": 0.974999976158142, + "reward_std": 0.29154602289199827, + "rewards/reward_fn/mean": 0.974999976158142, + "rewards/reward_fn/std": 0.29154603481292723, + "step": 350, + "step_time": 25.847512384923174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 596.0, + "completions/max_terminated_length": 553.3, + "completions/mean_length": 166.5875, + "completions/mean_terminated_length": 161.59166717529297, + "completions/min_length": 18.1, + "completions/min_terminated_length": 18.1, + "entropy": 0.5305257711559535, + "epoch": 0.288, + "frac_reward_zero_std": 0.7, + "grad_norm": 1.6829164028167725, + "learning_rate": 3.564e-07, + "loss": 0.009494951367378235, + "num_tokens": 619034.0, + "reward": 0.9218749582767487, + "reward_std": 0.30490350127220156, + "rewards/reward_fn/mean": 0.9218749582767487, + "rewards/reward_fn/std": 0.30490352809429166, + "step": 360, + "step_time": 26.364935595309362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 623.8, + "completions/max_terminated_length": 538.2, + "completions/mean_length": 224.5, + "completions/mean_terminated_length": 214.15750122070312, + "completions/min_length": 51.2, + "completions/min_terminated_length": 51.2, + "entropy": 0.5600679079070687, + "epoch": 0.296, + "frac_reward_zero_std": 0.575, + "grad_norm": 2.4058868885040283, + "learning_rate": 3.5239999999999995e-07, + "loss": 0.059442996978759766, + "num_tokens": 662946.0, + "reward": 1.0337499618530273, + "reward_std": 0.3860698252916336, + "rewards/reward_fn/mean": 1.0337499618530273, + "rewards/reward_fn/std": 0.38606984317302706, + "step": 370, + "step_time": 27.650598523486405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 471.4, + "completions/max_terminated_length": 471.4, + "completions/mean_length": 211.99375, + "completions/mean_terminated_length": 211.99375, + "completions/min_length": 39.4, + "completions/min_terminated_length": 39.4, + "entropy": 0.523330201394856, + "epoch": 0.304, + "frac_reward_zero_std": 0.8, + "grad_norm": 1.523726463317871, + "learning_rate": 3.4839999999999997e-07, + "loss": -0.017889173328876497, + "num_tokens": 705285.0, + "reward": 0.9249999701976777, + "reward_std": 0.25987376272678375, + "rewards/reward_fn/mean": 0.9249999701976777, + "rewards/reward_fn/std": 0.2598737746477127, + "step": 380, + "step_time": 20.896892397897318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 584.2, + "completions/max_terminated_length": 520.2, + "completions/mean_length": 251.1375, + "completions/mean_terminated_length": 246.00750122070312, + "completions/min_length": 62.2, + "completions/min_terminated_length": 62.2, + "entropy": 0.5894505500793457, + "epoch": 0.312, + "frac_reward_zero_std": 0.575, + "grad_norm": 2.648047685623169, + "learning_rate": 3.444e-07, + "loss": 0.056687426567077634, + "num_tokens": 753319.0, + "reward": 1.0999999523162842, + "reward_std": 0.4478457897901535, + "rewards/reward_fn/mean": 1.0999999523162842, + "rewards/reward_fn/std": 0.4478458106517792, + "step": 390, + "step_time": 25.831602280260995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.9, + "completions/max_terminated_length": 555.9, + "completions/mean_length": 243.40625, + "completions/mean_terminated_length": 243.40625, + "completions/min_length": 40.4, + "completions/min_terminated_length": 40.4, + "entropy": 0.5201437229756266, + "epoch": 0.32, + "frac_reward_zero_std": 0.675, + "grad_norm": 0.0, + "learning_rate": 3.4039999999999995e-07, + "loss": 0.021798035502433775, + "num_tokens": 800348.0, + "reward": 1.0062499582767486, + "reward_std": 0.37394005358219146, + "rewards/reward_fn/mean": 1.0062499582767486, + "rewards/reward_fn/std": 0.3739400714635849, + "step": 400, + "step_time": 24.329527226556092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 613.3, + "completions/max_terminated_length": 534.9, + "completions/mean_length": 244.41875, + "completions/mean_terminated_length": 234.8495864868164, + "completions/min_length": 35.6, + "completions/min_terminated_length": 35.6, + "entropy": 0.5379184504970909, + "epoch": 0.328, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.274796962738037, + "learning_rate": 3.3639999999999997e-07, + "loss": 0.06314390301704406, + "num_tokens": 847859.0, + "reward": 0.9687499582767487, + "reward_std": 0.35331138372421267, + "rewards/reward_fn/mean": 0.9687499582767487, + "rewards/reward_fn/std": 0.35331140756607055, + "step": 410, + "step_time": 27.114492582622916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 594.1, + "completions/max_terminated_length": 562.2, + "completions/mean_length": 268.15, + "completions/mean_terminated_length": 264.0933349609375, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "entropy": 0.5545548873022199, + "epoch": 0.336, + "frac_reward_zero_std": 0.625, + "grad_norm": 3.0214762687683105, + "learning_rate": 3.3239999999999993e-07, + "loss": 0.0016927286982536317, + "num_tokens": 899211.0, + "reward": 1.0531249523162842, + "reward_std": 0.4457359969615936, + "rewards/reward_fn/mean": 1.0531249523162842, + "rewards/reward_fn/std": 0.44573602378368377, + "step": 420, + "step_time": 26.21310900649987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 566.0, + "completions/max_terminated_length": 566.0, + "completions/mean_length": 237.33125, + "completions/mean_terminated_length": 237.33125, + "completions/min_length": 43.4, + "completions/min_terminated_length": 43.4, + "entropy": 0.4608824389986694, + "epoch": 0.344, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.376124858856201, + "learning_rate": 3.284e-07, + "loss": 0.002047058567404747, + "num_tokens": 945568.0, + "reward": 1.2124999523162843, + "reward_std": 0.4772857129573822, + "rewards/reward_fn/mean": 1.2124999523162843, + "rewards/reward_fn/std": 0.47728572487831117, + "step": 430, + "step_time": 24.91375301098451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 620.7, + "completions/max_terminated_length": 519.3, + "completions/mean_length": 221.89375, + "completions/mean_terminated_length": 212.0312515258789, + "completions/min_length": 18.3, + "completions/min_terminated_length": 18.3, + "entropy": 0.45621285401284695, + "epoch": 0.352, + "frac_reward_zero_std": 0.725, + "grad_norm": 0.0, + "learning_rate": 3.244e-07, + "loss": 0.03366280496120453, + "num_tokens": 989283.0, + "reward": 1.0374999582767486, + "reward_std": 0.37986487746238706, + "rewards/reward_fn/mean": 1.0374999582767486, + "rewards/reward_fn/std": 0.37986490726470945, + "step": 440, + "step_time": 27.54319058242254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 526.3, + "completions/max_terminated_length": 470.2, + "completions/mean_length": 237.4625, + "completions/mean_terminated_length": 232.41791839599608, + "completions/min_length": 55.9, + "completions/min_terminated_length": 55.9, + "entropy": 0.4685343712568283, + "epoch": 0.36, + "frac_reward_zero_std": 0.8, + "grad_norm": 2.073087692260742, + "learning_rate": 3.204e-07, + "loss": 0.008559707552194595, + "num_tokens": 1035581.0, + "reward": 1.0281249642372132, + "reward_std": 0.3265856146812439, + "rewards/reward_fn/mean": 1.0281249642372132, + "rewards/reward_fn/std": 0.32658563554286957, + "step": 450, + "step_time": 23.314190701860934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 491.8, + "completions/max_terminated_length": 491.8, + "completions/mean_length": 263.15, + "completions/mean_terminated_length": 263.15, + "completions/min_length": 88.6, + "completions/min_terminated_length": 88.6, + "entropy": 0.5305897884070874, + "epoch": 0.368, + "frac_reward_zero_std": 0.675, + "grad_norm": 1.9565300941467285, + "learning_rate": 3.164e-07, + "loss": 0.028019136190414427, + "num_tokens": 1086409.0, + "reward": 1.093749964237213, + "reward_std": 0.3530050367116928, + "rewards/reward_fn/mean": 1.093749964237213, + "rewards/reward_fn/std": 0.35300505757331846, + "step": 460, + "step_time": 21.788672981457786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 585.6, + "completions/max_terminated_length": 549.8, + "completions/mean_length": 269.5, + "completions/mean_terminated_length": 264.71583557128906, + "completions/min_length": 81.6, + "completions/min_terminated_length": 81.6, + "entropy": 0.5114285530522465, + "epoch": 0.376, + "frac_reward_zero_std": 0.525, + "grad_norm": 2.9088966846466064, + "learning_rate": 3.124e-07, + "loss": 0.05687993168830872, + "num_tokens": 1137953.0, + "reward": 1.1312499582767486, + "reward_std": 0.42567238211631775, + "rewards/reward_fn/mean": 1.1312499582767486, + "rewards/reward_fn/std": 0.4256723940372467, + "step": 470, + "step_time": 25.771493053948507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 545.3, + "completions/max_terminated_length": 545.3, + "completions/mean_length": 252.63125, + "completions/mean_terminated_length": 252.63125, + "completions/min_length": 44.2, + "completions/min_terminated_length": 44.2, + "entropy": 0.5523815616965294, + "epoch": 0.384, + "frac_reward_zero_std": 0.775, + "grad_norm": 0.0, + "learning_rate": 3.084e-07, + "loss": 0.023237675428390503, + "num_tokens": 1186658.0, + "reward": 1.0437499582767487, + "reward_std": 0.38032626211643217, + "rewards/reward_fn/mean": 1.0437499582767487, + "rewards/reward_fn/std": 0.3803262859582901, + "step": 480, + "step_time": 23.983928591478616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 523.6, + "completions/max_terminated_length": 523.6, + "completions/mean_length": 243.5875, + "completions/mean_terminated_length": 243.5875, + "completions/min_length": 57.4, + "completions/min_terminated_length": 57.4, + "entropy": 0.5340582400560379, + "epoch": 0.392, + "frac_reward_zero_std": 0.75, + "grad_norm": 5.369383335113525, + "learning_rate": 3.044e-07, + "loss": 0.031129142642021178, + "num_tokens": 1234168.0, + "reward": 1.1312499701976777, + "reward_std": 0.2945299968123436, + "rewards/reward_fn/mean": 1.1312499701976777, + "rewards/reward_fn/std": 0.29453000277280805, + "step": 490, + "step_time": 23.040811748150738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 229.5125, + "completions/mean_terminated_length": 229.5125, + "completions/min_length": 25.8, + "completions/min_terminated_length": 25.8, + "entropy": 0.5556849464774132, + "epoch": 0.4, + "frac_reward_zero_std": 0.825, + "grad_norm": 0.0, + "learning_rate": 3.0039999999999996e-07, + "loss": 0.012525177001953125, + "num_tokens": 1279254.0, + "reward": 1.0749999642372132, + "reward_std": 0.32434508502483367, + "rewards/reward_fn/mean": 1.0749999642372132, + "rewards/reward_fn/std": 0.3243451029062271, + "step": 500, + "step_time": 24.467384714726357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 610.0, + "completions/max_terminated_length": 546.3, + "completions/mean_length": 225.625, + "completions/mean_terminated_length": 220.79625091552734, + "completions/min_length": 41.8, + "completions/min_terminated_length": 41.8, + "entropy": 0.5509622530080378, + "epoch": 0.408, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 2.964e-07, + "loss": 0.06597378849983215, + "num_tokens": 1323630.0, + "reward": 0.9843749582767487, + "reward_std": 0.35697369575500487, + "rewards/reward_fn/mean": 0.9843749582767487, + "rewards/reward_fn/std": 0.3569737136363983, + "step": 510, + "step_time": 26.97945318124257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 546.9, + "completions/max_terminated_length": 496.5, + "completions/mean_length": 213.04375, + "completions/mean_terminated_length": 208.10416717529296, + "completions/min_length": 28.9, + "completions/min_terminated_length": 28.9, + "entropy": 0.5530080372467637, + "epoch": 0.416, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 2.924e-07, + "loss": 0.020942620933055878, + "num_tokens": 1365517.0, + "reward": 0.9343749701976776, + "reward_std": 0.27816103398799896, + "rewards/reward_fn/mean": 0.9343749701976776, + "rewards/reward_fn/std": 0.2781610548496246, + "step": 520, + "step_time": 24.255431303568184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 568.6, + "completions/max_terminated_length": 523.8, + "completions/mean_length": 182.9625, + "completions/mean_terminated_length": 173.29708557128907, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "entropy": 0.5604467433411628, + "epoch": 0.424, + "frac_reward_zero_std": 0.75, + "grad_norm": 5.100892066955566, + "learning_rate": 2.8839999999999996e-07, + "loss": 0.050272423028945926, + "num_tokens": 1402911.0, + "reward": 0.8937499821186066, + "reward_std": 0.20596824288368226, + "rewards/reward_fn/mean": 0.8937499821186066, + "rewards/reward_fn/std": 0.2059682548046112, + "step": 530, + "step_time": 25.345189223485068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 388.3, + "completions/max_terminated_length": 388.3, + "completions/mean_length": 162.04375, + "completions/mean_terminated_length": 162.04375, + "completions/min_length": 36.9, + "completions/min_terminated_length": 36.9, + "entropy": 0.5442286553792656, + "epoch": 0.432, + "frac_reward_zero_std": 0.825, + "grad_norm": 4.153268814086914, + "learning_rate": 2.844e-07, + "loss": -0.00437091588973999, + "num_tokens": 1437314.0, + "reward": 1.093749964237213, + "reward_std": 0.3850394904613495, + "rewards/reward_fn/mean": 1.093749964237213, + "rewards/reward_fn/std": 0.3850395202636719, + "step": 540, + "step_time": 17.517217593453825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.8, + "completions/max_terminated_length": 478.8, + "completions/mean_length": 183.76875, + "completions/mean_terminated_length": 183.76875, + "completions/min_length": 18.3, + "completions/min_terminated_length": 18.3, + "entropy": 0.5666915670037269, + "epoch": 0.44, + "frac_reward_zero_std": 0.8, + "grad_norm": 3.4264976978302, + "learning_rate": 2.804e-07, + "loss": 0.027844130992889404, + "num_tokens": 1475125.0, + "reward": 0.9687499701976776, + "reward_std": 0.2758553087711334, + "rewards/reward_fn/mean": 0.9687499701976776, + "rewards/reward_fn/std": 0.27585532069206237, + "step": 550, + "step_time": 21.227726350305602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.8, + "completions/max_terminated_length": 416.8, + "completions/mean_length": 147.075, + "completions/mean_terminated_length": 147.075, + "completions/min_length": 17.8, + "completions/min_terminated_length": 17.8, + "entropy": 0.49057656023651364, + "epoch": 0.448, + "frac_reward_zero_std": 0.85, + "grad_norm": 3.256361722946167, + "learning_rate": 2.7639999999999996e-07, + "loss": 0.02265160083770752, + "num_tokens": 1506917.0, + "reward": 0.9812499761581421, + "reward_std": 0.2361401915550232, + "rewards/reward_fn/mean": 0.9812499761581421, + "rewards/reward_fn/std": 0.2361402153968811, + "step": 560, + "step_time": 18.685880808066578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 384.4, + "completions/max_terminated_length": 384.4, + "completions/mean_length": 160.1875, + "completions/mean_terminated_length": 160.1875, + "completions/min_length": 18.3, + "completions/min_terminated_length": 18.3, + "entropy": 0.5117679107002914, + "epoch": 0.456, + "frac_reward_zero_std": 0.8, + "grad_norm": 2.30198073387146, + "learning_rate": 2.724e-07, + "loss": -0.01834808886051178, + "num_tokens": 1540791.0, + "reward": 1.0062499523162842, + "reward_std": 0.3968144774436951, + "rewards/reward_fn/mean": 1.0062499523162842, + "rewards/reward_fn/std": 0.39681450724601747, + "step": 570, + "step_time": 17.398948775697498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 472.1, + "completions/max_terminated_length": 472.1, + "completions/mean_length": 161.95, + "completions/mean_terminated_length": 161.95, + "completions/min_length": 18.1, + "completions/min_terminated_length": 18.1, + "entropy": 0.47344469791278243, + "epoch": 0.464, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.3436312675476074, + "learning_rate": 2.684e-07, + "loss": -0.026393899321556093, + "num_tokens": 1575619.0, + "reward": 0.9874999761581421, + "reward_std": 0.2700622081756592, + "rewards/reward_fn/mean": 0.9874999761581421, + "rewards/reward_fn/std": 0.27006221413612364, + "step": 580, + "step_time": 20.999267899850384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 601.7, + "completions/max_terminated_length": 546.1, + "completions/mean_length": 233.43125, + "completions/mean_terminated_length": 228.44708404541015, + "completions/min_length": 47.7, + "completions/min_terminated_length": 47.7, + "entropy": 0.5758602514863014, + "epoch": 0.472, + "frac_reward_zero_std": 0.65, + "grad_norm": 2.945976734161377, + "learning_rate": 2.644e-07, + "loss": 0.0019540391862392426, + "num_tokens": 1621388.0, + "reward": 1.0218749582767486, + "reward_std": 0.3570388913154602, + "rewards/reward_fn/mean": 1.0218749582767486, + "rewards/reward_fn/std": 0.3570389151573181, + "step": 590, + "step_time": 26.790125040663405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.8, + "completions/max_terminated_length": 509.8, + "completions/mean_length": 223.8, + "completions/mean_terminated_length": 223.8, + "completions/min_length": 50.6, + "completions/min_terminated_length": 50.6, + "entropy": 0.5903411463834345, + "epoch": 0.48, + "frac_reward_zero_std": 0.775, + "grad_norm": 2.2895243167877197, + "learning_rate": 2.6040000000000003e-07, + "loss": 0.01556304395198822, + "num_tokens": 1665464.0, + "reward": 1.031249964237213, + "reward_std": 0.3282184362411499, + "rewards/reward_fn/mean": 1.031249964237213, + "rewards/reward_fn/std": 0.32821846306324004, + "step": 600, + "step_time": 22.54900577166118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 446.0, + "completions/max_terminated_length": 446.0, + "completions/mean_length": 232.44375, + "completions/mean_terminated_length": 232.44375, + "completions/min_length": 68.2, + "completions/min_terminated_length": 68.2, + "entropy": 0.6003101659938693, + "epoch": 0.488, + "frac_reward_zero_std": 0.65, + "grad_norm": 2.4930214881896973, + "learning_rate": 2.564e-07, + "loss": 0.044041958451271054, + "num_tokens": 1711303.0, + "reward": 1.0312499523162841, + "reward_std": 0.3602029472589493, + "rewards/reward_fn/mean": 1.0312499523162841, + "rewards/reward_fn/std": 0.3602029770612717, + "step": 610, + "step_time": 19.94395455373451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.6, + "completions/max_terminated_length": 619.6, + "completions/mean_length": 262.50625, + "completions/mean_terminated_length": 262.50625, + "completions/min_length": 36.9, + "completions/min_terminated_length": 36.9, + "entropy": 0.625312153622508, + "epoch": 0.496, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.5237791538238525, + "learning_rate": 2.524e-07, + "loss": -0.01255713552236557, + "num_tokens": 1761532.0, + "reward": 0.9999999582767487, + "reward_std": 0.3505753219127655, + "rewards/reward_fn/mean": 0.9999999582767487, + "rewards/reward_fn/std": 0.3505753517150879, + "step": 620, + "step_time": 27.551811824087054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.6, + "completions/max_terminated_length": 514.6, + "completions/mean_length": 243.85, + "completions/mean_terminated_length": 243.85, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "entropy": 0.5866754692047834, + "epoch": 0.504, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.4999687671661377, + "learning_rate": 2.484e-07, + "loss": 0.015758931636810303, + "num_tokens": 1808952.0, + "reward": 1.0937499582767487, + "reward_std": 0.39699949622154235, + "rewards/reward_fn/mean": 1.0937499582767487, + "rewards/reward_fn/std": 0.39699951112270354, + "step": 630, + "step_time": 22.812671538256108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 531.3, + "completions/max_terminated_length": 485.5, + "completions/mean_length": 242.3, + "completions/mean_terminated_length": 238.0504180908203, + "completions/min_length": 64.9, + "completions/min_terminated_length": 64.9, + "entropy": 0.6033073195256293, + "epoch": 0.512, + "frac_reward_zero_std": 0.8, + "grad_norm": 0.0, + "learning_rate": 2.444e-07, + "loss": 0.010235734283924103, + "num_tokens": 1856144.0, + "reward": 0.9718749761581421, + "reward_std": 0.26936398148536683, + "rewards/reward_fn/mean": 0.9718749761581421, + "rewards/reward_fn/std": 0.2693639874458313, + "step": 640, + "step_time": 23.65593868405558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 539.9, + "completions/max_terminated_length": 539.9, + "completions/mean_length": 242.63125, + "completions/mean_terminated_length": 242.63125, + "completions/min_length": 84.8, + "completions/min_terminated_length": 84.8, + "entropy": 0.5526138002052903, + "epoch": 0.52, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.245793342590332, + "learning_rate": 2.404e-07, + "loss": -0.002822137624025345, + "num_tokens": 1903209.0, + "reward": 1.0781249523162841, + "reward_std": 0.43106013536453247, + "rewards/reward_fn/mean": 1.0781249523162841, + "rewards/reward_fn/std": 0.4310601681470871, + "step": 650, + "step_time": 24.121668337099255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 592.5, + "completions/max_terminated_length": 592.5, + "completions/mean_length": 243.65, + "completions/mean_terminated_length": 243.65, + "completions/min_length": 63.2, + "completions/min_terminated_length": 63.2, + "entropy": 0.557934966403991, + "epoch": 0.528, + "frac_reward_zero_std": 0.8, + "grad_norm": 2.7416722774505615, + "learning_rate": 2.364e-07, + "loss": 0.009367964416742324, + "num_tokens": 1950505.0, + "reward": 0.9874999642372131, + "reward_std": 0.3327379524707794, + "rewards/reward_fn/mean": 0.9874999642372131, + "rewards/reward_fn/std": 0.33273797333240507, + "step": 660, + "step_time": 26.20483476021327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 606.8, + "completions/max_terminated_length": 487.6, + "completions/mean_length": 262.125, + "completions/mean_terminated_length": 252.27166748046875, + "completions/min_length": 99.2, + "completions/min_terminated_length": 99.2, + "entropy": 0.5527632829733193, + "epoch": 0.536, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 2.324e-07, + "loss": 0.03527545630931854, + "num_tokens": 2000653.0, + "reward": 1.0562499582767486, + "reward_std": 0.3691807985305786, + "rewards/reward_fn/mean": 1.0562499582767486, + "rewards/reward_fn/std": 0.3691808253526688, + "step": 670, + "step_time": 27.034338617976754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.8, + "completions/max_terminated_length": 584.8, + "completions/mean_length": 285.025, + "completions/mean_terminated_length": 285.025, + "completions/min_length": 126.6, + "completions/min_terminated_length": 126.6, + "entropy": 0.5860134104266763, + "epoch": 0.544, + "frac_reward_zero_std": 0.775, + "grad_norm": 0.0, + "learning_rate": 2.2839999999999998e-07, + "loss": 0.01230028122663498, + "num_tokens": 2054357.0, + "reward": 1.0124999582767487, + "reward_std": 0.37343316674232485, + "rewards/reward_fn/mean": 1.0124999582767487, + "rewards/reward_fn/std": 0.3734331876039505, + "step": 680, + "step_time": 25.73272733730264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 535.7, + "completions/max_terminated_length": 535.7, + "completions/mean_length": 258.66875, + "completions/mean_terminated_length": 258.66875, + "completions/min_length": 102.7, + "completions/min_terminated_length": 102.7, + "entropy": 0.5561068987473845, + "epoch": 0.552, + "frac_reward_zero_std": 0.725, + "grad_norm": 2.3947577476501465, + "learning_rate": 2.2439999999999997e-07, + "loss": -0.0206025630235672, + "num_tokens": 2104000.0, + "reward": 1.0562499582767486, + "reward_std": 0.40473316311836244, + "rewards/reward_fn/mean": 1.0562499582767486, + "rewards/reward_fn/std": 0.4047331750392914, + "step": 690, + "step_time": 23.7531999821309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 543.3, + "completions/max_terminated_length": 508.4, + "completions/mean_length": 255.71875, + "completions/mean_terminated_length": 251.499169921875, + "completions/min_length": 79.4, + "completions/min_terminated_length": 79.4, + "entropy": 0.5383795527275652, + "epoch": 0.56, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.0125722885131836, + "learning_rate": 2.2040000000000001e-07, + "loss": 0.030927711725234987, + "num_tokens": 2153891.0, + "reward": 0.9687499642372132, + "reward_std": 0.31985312402248384, + "rewards/reward_fn/mean": 0.9687499642372132, + "rewards/reward_fn/std": 0.31985313892364503, + "step": 700, + "step_time": 24.29951238576323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 624.0, + "completions/max_terminated_length": 602.9, + "completions/mean_length": 278.2875, + "completions/mean_terminated_length": 274.3729187011719, + "completions/min_length": 91.9, + "completions/min_terminated_length": 91.9, + "entropy": 0.527317856438458, + "epoch": 0.568, + "frac_reward_zero_std": 0.625, + "grad_norm": 1.9104549884796143, + "learning_rate": 2.164e-07, + "loss": 0.03166455924510956, + "num_tokens": 2206773.0, + "reward": 1.090624952316284, + "reward_std": 0.4429534524679184, + "rewards/reward_fn/mean": 1.090624952316284, + "rewards/reward_fn/std": 0.4429534673690796, + "step": 710, + "step_time": 27.915992458211257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 548.3, + "completions/max_terminated_length": 527.2, + "completions/mean_length": 273.175, + "completions/mean_terminated_length": 269.2104187011719, + "completions/min_length": 99.8, + "completions/min_terminated_length": 99.8, + "entropy": 0.5130317708477378, + "epoch": 0.576, + "frac_reward_zero_std": 0.7, + "grad_norm": 2.303006410598755, + "learning_rate": 2.124e-07, + "loss": 0.011933594197034835, + "num_tokens": 2258889.0, + "reward": 1.0312499582767487, + "reward_std": 0.39556107819080355, + "rewards/reward_fn/mean": 1.0312499582767487, + "rewards/reward_fn/std": 0.39556109607219697, + "step": 720, + "step_time": 24.53683318160474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 579.1, + "completions/max_terminated_length": 536.6, + "completions/mean_length": 264.69375, + "completions/mean_terminated_length": 259.7870849609375, + "completions/min_length": 95.5, + "completions/min_terminated_length": 95.5, + "entropy": 0.5283560438081623, + "epoch": 0.584, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 2.0839999999999999e-07, + "loss": 0.049981740117073056, + "num_tokens": 2309684.0, + "reward": 1.0218749642372131, + "reward_std": 0.34613644182682035, + "rewards/reward_fn/mean": 1.0218749642372131, + "rewards/reward_fn/std": 0.34613647162914274, + "step": 730, + "step_time": 25.811987762106583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 547.0, + "completions/max_terminated_length": 547.0, + "completions/mean_length": 241.6, + "completions/mean_terminated_length": 241.6, + "completions/min_length": 64.1, + "completions/min_terminated_length": 64.1, + "entropy": 0.49383773263543845, + "epoch": 0.592, + "frac_reward_zero_std": 0.7, + "grad_norm": 2.562471628189087, + "learning_rate": 2.0439999999999998e-07, + "loss": 0.028516930341720582, + "num_tokens": 2356300.0, + "reward": 1.0249999701976775, + "reward_std": 0.3171436250209808, + "rewards/reward_fn/mean": 1.0249999701976775, + "rewards/reward_fn/std": 0.317143639922142, + "step": 740, + "step_time": 24.14920071642846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 462.0, + "completions/max_terminated_length": 462.0, + "completions/mean_length": 239.65625, + "completions/mean_terminated_length": 239.65625, + "completions/min_length": 74.7, + "completions/min_terminated_length": 74.7, + "entropy": 0.5035146079957485, + "epoch": 0.6, + "frac_reward_zero_std": 0.775, + "grad_norm": 0.0, + "learning_rate": 2.004e-07, + "loss": -0.014082185924053192, + "num_tokens": 2402669.0, + "reward": 0.9781249642372132, + "reward_std": 0.34400319755077363, + "rewards/reward_fn/mean": 0.9781249642372132, + "rewards/reward_fn/std": 0.3440032213926315, + "step": 750, + "step_time": 20.69694092241116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 517.6, + "completions/max_terminated_length": 517.6, + "completions/mean_length": 266.775, + "completions/mean_terminated_length": 266.775, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "entropy": 0.5417108541354537, + "epoch": 0.608, + "frac_reward_zero_std": 0.75, + "grad_norm": 1.9890964031219482, + "learning_rate": 1.9639999999999999e-07, + "loss": -0.026402422785758974, + "num_tokens": 2453809.0, + "reward": 0.9462499558925629, + "reward_std": 0.3611013382673264, + "rewards/reward_fn/mean": 0.9462499558925629, + "rewards/reward_fn/std": 0.361101371049881, + "step": 760, + "step_time": 22.986824012873694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 590.4, + "completions/max_terminated_length": 532.4, + "completions/mean_length": 252.18125, + "completions/mean_terminated_length": 247.59083557128906, + "completions/min_length": 101.1, + "completions/min_terminated_length": 101.1, + "entropy": 0.4916321000084281, + "epoch": 0.616, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 1.9239999999999998e-07, + "loss": 0.030572971701622008, + "num_tokens": 2502362.0, + "reward": 1.0468749582767487, + "reward_std": 0.415197890996933, + "rewards/reward_fn/mean": 1.0468749582767487, + "rewards/reward_fn/std": 0.41519791185855864, + "step": 770, + "step_time": 26.27881493680179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 477.3, + "completions/max_terminated_length": 477.3, + "completions/mean_length": 238.68125, + "completions/mean_terminated_length": 238.68125, + "completions/min_length": 94.1, + "completions/min_terminated_length": 94.1, + "entropy": 0.518075543269515, + "epoch": 0.624, + "frac_reward_zero_std": 0.775, + "grad_norm": 0.0, + "learning_rate": 1.884e-07, + "loss": -0.016722193360328673, + "num_tokens": 2548967.0, + "reward": 1.0437499642372132, + "reward_std": 0.35573128461837766, + "rewards/reward_fn/mean": 1.0437499642372132, + "rewards/reward_fn/std": 0.3557313114404678, + "step": 780, + "step_time": 21.362713638367133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 519.0, + "completions/max_terminated_length": 445.2, + "completions/mean_length": 248.11875, + "completions/mean_terminated_length": 239.45292053222656, + "completions/min_length": 66.9, + "completions/min_terminated_length": 66.9, + "entropy": 0.5092946726828813, + "epoch": 0.632, + "frac_reward_zero_std": 0.7, + "grad_norm": 2.420226573944092, + "learning_rate": 1.844e-07, + "loss": 0.04115874171257019, + "num_tokens": 2596974.0, + "reward": 1.0187499582767487, + "reward_std": 0.3817029446363449, + "rewards/reward_fn/mean": 1.0187499582767487, + "rewards/reward_fn/std": 0.3817029595375061, + "step": 790, + "step_time": 23.515300380950794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.6, + "completions/max_terminated_length": 524.6, + "completions/mean_length": 266.76875, + "completions/mean_terminated_length": 266.76875, + "completions/min_length": 84.8, + "completions/min_terminated_length": 84.8, + "entropy": 0.5524313434958458, + "epoch": 0.64, + "frac_reward_zero_std": 0.8, + "grad_norm": 2.9757630825042725, + "learning_rate": 1.804e-07, + "loss": -0.0031020037829875948, + "num_tokens": 2648317.0, + "reward": 1.0437499642372132, + "reward_std": 0.3630165934562683, + "rewards/reward_fn/mean": 1.0437499642372132, + "rewards/reward_fn/std": 0.3630166083574295, + "step": 800, + "step_time": 23.433115491084756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 594.1, + "completions/max_terminated_length": 556.5, + "completions/mean_length": 261.55, + "completions/mean_terminated_length": 257.0808349609375, + "completions/min_length": 94.1, + "completions/min_terminated_length": 94.1, + "entropy": 0.5408615570515394, + "epoch": 0.648, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 1.764e-07, + "loss": 0.005869099497795105, + "num_tokens": 2698233.0, + "reward": 1.0718749642372132, + "reward_std": 0.3241104021668434, + "rewards/reward_fn/mean": 1.0718749642372132, + "rewards/reward_fn/std": 0.32411042004823687, + "step": 810, + "step_time": 26.462639589840546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 514.8, + "completions/max_terminated_length": 464.8, + "completions/mean_length": 250.34375, + "completions/mean_terminated_length": 245.92958374023436, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.5274556184187531, + "epoch": 0.656, + "frac_reward_zero_std": 0.7, + "grad_norm": 3.2795684337615967, + "learning_rate": 1.7239999999999998e-07, + "loss": 0.03866562247276306, + "num_tokens": 2746656.0, + "reward": 1.015624964237213, + "reward_std": 0.36523938179016113, + "rewards/reward_fn/mean": 1.015624964237213, + "rewards/reward_fn/std": 0.3652394026517868, + "step": 820, + "step_time": 23.215507409302518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 604.7, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 284.25, + "completions/mean_terminated_length": 279.5695861816406, + "completions/min_length": 98.3, + "completions/min_terminated_length": 98.3, + "entropy": 0.585433507245034, + "epoch": 0.664, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.278695583343506, + "learning_rate": 1.684e-07, + "loss": 0.027391403913497925, + "num_tokens": 2800636.0, + "reward": 0.9749999582767487, + "reward_std": 0.3571206539869308, + "rewards/reward_fn/mean": 0.9749999582767487, + "rewards/reward_fn/std": 0.357120668888092, + "step": 830, + "step_time": 27.087428363552316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 605.3, + "completions/max_terminated_length": 541.8, + "completions/mean_length": 256.64375, + "completions/mean_terminated_length": 251.76333465576172, + "completions/min_length": 78.1, + "completions/min_terminated_length": 78.1, + "entropy": 0.5444993877783417, + "epoch": 0.672, + "frac_reward_zero_std": 0.65, + "grad_norm": 2.2607951164245605, + "learning_rate": 1.644e-07, + "loss": 0.022679784893989564, + "num_tokens": 2850327.0, + "reward": 1.034374964237213, + "reward_std": 0.35396216809749603, + "rewards/reward_fn/mean": 1.034374964237213, + "rewards/reward_fn/std": 0.35396219193935397, + "step": 840, + "step_time": 27.1366524099838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 250.46875, + "completions/mean_terminated_length": 250.46875, + "completions/min_length": 94.7, + "completions/min_terminated_length": 94.7, + "entropy": 0.5348553240299225, + "epoch": 0.68, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.5916006565093994, + "learning_rate": 1.6039999999999998e-07, + "loss": -0.023145222663879396, + "num_tokens": 2899042.0, + "reward": 1.0624999582767487, + "reward_std": 0.38501180410385133, + "rewards/reward_fn/mean": 1.0624999582767487, + "rewards/reward_fn/std": 0.3850118160247803, + "step": 850, + "step_time": 24.06133564542979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 492.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 248.4, + "completions/mean_terminated_length": 248.4, + "completions/min_length": 76.7, + "completions/min_terminated_length": 76.7, + "entropy": 0.5445443953387439, + "epoch": 0.688, + "frac_reward_zero_std": 0.825, + "grad_norm": 0.0, + "learning_rate": 1.564e-07, + "loss": 0.008287916332483292, + "num_tokens": 2946790.0, + "reward": 1.0093749642372132, + "reward_std": 0.35188313722610476, + "rewards/reward_fn/mean": 1.0093749642372132, + "rewards/reward_fn/std": 0.3518831551074982, + "step": 860, + "step_time": 21.985819199867546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 594.7, + "completions/max_terminated_length": 532.9, + "completions/mean_length": 261.8625, + "completions/mean_terminated_length": 256.7483337402344, + "completions/min_length": 92.5, + "completions/min_terminated_length": 92.5, + "entropy": 0.5800149150192737, + "epoch": 0.696, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1.524e-07, + "loss": 0.04065183401107788, + "num_tokens": 2997048.0, + "reward": 0.9968749701976776, + "reward_std": 0.30479497015476226, + "rewards/reward_fn/mean": 0.9968749701976776, + "rewards/reward_fn/std": 0.30479499995708464, + "step": 870, + "step_time": 26.580935311084612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.3, + "completions/max_terminated_length": 513.3, + "completions/mean_length": 253.06875, + "completions/mean_terminated_length": 253.06875, + "completions/min_length": 77.6, + "completions/min_terminated_length": 77.6, + "entropy": 0.5305031981319189, + "epoch": 0.704, + "frac_reward_zero_std": 0.75, + "grad_norm": 4.454606533050537, + "learning_rate": 1.484e-07, + "loss": 0.022549983859062196, + "num_tokens": 3045711.0, + "reward": 1.0312499582767487, + "reward_std": 0.38086153864860534, + "rewards/reward_fn/mean": 1.0312499582767487, + "rewards/reward_fn/std": 0.3808615684509277, + "step": 880, + "step_time": 23.01209013015032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 502.9, + "completions/max_terminated_length": 502.9, + "completions/mean_length": 256.13125, + "completions/mean_terminated_length": 256.13125, + "completions/min_length": 66.3, + "completions/min_terminated_length": 66.3, + "entropy": 0.5786763843148947, + "epoch": 0.712, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.9759247303009033, + "learning_rate": 1.444e-07, + "loss": -0.02140951305627823, + "num_tokens": 3094856.0, + "reward": 1.0437499582767487, + "reward_std": 0.3924266636371613, + "rewards/reward_fn/mean": 1.0437499582767487, + "rewards/reward_fn/std": 0.39242667853832247, + "step": 890, + "step_time": 22.476798066869378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 487.1, + "completions/max_terminated_length": 487.1, + "completions/mean_length": 236.9625, + "completions/mean_terminated_length": 236.9625, + "completions/min_length": 78.6, + "completions/min_terminated_length": 78.6, + "entropy": 0.5970848154276609, + "epoch": 0.72, + "frac_reward_zero_std": 0.7, + "grad_norm": 1.90382719039917, + "learning_rate": 1.4039999999999999e-07, + "loss": 0.013022461533546447, + "num_tokens": 3140922.0, + "reward": 1.0562499523162843, + "reward_std": 0.4089065968990326, + "rewards/reward_fn/mean": 1.0562499523162843, + "rewards/reward_fn/std": 0.40890662670135497, + "step": 900, + "step_time": 21.815543547831474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.1, + "completions/max_terminated_length": 585.1, + "completions/mean_length": 267.65625, + "completions/mean_terminated_length": 267.65625, + "completions/min_length": 83.9, + "completions/min_terminated_length": 83.9, + "entropy": 0.5671312633901835, + "epoch": 0.728, + "frac_reward_zero_std": 0.7, + "grad_norm": 3.2715682983398438, + "learning_rate": 1.3639999999999998e-07, + "loss": -0.008570893108844757, + "num_tokens": 3192063.0, + "reward": 1.0312499582767487, + "reward_std": 0.385887947678566, + "rewards/reward_fn/mean": 1.0312499582767487, + "rewards/reward_fn/std": 0.3858879655599594, + "step": 910, + "step_time": 25.983504464896395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 597.8, + "completions/max_terminated_length": 572.6, + "completions/mean_length": 280.96875, + "completions/mean_terminated_length": 276.9291687011719, + "completions/min_length": 112.6, + "completions/min_terminated_length": 112.6, + "entropy": 0.5547854236327112, + "epoch": 0.736, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.8137001991271973, + "learning_rate": 1.324e-07, + "loss": -0.0006064340472221374, + "num_tokens": 3245226.0, + "reward": 1.0593749642372132, + "reward_std": 0.3496703714132309, + "rewards/reward_fn/mean": 1.0593749642372132, + "rewards/reward_fn/std": 0.3496703922748566, + "step": 920, + "step_time": 26.724192412989215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.3, + "completions/max_terminated_length": 521.3, + "completions/mean_length": 258.66875, + "completions/mean_terminated_length": 258.66875, + "completions/min_length": 73.2, + "completions/min_terminated_length": 73.2, + "entropy": 0.55043915361166, + "epoch": 0.744, + "frac_reward_zero_std": 0.75, + "grad_norm": 1.9904998540878296, + "learning_rate": 1.2839999999999999e-07, + "loss": 0.002445448562502861, + "num_tokens": 3295165.0, + "reward": 1.0499999582767487, + "reward_std": 0.40058289766311644, + "rewards/reward_fn/mean": 1.0499999582767487, + "rewards/reward_fn/std": 0.40058292746543883, + "step": 930, + "step_time": 23.23095205714926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.2, + "completions/max_terminated_length": 649.2, + "completions/mean_length": 293.1125, + "completions/mean_terminated_length": 293.1125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "entropy": 0.5630233686417341, + "epoch": 0.752, + "frac_reward_zero_std": 0.75, + "grad_norm": 1.8717433214187622, + "learning_rate": 1.244e-07, + "loss": 0.015195921063423157, + "num_tokens": 3350455.0, + "reward": 0.9562499582767486, + "reward_std": 0.3392513394355774, + "rewards/reward_fn/mean": 0.9562499582767486, + "rewards/reward_fn/std": 0.33925136625766755, + "step": 940, + "step_time": 28.78155018389225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 566.8, + "completions/max_terminated_length": 566.8, + "completions/mean_length": 279.64375, + "completions/mean_terminated_length": 279.64375, + "completions/min_length": 92.6, + "completions/min_terminated_length": 92.6, + "entropy": 0.5822894293814898, + "epoch": 0.76, + "frac_reward_zero_std": 0.725, + "grad_norm": 1.3468974828720093, + "learning_rate": 1.204e-07, + "loss": -0.006339100748300552, + "num_tokens": 3403710.0, + "reward": 1.0437499642372132, + "reward_std": 0.3522812038660049, + "rewards/reward_fn/mean": 1.0437499642372132, + "rewards/reward_fn/std": 0.35228122770786285, + "step": 950, + "step_time": 25.218540608556943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 550.5, + "completions/max_terminated_length": 550.5, + "completions/mean_length": 267.8625, + "completions/mean_terminated_length": 267.8625, + "completions/min_length": 101.9, + "completions/min_terminated_length": 101.9, + "entropy": 0.5490182695910335, + "epoch": 0.768, + "frac_reward_zero_std": 0.775, + "grad_norm": 2.347418785095215, + "learning_rate": 1.164e-07, + "loss": -0.018277975916862487, + "num_tokens": 3454788.0, + "reward": 1.0874999523162843, + "reward_std": 0.4399394065141678, + "rewards/reward_fn/mean": 1.0874999523162843, + "rewards/reward_fn/std": 0.43993942737579345, + "step": 960, + "step_time": 24.56708482489921 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.2, + "completions/max_terminated_length": 563.2, + "completions/mean_length": 286.8625, + "completions/mean_terminated_length": 286.8625, + "completions/min_length": 111.1, + "completions/min_terminated_length": 111.1, + "entropy": 0.5671119350939989, + "epoch": 0.776, + "frac_reward_zero_std": 0.825, + "grad_norm": 0.0, + "learning_rate": 1.124e-07, + "loss": 0.001966666430234909, + "num_tokens": 3508722.0, + "reward": 0.9843749582767487, + "reward_std": 0.3477096170186996, + "rewards/reward_fn/mean": 0.9843749582767487, + "rewards/reward_fn/std": 0.3477096438407898, + "step": 970, + "step_time": 25.104713304387406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01875, + "completions/max_length": 572.8, + "completions/max_terminated_length": 515.7, + "completions/mean_length": 258.20625, + "completions/mean_terminated_length": 244.45547790527343, + "completions/min_length": 68.6, + "completions/min_terminated_length": 68.6, + "entropy": 0.4816724480129778, + "epoch": 0.784, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.33716082572937, + "learning_rate": 1.0839999999999999e-07, + "loss": 0.021231548488140108, + "num_tokens": 3558171.0, + "reward": 1.0968749582767487, + "reward_std": 0.3681450128555298, + "rewards/reward_fn/mean": 1.0968749582767487, + "rewards/reward_fn/std": 0.3681450396776199, + "step": 980, + "step_time": 25.863722542859612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 474.4, + "completions/max_terminated_length": 474.4, + "completions/mean_length": 241.725, + "completions/mean_terminated_length": 241.725, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "entropy": 0.5186899159103632, + "epoch": 0.792, + "frac_reward_zero_std": 0.775, + "grad_norm": 2.8566486835479736, + "learning_rate": 1.0440000000000001e-07, + "loss": -0.013251829147338866, + "num_tokens": 3604823.0, + "reward": 1.0312499582767487, + "reward_std": 0.3475317031145096, + "rewards/reward_fn/mean": 1.0312499582767487, + "rewards/reward_fn/std": 0.347531720995903, + "step": 990, + "step_time": 21.354597127251328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 513.8, + "completions/max_terminated_length": 472.6, + "completions/mean_length": 276.25625, + "completions/mean_terminated_length": 271.7895843505859, + "completions/min_length": 129.9, + "completions/min_terminated_length": 129.9, + "entropy": 0.5287249894812703, + "epoch": 0.8, + "frac_reward_zero_std": 0.8, + "grad_norm": 1.7595624923706055, + "learning_rate": 1.004e-07, + "loss": 0.030878221988677977, + "num_tokens": 3657228.0, + "reward": 1.0406249701976775, + "reward_std": 0.31898270547389984, + "rewards/reward_fn/mean": 1.0406249701976775, + "rewards/reward_fn/std": 0.31898272335529326, + "step": 1000, + "step_time": 23.230659662559628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 557.3, + "completions/max_terminated_length": 491.5, + "completions/mean_length": 266.30625, + "completions/mean_terminated_length": 261.2725006103516, + "completions/min_length": 75.9, + "completions/min_terminated_length": 75.9, + "entropy": 0.5200971383601427, + "epoch": 0.808, + "frac_reward_zero_std": 0.725, + "grad_norm": 2.3148114681243896, + "learning_rate": 9.639999999999999e-08, + "loss": 0.003333679959177971, + "num_tokens": 3708309.0, + "reward": 1.0343749582767487, + "reward_std": 0.3490926504135132, + "rewards/reward_fn/mean": 1.0343749582767487, + "rewards/reward_fn/std": 0.34909267723560333, + "step": 1010, + "step_time": 25.057548108976334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.4, + "completions/max_terminated_length": 537.4, + "completions/mean_length": 272.03125, + "completions/mean_terminated_length": 272.03125, + "completions/min_length": 110.4, + "completions/min_terminated_length": 110.4, + "entropy": 0.5065994596108794, + "epoch": 0.816, + "frac_reward_zero_std": 0.725, + "grad_norm": 3.423994541168213, + "learning_rate": 9.24e-08, + "loss": 0.023444755375385283, + "num_tokens": 3760274.0, + "reward": 1.0218749582767486, + "reward_std": 0.35771805346012114, + "rewards/reward_fn/mean": 1.0218749582767486, + "rewards/reward_fn/std": 0.35771807432174685, + "step": 1020, + "step_time": 23.9467576073017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.8, + "completions/max_terminated_length": 510.8, + "completions/mean_length": 283.375, + "completions/mean_terminated_length": 283.375, + "completions/min_length": 129.7, + "completions/min_terminated_length": 129.7, + "entropy": 0.5147501945495605, + "epoch": 0.824, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.1647582054138184, + "learning_rate": 8.84e-08, + "loss": 0.0007140228524804115, + "num_tokens": 3813922.0, + "reward": 1.199999952316284, + "reward_std": 0.491314160823822, + "rewards/reward_fn/mean": 1.199999952316284, + "rewards/reward_fn/std": 0.49131418466567994, + "step": 1030, + "step_time": 23.01038688295521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.3, + "completions/max_terminated_length": 521.3, + "completions/mean_length": 264.31875, + "completions/mean_terminated_length": 264.31875, + "completions/min_length": 121.5, + "completions/min_terminated_length": 121.5, + "entropy": 0.5070869445800781, + "epoch": 0.832, + "frac_reward_zero_std": 0.625, + "grad_norm": 2.585664749145508, + "learning_rate": 8.44e-08, + "loss": 0.028041335940361022, + "num_tokens": 3864485.0, + "reward": 1.0937499582767487, + "reward_std": 0.4248550355434418, + "rewards/reward_fn/mean": 1.0937499582767487, + "rewards/reward_fn/std": 0.42485505938529966, + "step": 1040, + "step_time": 23.34787617237307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 549.5, + "completions/max_terminated_length": 502.5, + "completions/mean_length": 267.49375, + "completions/mean_terminated_length": 262.67791748046875, + "completions/min_length": 93.5, + "completions/min_terminated_length": 93.5, + "entropy": 0.5002012770622969, + "epoch": 0.84, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 8.039999999999999e-08, + "loss": 0.04210644662380218, + "num_tokens": 3915900.0, + "reward": 1.0968749582767487, + "reward_std": 0.41755713522434235, + "rewards/reward_fn/mean": 1.0968749582767487, + "rewards/reward_fn/std": 0.4175571441650391, + "step": 1050, + "step_time": 24.77414979697205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 627.7, + "completions/max_terminated_length": 555.7, + "completions/mean_length": 305.38125, + "completions/mean_terminated_length": 289.6212219238281, + "completions/min_length": 135.6, + "completions/min_terminated_length": 135.6, + "entropy": 0.5611137403175235, + "epoch": 0.848, + "frac_reward_zero_std": 0.675, + "grad_norm": 2.05741024017334, + "learning_rate": 7.64e-08, + "loss": 0.0378756046295166, + "num_tokens": 3973093.0, + "reward": 0.9687499642372132, + "reward_std": 0.332451206445694, + "rewards/reward_fn/mean": 0.9687499642372132, + "rewards/reward_fn/std": 0.33245123326778414, + "step": 1060, + "step_time": 28.287785303639247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 746.1, + "completions/max_terminated_length": 658.4, + "completions/mean_length": 300.1625, + "completions/mean_terminated_length": 290.8487518310547, + "completions/min_length": 116.9, + "completions/min_terminated_length": 116.9, + "entropy": 0.5516405990347266, + "epoch": 0.856, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.4637022018432617, + "learning_rate": 7.24e-08, + "loss": 0.02026980072259903, + "num_tokens": 4029403.0, + "reward": 0.9187499821186066, + "reward_std": 0.22319530844688415, + "rewards/reward_fn/mean": 0.9187499821186066, + "rewards/reward_fn/std": 0.22319531738758086, + "step": 1070, + "step_time": 33.521394540695475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 534.6, + "completions/max_terminated_length": 488.9, + "completions/mean_length": 263.6125, + "completions/mean_terminated_length": 258.89708557128904, + "completions/min_length": 100.4, + "completions/min_terminated_length": 100.4, + "entropy": 0.5176247822120785, + "epoch": 0.864, + "frac_reward_zero_std": 0.75, + "grad_norm": 3.2136309146881104, + "learning_rate": 6.84e-08, + "loss": 0.035635238885879515, + "num_tokens": 4080145.0, + "reward": 1.015624964237213, + "reward_std": 0.3429378718137741, + "rewards/reward_fn/mean": 1.015624964237213, + "rewards/reward_fn/std": 0.34293788969516753, + "step": 1080, + "step_time": 24.050806782906875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.3, + "completions/max_terminated_length": 542.3, + "completions/mean_length": 260.6125, + "completions/mean_terminated_length": 260.6125, + "completions/min_length": 112.2, + "completions/min_terminated_length": 112.2, + "entropy": 0.5131400337442755, + "epoch": 0.872, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 6.44e-08, + "loss": 0.0009937494993209838, + "num_tokens": 4129863.0, + "reward": 0.9937499642372132, + "reward_std": 0.3179366230964661, + "rewards/reward_fn/mean": 0.9937499642372132, + "rewards/reward_fn/std": 0.31793664693832396, + "step": 1090, + "step_time": 24.295423823781313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01875, + "completions/max_length": 593.4, + "completions/max_terminated_length": 483.5, + "completions/mean_length": 250.2875, + "completions/mean_terminated_length": 235.8367889404297, + "completions/min_length": 98.2, + "completions/min_terminated_length": 98.2, + "entropy": 0.4774348322302103, + "epoch": 0.88, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.0, + "learning_rate": 6.04e-08, + "loss": 0.05956764221191406, + "num_tokens": 4178285.0, + "reward": 1.0656249523162842, + "reward_std": 0.4110028088092804, + "rewards/reward_fn/mean": 1.0656249523162842, + "rewards/reward_fn/std": 0.4110028326511383, + "step": 1100, + "step_time": 26.840810445183887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 591.2, + "completions/max_terminated_length": 540.6, + "completions/mean_length": 274.45625, + "completions/mean_terminated_length": 269.875, + "completions/min_length": 124.3, + "completions/min_terminated_length": 124.3, + "entropy": 0.5233237653970718, + "epoch": 0.888, + "frac_reward_zero_std": 0.7, + "grad_norm": 4.72261381149292, + "learning_rate": 5.6399999999999995e-08, + "loss": -0.01764392852783203, + "num_tokens": 4230406.0, + "reward": 1.0343749582767487, + "reward_std": 0.40663672983646393, + "rewards/reward_fn/mean": 1.0343749582767487, + "rewards/reward_fn/std": 0.4066367566585541, + "step": 1110, + "step_time": 26.58696360127069 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 544.6, + "completions/max_terminated_length": 486.7, + "completions/mean_length": 264.1125, + "completions/mean_terminated_length": 259.6304168701172, + "completions/min_length": 124.9, + "completions/min_terminated_length": 124.9, + "entropy": 0.49998724516481163, + "epoch": 0.896, + "frac_reward_zero_std": 0.725, + "grad_norm": 0.0, + "learning_rate": 5.24e-08, + "loss": 0.03018750548362732, + "num_tokens": 4280868.0, + "reward": 1.0843749582767486, + "reward_std": 0.4210015803575516, + "rewards/reward_fn/mean": 1.0843749582767486, + "rewards/reward_fn/std": 0.421001598238945, + "step": 1120, + "step_time": 24.553724389290437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 616.2, + "completions/max_terminated_length": 564.2, + "completions/mean_length": 266.13125, + "completions/mean_terminated_length": 261.57250061035154, + "completions/min_length": 93.2, + "completions/min_terminated_length": 93.2, + "entropy": 0.5146013794466853, + "epoch": 0.904, + "frac_reward_zero_std": 0.65, + "grad_norm": 3.1288528442382812, + "learning_rate": 4.8399999999999997e-08, + "loss": 0.03101418912410736, + "num_tokens": 4331401.0, + "reward": 1.0718749582767486, + "reward_std": 0.37411039769649507, + "rewards/reward_fn/mean": 1.0718749582767486, + "rewards/reward_fn/std": 0.3741104155778885, + "step": 1130, + "step_time": 27.615588352596387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 624.7, + "completions/max_terminated_length": 591.7, + "completions/mean_length": 291.43125, + "completions/mean_terminated_length": 286.9612518310547, + "completions/min_length": 109.7, + "completions/min_terminated_length": 109.7, + "entropy": 0.528622365463525, + "epoch": 0.912, + "frac_reward_zero_std": 0.7, + "grad_norm": 1.9404244422912598, + "learning_rate": 4.44e-08, + "loss": 0.027211108803749086, + "num_tokens": 4386302.0, + "reward": 1.0781249582767487, + "reward_std": 0.3990673363208771, + "rewards/reward_fn/mean": 1.0781249582767487, + "rewards/reward_fn/std": 0.3990673542022705, + "step": 1140, + "step_time": 27.958656183769925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 527.2, + "completions/max_terminated_length": 527.2, + "completions/mean_length": 286.13125, + "completions/mean_terminated_length": 286.13125, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "entropy": 0.513075502589345, + "epoch": 0.92, + "frac_reward_zero_std": 0.725, + "grad_norm": 1.4120060205459595, + "learning_rate": 4.04e-08, + "loss": -0.000885472446680069, + "num_tokens": 4440643.0, + "reward": 1.062499964237213, + "reward_std": 0.3273422926664352, + "rewards/reward_fn/mean": 1.062499964237213, + "rewards/reward_fn/std": 0.3273423135280609, + "step": 1150, + "step_time": 23.63183649624698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 599.6, + "completions/max_terminated_length": 599.6, + "completions/mean_length": 280.80625, + "completions/mean_terminated_length": 280.80625, + "completions/min_length": 125.5, + "completions/min_terminated_length": 125.5, + "entropy": 0.5326275195926428, + "epoch": 0.928, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.777073860168457, + "learning_rate": 3.64e-08, + "loss": 0.021264398097991945, + "num_tokens": 4493740.0, + "reward": 1.0749999582767487, + "reward_std": 0.4012425780296326, + "rewards/reward_fn/mean": 1.0749999582767487, + "rewards/reward_fn/std": 0.40124259889125824, + "step": 1160, + "step_time": 27.038261169008912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 626.3, + "completions/max_terminated_length": 570.1, + "completions/mean_length": 281.4875, + "completions/mean_terminated_length": 272.0464294433594, + "completions/min_length": 88.5, + "completions/min_terminated_length": 88.5, + "entropy": 0.5435919009149075, + "epoch": 0.936, + "frac_reward_zero_std": 0.7, + "grad_norm": 2.009503126144409, + "learning_rate": 3.24e-08, + "loss": 0.014678403735160828, + "num_tokens": 4547326.0, + "reward": 0.9749999642372131, + "reward_std": 0.3376161724328995, + "rewards/reward_fn/mean": 0.9749999642372131, + "rewards/reward_fn/std": 0.33761619329452514, + "step": 1170, + "step_time": 28.00773431826383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 540.3, + "completions/max_terminated_length": 540.3, + "completions/mean_length": 281.43125, + "completions/mean_terminated_length": 281.43125, + "completions/min_length": 134.6, + "completions/min_terminated_length": 134.6, + "entropy": 0.5330241896212101, + "epoch": 0.944, + "frac_reward_zero_std": 0.75, + "grad_norm": 2.01973557472229, + "learning_rate": 2.84e-08, + "loss": 0.010066453367471695, + "num_tokens": 4600699.0, + "reward": 0.9749999701976776, + "reward_std": 0.30032309591770173, + "rewards/reward_fn/mean": 0.9749999701976776, + "rewards/reward_fn/std": 0.3003231108188629, + "step": 1180, + "step_time": 24.195893923100083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.7, + "completions/max_terminated_length": 531.7, + "completions/mean_length": 278.15, + "completions/mean_terminated_length": 278.15, + "completions/min_length": 110.2, + "completions/min_terminated_length": 110.2, + "entropy": 0.5508426167070866, + "epoch": 0.952, + "frac_reward_zero_std": 0.875, + "grad_norm": 0.0, + "learning_rate": 2.44e-08, + "loss": 0.009711582958698273, + "num_tokens": 4653463.0, + "reward": 1.031249964237213, + "reward_std": 0.34981620609760283, + "rewards/reward_fn/mean": 1.031249964237213, + "rewards/reward_fn/std": 0.34981622397899625, + "step": 1190, + "step_time": 23.97323694275692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01875, + "completions/max_length": 658.7, + "completions/max_terminated_length": 541.0, + "completions/mean_length": 285.43125, + "completions/mean_terminated_length": 271.64125671386716, + "completions/min_length": 99.9, + "completions/min_terminated_length": 99.9, + "entropy": 0.5076606808230281, + "epoch": 0.96, + "frac_reward_zero_std": 0.725, + "grad_norm": 2.406820774078369, + "learning_rate": 2.04e-08, + "loss": 0.07293225526809692, + "num_tokens": 4707780.0, + "reward": 0.9531249642372132, + "reward_std": 0.3397530823945999, + "rewards/reward_fn/mean": 0.9531249642372132, + "rewards/reward_fn/std": 0.3397530972957611, + "step": 1200, + "step_time": 29.918243083078416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025, + "completions/max_length": 778.4, + "completions/max_terminated_length": 598.6, + "completions/mean_length": 298.01875, + "completions/mean_terminated_length": 279.6341720581055, + "completions/min_length": 107.9, + "completions/min_terminated_length": 107.9, + "entropy": 0.4813551393337548, + "epoch": 0.968, + "frac_reward_zero_std": 0.675, + "grad_norm": 0.0, + "learning_rate": 1.64e-08, + "loss": 0.07352781295776367, + "num_tokens": 4763607.0, + "reward": 1.0562499523162843, + "reward_std": 0.444338920712471, + "rewards/reward_fn/mean": 1.0562499523162843, + "rewards/reward_fn/std": 0.44433894753456116, + "step": 1210, + "step_time": 35.2928637356963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.4, + "completions/max_terminated_length": 542.4, + "completions/mean_length": 273.9375, + "completions/mean_terminated_length": 273.9375, + "completions/min_length": 112.5, + "completions/min_terminated_length": 112.5, + "entropy": 0.5351916439831257, + "epoch": 0.976, + "frac_reward_zero_std": 0.7, + "grad_norm": 2.6699888706207275, + "learning_rate": 1.2399999999999999e-08, + "loss": 0.020609369874000548, + "num_tokens": 4815925.0, + "reward": 1.043749952316284, + "reward_std": 0.409377720952034, + "rewards/reward_fn/mean": 1.043749952316284, + "rewards/reward_fn/std": 0.4093777507543564, + "step": 1220, + "step_time": 24.284022097615527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.8, + "completions/max_terminated_length": 555.8, + "completions/mean_length": 295.23125, + "completions/mean_terminated_length": 295.23125, + "completions/min_length": 116.1, + "completions/min_terminated_length": 116.1, + "entropy": 0.5482900742441416, + "epoch": 0.984, + "frac_reward_zero_std": 0.725, + "grad_norm": 1.7745944261550903, + "learning_rate": 8.399999999999999e-09, + "loss": 0.003481149673461914, + "num_tokens": 4871670.0, + "reward": 0.9249999582767486, + "reward_std": 0.30596340596675875, + "rewards/reward_fn/mean": 0.9249999582767486, + "rewards/reward_fn/std": 0.30596343576908114, + "step": 1230, + "step_time": 24.858759614871815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0125, + "completions/max_length": 614.1, + "completions/max_terminated_length": 498.9, + "completions/mean_length": 273.8625, + "completions/mean_terminated_length": 264.3100021362305, + "completions/min_length": 131.8, + "completions/min_terminated_length": 131.8, + "entropy": 0.5003731534816325, + "epoch": 0.992, + "frac_reward_zero_std": 0.6, + "grad_norm": 2.952770471572876, + "learning_rate": 4.4e-09, + "loss": 0.05144896507263184, + "num_tokens": 4923236.0, + "reward": 1.0687499582767486, + "reward_std": 0.3899788945913315, + "rewards/reward_fn/mean": 1.0687499582767486, + "rewards/reward_fn/std": 0.3899789124727249, + "step": 1240, + "step_time": 27.80349059002474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 493.7, + "completions/max_terminated_length": 493.7, + "completions/mean_length": 250.225, + "completions/mean_terminated_length": 250.225, + "completions/min_length": 87.9, + "completions/min_terminated_length": 87.9, + "entropy": 0.5107808768749237, + "epoch": 1.0, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 4e-10, + "loss": 0.010706515610218048, + "num_tokens": 4971516.0, + "reward": 1.0812499701976777, + "reward_std": 0.31392850875854494, + "rewards/reward_fn/mean": 1.0812499701976777, + "rewards/reward_fn/std": 0.31392852365970614, + "step": 1250, + "step_time": 22.25712463380769 + } + ], + "logging_steps": 10, + "max_steps": 1250, + "num_input_tokens_seen": 4971516, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1250/training_args.bin b/checkpoint-1250/training_args.bin new file mode 100644 index 0000000..10573ae --- /dev/null +++ b/checkpoint-1250/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c73d2359305892bf20c227daabb9915c6f04679f3201e98f6362c5ce2c07316a +size 7185 diff --git a/checkpoint-2375/chat_template.jinja b/checkpoint-2375/chat_template.jinja new file mode 100644 index 0000000..a43db74 --- /dev/null +++ b/checkpoint-2375/chat_template.jinja @@ -0,0 +1,94 @@ +{%- set enable_thinking = true %} +{%- set default_system_message = "You are a knowledgeable assistant. For multiple-choice questions, reason step by step, then write only the letter of the correct answer inside \\boxed{}." %} +{%- if messages[0]['role'] != 'system' %} + {%- set messages = [{'role': 'system', 'content': default_system_message}] + messages %} +{%- endif %} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-2375/config.json b/checkpoint-2375/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-2375/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2375/generation_config.json b/checkpoint-2375/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-2375/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-2375/model.safetensors b/checkpoint-2375/model.safetensors new file mode 100644 index 0000000..7dea07d --- /dev/null +++ b/checkpoint-2375/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79b21686847c40ebe6d4347ee1596f5eec6a74786ef5f2aefd84df8eed629169 +size 3441185608 diff --git a/checkpoint-2375/optimizer.pt b/checkpoint-2375/optimizer.pt new file mode 100644 index 0000000..caf29b0 --- /dev/null +++ b/checkpoint-2375/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91104ff0dcdd3c6ba85eb6f6db7f27dc5e9d1b95b3fcff2cadd467e012bbe0d2 +size 6882572207 diff --git a/checkpoint-2375/rng_state.pth b/checkpoint-2375/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-2375/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-2375/scheduler.pt b/checkpoint-2375/scheduler.pt new file mode 100644 index 0000000..fea89dc --- /dev/null +++ b/checkpoint-2375/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:562cfbf5dd82c88dd3ff82d72a61b2d4f6fa52756c99822b4af3792b8d4664d6 +size 1465 diff --git a/checkpoint-2375/tokenizer.json b/checkpoint-2375/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-2375/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-2375/tokenizer_config.json b/checkpoint-2375/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/checkpoint-2375/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2375/trainer_state.json b/checkpoint-2375/trainer_state.json new file mode 100644 index 0000000..1fcc993 --- /dev/null +++ b/checkpoint-2375/trainer_state.json @@ -0,0 +1,2404 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 2375, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.4990599287673831, + "epoch": 0.004210526315789474, + "grad_norm": 69.0, + "learning_rate": 6.338028169014085e-07, + "loss": 1.671738624572754, + "mean_token_accuracy": 0.7763976119458675, + "num_tokens": 31553.0, + "step": 10 + }, + { + "entropy": 0.4624864859506488, + "epoch": 0.008421052631578947, + "grad_norm": 65.0, + "learning_rate": 1.3380281690140844e-06, + "loss": 1.5600993156433105, + "mean_token_accuracy": 0.7901717454195023, + "num_tokens": 63579.0, + "step": 20 + }, + { + "entropy": 0.5733067244291306, + "epoch": 0.01263157894736842, + "grad_norm": 54.5, + "learning_rate": 2.0422535211267608e-06, + "loss": 1.6138050079345703, + "mean_token_accuracy": 0.7659219928085804, + "num_tokens": 92495.0, + "step": 30 + }, + { + "entropy": 0.576881331205368, + "epoch": 0.016842105263157894, + "grad_norm": 30.75, + "learning_rate": 2.746478873239437e-06, + "loss": 1.3854169845581055, + "mean_token_accuracy": 0.7819479435682297, + "num_tokens": 122375.0, + "step": 40 + }, + { + "entropy": 0.6074449960142374, + "epoch": 0.021052631578947368, + "grad_norm": 21.375, + "learning_rate": 3.4507042253521127e-06, + "loss": 1.1678359985351563, + "mean_token_accuracy": 0.788398090749979, + "num_tokens": 152668.0, + "step": 50 + }, + { + "entropy": 0.7060422364622354, + "epoch": 0.02526315789473684, + "grad_norm": 12.1875, + "learning_rate": 4.154929577464789e-06, + "loss": 0.9693965911865234, + "mean_token_accuracy": 0.798081835359335, + "num_tokens": 183863.0, + "step": 60 + }, + { + "entropy": 0.741611959785223, + "epoch": 0.029473684210526315, + "grad_norm": 12.0, + "learning_rate": 4.859154929577465e-06, + "loss": 0.7716450214385986, + "mean_token_accuracy": 0.8399579018354416, + "num_tokens": 212992.0, + "step": 70 + }, + { + "entropy": 0.6091062590479851, + "epoch": 0.03368421052631579, + "grad_norm": 15.0, + "learning_rate": 4.982638888888889e-06, + "loss": 0.5984757900238037, + "mean_token_accuracy": 0.8683908067643642, + "num_tokens": 242894.0, + "step": 80 + }, + { + "entropy": 0.4648941744118929, + "epoch": 0.037894736842105266, + "grad_norm": 9.25, + "learning_rate": 4.9609375e-06, + "loss": 0.45642986297607424, + "mean_token_accuracy": 0.8944785602390766, + "num_tokens": 273459.0, + "step": 90 + }, + { + "entropy": 0.3587808248586953, + "epoch": 0.042105263157894736, + "grad_norm": 8.375, + "learning_rate": 4.9392361111111115e-06, + "loss": 0.38718841075897215, + "mean_token_accuracy": 0.9098845317959785, + "num_tokens": 305452.0, + "step": 100 + }, + { + "entropy": 0.31363106137141583, + "epoch": 0.04631578947368421, + "grad_norm": 7.71875, + "learning_rate": 4.917534722222223e-06, + "loss": 0.35190615653991697, + "mean_token_accuracy": 0.9152749516069889, + "num_tokens": 336691.0, + "step": 110 + }, + { + "entropy": 0.30077689900062976, + "epoch": 0.05052631578947368, + "grad_norm": 7.5625, + "learning_rate": 4.895833333333333e-06, + "loss": 0.3593594551086426, + "mean_token_accuracy": 0.9169687710702419, + "num_tokens": 368879.0, + "step": 120 + }, + { + "entropy": 0.2662435117177665, + "epoch": 0.05473684210526316, + "grad_norm": 8.1875, + "learning_rate": 4.8741319444444444e-06, + "loss": 0.316438627243042, + "mean_token_accuracy": 0.9221884422004223, + "num_tokens": 399165.0, + "step": 130 + }, + { + "entropy": 0.26467454619705677, + "epoch": 0.05894736842105263, + "grad_norm": 8.8125, + "learning_rate": 4.8524305555555565e-06, + "loss": 0.27219557762145996, + "mean_token_accuracy": 0.9298946231603622, + "num_tokens": 428931.0, + "step": 140 + }, + { + "entropy": 0.2657210439443588, + "epoch": 0.06315789473684211, + "grad_norm": 16.875, + "learning_rate": 4.830729166666667e-06, + "loss": 0.3098090171813965, + "mean_token_accuracy": 0.9222414299845696, + "num_tokens": 458794.0, + "step": 150 + }, + { + "entropy": 0.23787126671522857, + "epoch": 0.06736842105263158, + "grad_norm": 10.0, + "learning_rate": 4.809027777777778e-06, + "loss": 0.2524611711502075, + "mean_token_accuracy": 0.9230330072343349, + "num_tokens": 489763.0, + "step": 160 + }, + { + "entropy": 0.2247232544235885, + "epoch": 0.07157894736842105, + "grad_norm": 10.75, + "learning_rate": 4.787326388888889e-06, + "loss": 0.2346771240234375, + "mean_token_accuracy": 0.93573377430439, + "num_tokens": 520029.0, + "step": 170 + }, + { + "entropy": 0.22070267596282064, + "epoch": 0.07578947368421053, + "grad_norm": 7.75, + "learning_rate": 4.765625000000001e-06, + "loss": 0.21888349056243897, + "mean_token_accuracy": 0.9352433510124684, + "num_tokens": 550742.0, + "step": 180 + }, + { + "entropy": 0.21396164572797716, + "epoch": 0.08, + "grad_norm": 7.65625, + "learning_rate": 4.743923611111111e-06, + "loss": 0.21159236431121825, + "mean_token_accuracy": 0.9358419619500637, + "num_tokens": 580071.0, + "step": 190 + }, + { + "entropy": 0.20088617503643036, + "epoch": 0.08421052631578947, + "grad_norm": 9.0625, + "learning_rate": 4.722222222222222e-06, + "loss": 0.210105037689209, + "mean_token_accuracy": 0.9405015103518963, + "num_tokens": 610346.0, + "step": 200 + }, + { + "entropy": 0.2197779224254191, + "epoch": 0.08842105263157894, + "grad_norm": 6.25, + "learning_rate": 4.7005208333333335e-06, + "loss": 0.24622466564178466, + "mean_token_accuracy": 0.9350829392671585, + "num_tokens": 640068.0, + "step": 210 + }, + { + "entropy": 0.22689010314643382, + "epoch": 0.09263157894736843, + "grad_norm": 12.25, + "learning_rate": 4.678819444444445e-06, + "loss": 0.25157005786895753, + "mean_token_accuracy": 0.9245235815644264, + "num_tokens": 668586.0, + "step": 220 + }, + { + "entropy": 0.2026051654946059, + "epoch": 0.0968421052631579, + "grad_norm": 11.625, + "learning_rate": 4.657118055555556e-06, + "loss": 0.22865614891052247, + "mean_token_accuracy": 0.9291222095489502, + "num_tokens": 700113.0, + "step": 230 + }, + { + "entropy": 0.20945081664249302, + "epoch": 0.10105263157894737, + "grad_norm": 8.4375, + "learning_rate": 4.635416666666667e-06, + "loss": 0.20557198524475098, + "mean_token_accuracy": 0.9318756103515625, + "num_tokens": 730359.0, + "step": 240 + }, + { + "entropy": 0.1935979576781392, + "epoch": 0.10526315789473684, + "grad_norm": 9.75, + "learning_rate": 4.6137152777777785e-06, + "loss": 0.20995285511016845, + "mean_token_accuracy": 0.9389079615473748, + "num_tokens": 761042.0, + "step": 250 + }, + { + "entropy": 0.21530479132197797, + "epoch": 0.10947368421052632, + "grad_norm": 10.25, + "learning_rate": 4.59201388888889e-06, + "loss": 0.21937789916992187, + "mean_token_accuracy": 0.9335255794227123, + "num_tokens": 792117.0, + "step": 260 + }, + { + "entropy": 0.24162108548916877, + "epoch": 0.11368421052631579, + "grad_norm": 10.5, + "learning_rate": 4.5703125e-06, + "loss": 0.2889341115951538, + "mean_token_accuracy": 0.9264158695936203, + "num_tokens": 822509.0, + "step": 270 + }, + { + "entropy": 0.22190025104209782, + "epoch": 0.11789473684210526, + "grad_norm": 9.4375, + "learning_rate": 4.548611111111111e-06, + "loss": 0.22677860260009766, + "mean_token_accuracy": 0.9339636966586113, + "num_tokens": 852474.0, + "step": 280 + }, + { + "entropy": 0.21453084875829517, + "epoch": 0.12210526315789473, + "grad_norm": 7.65625, + "learning_rate": 4.526909722222223e-06, + "loss": 0.20040695667266845, + "mean_token_accuracy": 0.9411987900733948, + "num_tokens": 882821.0, + "step": 290 + }, + { + "entropy": 0.2034795220475644, + "epoch": 0.12631578947368421, + "grad_norm": 12.6875, + "learning_rate": 4.505208333333334e-06, + "loss": 0.24240901470184326, + "mean_token_accuracy": 0.9343540959060193, + "num_tokens": 913298.0, + "step": 300 + }, + { + "entropy": 0.20725304959341884, + "epoch": 0.13052631578947368, + "grad_norm": 10.25, + "learning_rate": 4.483506944444444e-06, + "loss": 0.22464065551757811, + "mean_token_accuracy": 0.9300798162817955, + "num_tokens": 944669.0, + "step": 310 + }, + { + "entropy": 0.18791212746873498, + "epoch": 0.13473684210526315, + "grad_norm": 8.0, + "learning_rate": 4.461805555555556e-06, + "loss": 0.17531607151031495, + "mean_token_accuracy": 0.9410846054553985, + "num_tokens": 973038.0, + "step": 320 + }, + { + "entropy": 0.1887997885234654, + "epoch": 0.13894736842105262, + "grad_norm": 8.25, + "learning_rate": 4.440104166666668e-06, + "loss": 0.20259859561920165, + "mean_token_accuracy": 0.9379140719771385, + "num_tokens": 1003504.0, + "step": 330 + }, + { + "entropy": 0.21206430648453534, + "epoch": 0.1431578947368421, + "grad_norm": 8.625, + "learning_rate": 4.418402777777778e-06, + "loss": 0.24106347560882568, + "mean_token_accuracy": 0.9289649538695812, + "num_tokens": 1032582.0, + "step": 340 + }, + { + "entropy": 0.1938377027399838, + "epoch": 0.14736842105263157, + "grad_norm": 9.5, + "learning_rate": 4.396701388888889e-06, + "loss": 0.19278082847595215, + "mean_token_accuracy": 0.9380026146769523, + "num_tokens": 1064109.0, + "step": 350 + }, + { + "entropy": 0.1793072429485619, + "epoch": 0.15157894736842106, + "grad_norm": 7.65625, + "learning_rate": 4.3750000000000005e-06, + "loss": 0.18131839036941527, + "mean_token_accuracy": 0.9433922998607158, + "num_tokens": 1094818.0, + "step": 360 + }, + { + "entropy": 0.19749868060462178, + "epoch": 0.15578947368421053, + "grad_norm": 4.75, + "learning_rate": 4.353298611111112e-06, + "loss": 0.18917640447616577, + "mean_token_accuracy": 0.9424469873309136, + "num_tokens": 1123344.0, + "step": 370 + }, + { + "entropy": 0.1859517907258123, + "epoch": 0.16, + "grad_norm": 10.375, + "learning_rate": 4.331597222222222e-06, + "loss": 0.19585838317871093, + "mean_token_accuracy": 0.9397093847393989, + "num_tokens": 1154149.0, + "step": 380 + }, + { + "entropy": 0.17284040013328195, + "epoch": 0.16421052631578947, + "grad_norm": 14.3125, + "learning_rate": 4.309895833333333e-06, + "loss": 0.16065752506256104, + "mean_token_accuracy": 0.9468757703900337, + "num_tokens": 1185496.0, + "step": 390 + }, + { + "entropy": 0.2033068159595132, + "epoch": 0.16842105263157894, + "grad_norm": 9.0, + "learning_rate": 4.288194444444445e-06, + "loss": 0.22716670036315917, + "mean_token_accuracy": 0.9365929305553437, + "num_tokens": 1215776.0, + "step": 400 + }, + { + "entropy": 0.20319096064195036, + "epoch": 0.1726315789473684, + "grad_norm": 9.3125, + "learning_rate": 4.266493055555556e-06, + "loss": 0.2225698471069336, + "mean_token_accuracy": 0.9277164503931999, + "num_tokens": 1245574.0, + "step": 410 + }, + { + "entropy": 0.1901295615825802, + "epoch": 0.17684210526315788, + "grad_norm": 10.375, + "learning_rate": 4.244791666666667e-06, + "loss": 0.22110774517059326, + "mean_token_accuracy": 0.9325532615184784, + "num_tokens": 1274979.0, + "step": 420 + }, + { + "entropy": 0.19151646881364287, + "epoch": 0.18105263157894738, + "grad_norm": 6.9375, + "learning_rate": 4.223090277777778e-06, + "loss": 0.1919734835624695, + "mean_token_accuracy": 0.9378305852413178, + "num_tokens": 1306036.0, + "step": 430 + }, + { + "entropy": 0.2085381604731083, + "epoch": 0.18526315789473685, + "grad_norm": 7.53125, + "learning_rate": 4.2013888888888896e-06, + "loss": 0.21823127269744874, + "mean_token_accuracy": 0.9345854863524437, + "num_tokens": 1335047.0, + "step": 440 + }, + { + "entropy": 0.2007679786067456, + "epoch": 0.18947368421052632, + "grad_norm": 8.75, + "learning_rate": 4.1796875e-06, + "loss": 0.21735711097717286, + "mean_token_accuracy": 0.9340473741292954, + "num_tokens": 1366790.0, + "step": 450 + }, + { + "entropy": 0.17802681485190988, + "epoch": 0.1936842105263158, + "grad_norm": 5.34375, + "learning_rate": 4.157986111111111e-06, + "loss": 0.19400970935821532, + "mean_token_accuracy": 0.9380262985825538, + "num_tokens": 1394887.0, + "step": 460 + }, + { + "entropy": 0.21595997624099256, + "epoch": 0.19789473684210526, + "grad_norm": 7.1875, + "learning_rate": 4.1362847222222224e-06, + "loss": 0.22792208194732666, + "mean_token_accuracy": 0.9314667269587517, + "num_tokens": 1425391.0, + "step": 470 + }, + { + "entropy": 0.18051641024649143, + "epoch": 0.20210526315789473, + "grad_norm": 4.9375, + "learning_rate": 4.114583333333334e-06, + "loss": 0.19123028516769408, + "mean_token_accuracy": 0.9385565176606179, + "num_tokens": 1453952.0, + "step": 480 + }, + { + "entropy": 0.2041733751539141, + "epoch": 0.2063157894736842, + "grad_norm": 11.1875, + "learning_rate": 4.092881944444445e-06, + "loss": 0.21048731803894044, + "mean_token_accuracy": 0.9365776918828488, + "num_tokens": 1487071.0, + "step": 490 + }, + { + "entropy": 0.17413038527593017, + "epoch": 0.21052631578947367, + "grad_norm": 8.4375, + "learning_rate": 4.071180555555556e-06, + "loss": 0.1920235872268677, + "mean_token_accuracy": 0.9346710205078125, + "num_tokens": 1518557.0, + "step": 500 + }, + { + "entropy": 0.1804610774386674, + "epoch": 0.21473684210526317, + "grad_norm": 6.71875, + "learning_rate": 4.049479166666667e-06, + "loss": 0.18783496618270873, + "mean_token_accuracy": 0.9396589122712612, + "num_tokens": 1548878.0, + "step": 510 + }, + { + "entropy": 0.19870298462919891, + "epoch": 0.21894736842105264, + "grad_norm": 10.5, + "learning_rate": 4.027777777777779e-06, + "loss": 0.20456252098083497, + "mean_token_accuracy": 0.93275361135602, + "num_tokens": 1581157.0, + "step": 520 + }, + { + "entropy": 0.18834507754072546, + "epoch": 0.2231578947368421, + "grad_norm": 9.5, + "learning_rate": 4.006076388888889e-06, + "loss": 0.1853790521621704, + "mean_token_accuracy": 0.9434393145143986, + "num_tokens": 1612111.0, + "step": 530 + }, + { + "entropy": 0.18060610676184297, + "epoch": 0.22736842105263158, + "grad_norm": 10.875, + "learning_rate": 3.984375e-06, + "loss": 0.18703898191452026, + "mean_token_accuracy": 0.940553817898035, + "num_tokens": 1643714.0, + "step": 540 + }, + { + "entropy": 0.2124157869257033, + "epoch": 0.23157894736842105, + "grad_norm": 16.375, + "learning_rate": 3.9626736111111115e-06, + "loss": 0.23256590366363525, + "mean_token_accuracy": 0.931666910648346, + "num_tokens": 1674981.0, + "step": 550 + }, + { + "entropy": 0.18916954104788603, + "epoch": 0.23578947368421052, + "grad_norm": 8.625, + "learning_rate": 3.940972222222223e-06, + "loss": 0.20892608165740967, + "mean_token_accuracy": 0.9369244277477264, + "num_tokens": 1704650.0, + "step": 560 + }, + { + "entropy": 0.18540409342385827, + "epoch": 0.24, + "grad_norm": 10.75, + "learning_rate": 3.919270833333333e-06, + "loss": 0.1709328293800354, + "mean_token_accuracy": 0.9410766020417214, + "num_tokens": 1736731.0, + "step": 570 + }, + { + "entropy": 0.1986150752287358, + "epoch": 0.24421052631578946, + "grad_norm": 12.3125, + "learning_rate": 3.897569444444444e-06, + "loss": 0.22206108570098876, + "mean_token_accuracy": 0.9357864424586296, + "num_tokens": 1768488.0, + "step": 580 + }, + { + "entropy": 0.1802667098119855, + "epoch": 0.24842105263157896, + "grad_norm": 6.375, + "learning_rate": 3.8758680555555565e-06, + "loss": 0.18710933923721312, + "mean_token_accuracy": 0.9412250213325024, + "num_tokens": 1799166.0, + "step": 590 + }, + { + "entropy": 0.17089016041718424, + "epoch": 0.25263157894736843, + "grad_norm": 12.0, + "learning_rate": 3.854166666666667e-06, + "loss": 0.18744715452194213, + "mean_token_accuracy": 0.9394571520388126, + "num_tokens": 1830122.0, + "step": 600 + }, + { + "entropy": 0.20793215851299465, + "epoch": 0.25684210526315787, + "grad_norm": 7.78125, + "learning_rate": 3.832465277777778e-06, + "loss": 0.234443998336792, + "mean_token_accuracy": 0.9333168506622315, + "num_tokens": 1862016.0, + "step": 610 + }, + { + "entropy": 0.20435764044523239, + "epoch": 0.26105263157894737, + "grad_norm": 5.78125, + "learning_rate": 3.8107638888888894e-06, + "loss": 0.23094916343688965, + "mean_token_accuracy": 0.9356762997806072, + "num_tokens": 1893262.0, + "step": 620 + }, + { + "entropy": 0.16803635912947357, + "epoch": 0.26526315789473687, + "grad_norm": 7.53125, + "learning_rate": 3.7890625e-06, + "loss": 0.17021799087524414, + "mean_token_accuracy": 0.9433802559971809, + "num_tokens": 1926661.0, + "step": 630 + }, + { + "entropy": 0.1832001986913383, + "epoch": 0.2694736842105263, + "grad_norm": 8.5625, + "learning_rate": 3.7673611111111114e-06, + "loss": 0.19909589290618895, + "mean_token_accuracy": 0.9378570787608623, + "num_tokens": 1957342.0, + "step": 640 + }, + { + "entropy": 0.1613461917731911, + "epoch": 0.2736842105263158, + "grad_norm": 6.0, + "learning_rate": 3.7456597222222223e-06, + "loss": 0.16491103172302246, + "mean_token_accuracy": 0.9445122793316841, + "num_tokens": 1988112.0, + "step": 650 + }, + { + "entropy": 0.19069023090414702, + "epoch": 0.27789473684210525, + "grad_norm": 10.8125, + "learning_rate": 3.7239583333333335e-06, + "loss": 0.19739652872085572, + "mean_token_accuracy": 0.9412198103964329, + "num_tokens": 2018814.0, + "step": 660 + }, + { + "entropy": 0.19063587812706828, + "epoch": 0.28210526315789475, + "grad_norm": 6.0, + "learning_rate": 3.7022569444444443e-06, + "loss": 0.19181153774261475, + "mean_token_accuracy": 0.9380034580826759, + "num_tokens": 2049341.0, + "step": 670 + }, + { + "entropy": 0.19904036330990493, + "epoch": 0.2863157894736842, + "grad_norm": 12.5625, + "learning_rate": 3.680555555555556e-06, + "loss": 0.20599710941314697, + "mean_token_accuracy": 0.9368999592959881, + "num_tokens": 2079296.0, + "step": 680 + }, + { + "entropy": 0.19427463631145656, + "epoch": 0.2905263157894737, + "grad_norm": 5.71875, + "learning_rate": 3.6588541666666672e-06, + "loss": 0.18761125802993775, + "mean_token_accuracy": 0.9367304682731629, + "num_tokens": 2109208.0, + "step": 690 + }, + { + "entropy": 0.16103245173580943, + "epoch": 0.29473684210526313, + "grad_norm": 4.03125, + "learning_rate": 3.637152777777778e-06, + "loss": 0.14467405080795287, + "mean_token_accuracy": 0.9430748097598552, + "num_tokens": 2140169.0, + "step": 700 + }, + { + "entropy": 0.18653456503525376, + "epoch": 0.29894736842105263, + "grad_norm": 8.5625, + "learning_rate": 3.6154513888888893e-06, + "loss": 0.19110026359558105, + "mean_token_accuracy": 0.9356968715786934, + "num_tokens": 2169188.0, + "step": 710 + }, + { + "entropy": 0.1778560657054186, + "epoch": 0.3031578947368421, + "grad_norm": 5.84375, + "learning_rate": 3.59375e-06, + "loss": 0.19346174001693725, + "mean_token_accuracy": 0.9388452127575875, + "num_tokens": 2201132.0, + "step": 720 + }, + { + "entropy": 0.1761186286341399, + "epoch": 0.30736842105263157, + "grad_norm": 6.1875, + "learning_rate": 3.5720486111111114e-06, + "loss": 0.17763736248016357, + "mean_token_accuracy": 0.9394733160734177, + "num_tokens": 2231368.0, + "step": 730 + }, + { + "entropy": 0.2061962445732206, + "epoch": 0.31157894736842107, + "grad_norm": 7.28125, + "learning_rate": 3.5503472222222226e-06, + "loss": 0.21218445301055908, + "mean_token_accuracy": 0.9361796006560326, + "num_tokens": 2260621.0, + "step": 740 + }, + { + "entropy": 0.18782778745517134, + "epoch": 0.3157894736842105, + "grad_norm": 8.375, + "learning_rate": 3.5286458333333334e-06, + "loss": 0.21144359111785888, + "mean_token_accuracy": 0.9322924494743348, + "num_tokens": 2292074.0, + "step": 750 + }, + { + "entropy": 0.16837490680627526, + "epoch": 0.32, + "grad_norm": 8.9375, + "learning_rate": 3.5069444444444447e-06, + "loss": 0.17376952171325682, + "mean_token_accuracy": 0.9448926664888859, + "num_tokens": 2321255.0, + "step": 760 + }, + { + "entropy": 0.1900732083246112, + "epoch": 0.32421052631578945, + "grad_norm": 9.75, + "learning_rate": 3.485243055555556e-06, + "loss": 0.21094374656677245, + "mean_token_accuracy": 0.9347058288753033, + "num_tokens": 2352817.0, + "step": 770 + }, + { + "entropy": 0.19717438989318908, + "epoch": 0.32842105263157895, + "grad_norm": 9.125, + "learning_rate": 3.463541666666667e-06, + "loss": 0.19277981519699097, + "mean_token_accuracy": 0.9365353070199489, + "num_tokens": 2382010.0, + "step": 780 + }, + { + "entropy": 0.18062740950845182, + "epoch": 0.33263157894736844, + "grad_norm": 7.21875, + "learning_rate": 3.4418402777777784e-06, + "loss": 0.1980929493904114, + "mean_token_accuracy": 0.9367008984088898, + "num_tokens": 2413871.0, + "step": 790 + }, + { + "entropy": 0.18262718091718852, + "epoch": 0.3368421052631579, + "grad_norm": 11.375, + "learning_rate": 3.420138888888889e-06, + "loss": 0.21326255798339844, + "mean_token_accuracy": 0.939043466001749, + "num_tokens": 2443556.0, + "step": 800 + }, + { + "entropy": 0.1931037397123873, + "epoch": 0.3410526315789474, + "grad_norm": 8.25, + "learning_rate": 3.3984375000000004e-06, + "loss": 0.2050724744796753, + "mean_token_accuracy": 0.9386087417602539, + "num_tokens": 2472709.0, + "step": 810 + }, + { + "entropy": 0.17652380443178117, + "epoch": 0.3452631578947368, + "grad_norm": 9.8125, + "learning_rate": 3.3767361111111113e-06, + "loss": 0.18632233142852783, + "mean_token_accuracy": 0.9410148099064827, + "num_tokens": 2502787.0, + "step": 820 + }, + { + "entropy": 0.18900102768093346, + "epoch": 0.3494736842105263, + "grad_norm": 20.75, + "learning_rate": 3.3550347222222225e-06, + "loss": 0.20314161777496337, + "mean_token_accuracy": 0.9356084302067756, + "num_tokens": 2534024.0, + "step": 830 + }, + { + "entropy": 0.1738276852760464, + "epoch": 0.35368421052631577, + "grad_norm": 10.1875, + "learning_rate": 3.3333333333333333e-06, + "loss": 0.19026525020599366, + "mean_token_accuracy": 0.9386918649077416, + "num_tokens": 2565340.0, + "step": 840 + }, + { + "entropy": 0.17577651813626288, + "epoch": 0.35789473684210527, + "grad_norm": 18.375, + "learning_rate": 3.3116319444444446e-06, + "loss": 0.1828848600387573, + "mean_token_accuracy": 0.9417739301919937, + "num_tokens": 2594835.0, + "step": 850 + }, + { + "entropy": 0.19011623142287135, + "epoch": 0.36210526315789476, + "grad_norm": 10.25, + "learning_rate": 3.2899305555555562e-06, + "loss": 0.20780632495880128, + "mean_token_accuracy": 0.9356612503528595, + "num_tokens": 2625348.0, + "step": 860 + }, + { + "entropy": 0.17401062482967972, + "epoch": 0.3663157894736842, + "grad_norm": 9.5, + "learning_rate": 3.268229166666667e-06, + "loss": 0.19232455492019654, + "mean_token_accuracy": 0.9413411311805249, + "num_tokens": 2656663.0, + "step": 870 + }, + { + "entropy": 0.2042026157025248, + "epoch": 0.3705263157894737, + "grad_norm": 5.625, + "learning_rate": 3.2465277777777783e-06, + "loss": 0.21286656856536865, + "mean_token_accuracy": 0.9318742267787457, + "num_tokens": 2686890.0, + "step": 880 + }, + { + "entropy": 0.18489634501747787, + "epoch": 0.37473684210526315, + "grad_norm": 5.875, + "learning_rate": 3.224826388888889e-06, + "loss": 0.1951343297958374, + "mean_token_accuracy": 0.9396946728229523, + "num_tokens": 2716021.0, + "step": 890 + }, + { + "entropy": 0.1702876826748252, + "epoch": 0.37894736842105264, + "grad_norm": 4.5, + "learning_rate": 3.2031250000000004e-06, + "loss": 0.20316264629364014, + "mean_token_accuracy": 0.9405752472579479, + "num_tokens": 2746087.0, + "step": 900 + }, + { + "entropy": 0.17025947191286833, + "epoch": 0.3831578947368421, + "grad_norm": 5.40625, + "learning_rate": 3.181423611111111e-06, + "loss": 0.1797630548477173, + "mean_token_accuracy": 0.9427746705710888, + "num_tokens": 2775880.0, + "step": 910 + }, + { + "entropy": 0.2002503348980099, + "epoch": 0.3873684210526316, + "grad_norm": 8.6875, + "learning_rate": 3.1597222222222224e-06, + "loss": 0.19509780406951904, + "mean_token_accuracy": 0.9372896805405617, + "num_tokens": 2806354.0, + "step": 920 + }, + { + "entropy": 0.18012529672123492, + "epoch": 0.391578947368421, + "grad_norm": 4.40625, + "learning_rate": 3.1380208333333332e-06, + "loss": 0.1744396448135376, + "mean_token_accuracy": 0.9429976627230644, + "num_tokens": 2838779.0, + "step": 930 + }, + { + "entropy": 0.1940267413854599, + "epoch": 0.3957894736842105, + "grad_norm": 7.25, + "learning_rate": 3.1163194444444445e-06, + "loss": 0.19945181608200074, + "mean_token_accuracy": 0.9355903774499893, + "num_tokens": 2870567.0, + "step": 940 + }, + { + "entropy": 0.1718983714701608, + "epoch": 0.4, + "grad_norm": 11.0, + "learning_rate": 3.094618055555556e-06, + "loss": 0.16918015480041504, + "mean_token_accuracy": 0.9435592129826545, + "num_tokens": 2900643.0, + "step": 950 + }, + { + "entropy": 0.17784137637354433, + "epoch": 0.40421052631578946, + "grad_norm": 7.75, + "learning_rate": 3.072916666666667e-06, + "loss": 0.18423346281051636, + "mean_token_accuracy": 0.9385634325444698, + "num_tokens": 2931200.0, + "step": 960 + }, + { + "entropy": 0.21101772908587008, + "epoch": 0.40842105263157896, + "grad_norm": 11.6875, + "learning_rate": 3.051215277777778e-06, + "loss": 0.2702996253967285, + "mean_token_accuracy": 0.9345157586038113, + "num_tokens": 2962480.0, + "step": 970 + }, + { + "entropy": 0.17290567103773355, + "epoch": 0.4126315789473684, + "grad_norm": 10.8125, + "learning_rate": 3.029513888888889e-06, + "loss": 0.17511870861053466, + "mean_token_accuracy": 0.942404218018055, + "num_tokens": 2993860.0, + "step": 980 + }, + { + "entropy": 0.16266593500040472, + "epoch": 0.4168421052631579, + "grad_norm": 8.3125, + "learning_rate": 3.0078125000000003e-06, + "loss": 0.1581684708595276, + "mean_token_accuracy": 0.9443985551595688, + "num_tokens": 3025733.0, + "step": 990 + }, + { + "entropy": 0.1775346302194521, + "epoch": 0.42105263157894735, + "grad_norm": 4.4375, + "learning_rate": 2.986111111111111e-06, + "loss": 0.18650579452514648, + "mean_token_accuracy": 0.938476724177599, + "num_tokens": 3055975.0, + "step": 1000 + }, + { + "entropy": 0.1736095615196973, + "epoch": 0.42526315789473684, + "grad_norm": 9.125, + "learning_rate": 2.9644097222222223e-06, + "loss": 0.18074491024017333, + "mean_token_accuracy": 0.9398947678506374, + "num_tokens": 3086913.0, + "step": 1010 + }, + { + "entropy": 0.1699895558413118, + "epoch": 0.42947368421052634, + "grad_norm": 8.6875, + "learning_rate": 2.9427083333333336e-06, + "loss": 0.19113610982894896, + "mean_token_accuracy": 0.9418164797127246, + "num_tokens": 3121488.0, + "step": 1020 + }, + { + "entropy": 0.16524689896032213, + "epoch": 0.4336842105263158, + "grad_norm": 8.75, + "learning_rate": 2.9210069444444444e-06, + "loss": 0.185565721988678, + "mean_token_accuracy": 0.9433472394943238, + "num_tokens": 3151445.0, + "step": 1030 + }, + { + "entropy": 0.14927028636448086, + "epoch": 0.4378947368421053, + "grad_norm": 8.1875, + "learning_rate": 2.899305555555556e-06, + "loss": 0.15171147584915162, + "mean_token_accuracy": 0.9442741066217423, + "num_tokens": 3181370.0, + "step": 1040 + }, + { + "entropy": 0.18830096670426427, + "epoch": 0.4421052631578947, + "grad_norm": 8.4375, + "learning_rate": 2.8776041666666673e-06, + "loss": 0.19921324253082276, + "mean_token_accuracy": 0.9361335694789886, + "num_tokens": 3212914.0, + "step": 1050 + }, + { + "entropy": 0.19236768442206084, + "epoch": 0.4463157894736842, + "grad_norm": 6.3125, + "learning_rate": 2.855902777777778e-06, + "loss": 0.20824000835418702, + "mean_token_accuracy": 0.9332416817545891, + "num_tokens": 3243370.0, + "step": 1060 + }, + { + "entropy": 0.19136800640262663, + "epoch": 0.45052631578947366, + "grad_norm": 6.5625, + "learning_rate": 2.8342013888888894e-06, + "loss": 0.1997889280319214, + "mean_token_accuracy": 0.9408642463386059, + "num_tokens": 3273424.0, + "step": 1070 + }, + { + "entropy": 0.17793954727239908, + "epoch": 0.45473684210526316, + "grad_norm": 10.625, + "learning_rate": 2.8125e-06, + "loss": 0.20664634704589843, + "mean_token_accuracy": 0.9388046510517597, + "num_tokens": 3303150.0, + "step": 1080 + }, + { + "entropy": 0.18900100397877395, + "epoch": 0.4589473684210526, + "grad_norm": 8.875, + "learning_rate": 2.7907986111111114e-06, + "loss": 0.2337502956390381, + "mean_token_accuracy": 0.9378764137625695, + "num_tokens": 3332125.0, + "step": 1090 + }, + { + "entropy": 0.2013873849529773, + "epoch": 0.4631578947368421, + "grad_norm": 4.90625, + "learning_rate": 2.7690972222222222e-06, + "loss": 0.2129305362701416, + "mean_token_accuracy": 0.9356573522090912, + "num_tokens": 3360726.0, + "step": 1100 + }, + { + "entropy": 0.16724729198031127, + "epoch": 0.4673684210526316, + "grad_norm": 6.78125, + "learning_rate": 2.7473958333333335e-06, + "loss": 0.17267029285430907, + "mean_token_accuracy": 0.9430627137422561, + "num_tokens": 3392537.0, + "step": 1110 + }, + { + "entropy": 0.1968531704682391, + "epoch": 0.47157894736842104, + "grad_norm": 6.40625, + "learning_rate": 2.7256944444444443e-06, + "loss": 0.2131186008453369, + "mean_token_accuracy": 0.9392065338790416, + "num_tokens": 3421034.0, + "step": 1120 + }, + { + "entropy": 0.18952712146565318, + "epoch": 0.47578947368421054, + "grad_norm": 8.1875, + "learning_rate": 2.703993055555556e-06, + "loss": 0.1882106900215149, + "mean_token_accuracy": 0.9399594850838184, + "num_tokens": 3451236.0, + "step": 1130 + }, + { + "entropy": 0.16791359800845385, + "epoch": 0.48, + "grad_norm": 6.90625, + "learning_rate": 2.682291666666667e-06, + "loss": 0.19798815250396729, + "mean_token_accuracy": 0.9428666599094868, + "num_tokens": 3483037.0, + "step": 1140 + }, + { + "entropy": 0.18943570028059184, + "epoch": 0.4842105263157895, + "grad_norm": 7.0625, + "learning_rate": 2.660590277777778e-06, + "loss": 0.18464272022247313, + "mean_token_accuracy": 0.938535039126873, + "num_tokens": 3512682.0, + "step": 1150 + }, + { + "entropy": 0.18527938476763667, + "epoch": 0.4884210526315789, + "grad_norm": 7.1875, + "learning_rate": 2.6388888888888893e-06, + "loss": 0.21876392364501954, + "mean_token_accuracy": 0.9350791074335575, + "num_tokens": 3542555.0, + "step": 1160 + }, + { + "entropy": 0.17846323440317063, + "epoch": 0.4926315789473684, + "grad_norm": 7.125, + "learning_rate": 2.6171875e-06, + "loss": 0.18390936851501466, + "mean_token_accuracy": 0.9420042358338833, + "num_tokens": 3574058.0, + "step": 1170 + }, + { + "entropy": 0.19397271373309194, + "epoch": 0.4968421052631579, + "grad_norm": 6.0625, + "learning_rate": 2.5954861111111113e-06, + "loss": 0.1811495304107666, + "mean_token_accuracy": 0.9370748721063137, + "num_tokens": 3606475.0, + "step": 1180 + }, + { + "entropy": 0.15141290938481688, + "epoch": 0.5010526315789474, + "grad_norm": 6.5, + "learning_rate": 2.573784722222222e-06, + "loss": 0.15798022747039794, + "mean_token_accuracy": 0.9513927109539508, + "num_tokens": 3638662.0, + "step": 1190 + }, + { + "entropy": 0.18826927775517105, + "epoch": 0.5052631578947369, + "grad_norm": 9.0625, + "learning_rate": 2.5520833333333334e-06, + "loss": 0.18200069665908813, + "mean_token_accuracy": 0.9414383672177792, + "num_tokens": 3669896.0, + "step": 1200 + }, + { + "entropy": 0.20096497626509519, + "epoch": 0.5094736842105263, + "grad_norm": 13.4375, + "learning_rate": 2.530381944444444e-06, + "loss": 0.22587316036224364, + "mean_token_accuracy": 0.9326581291854381, + "num_tokens": 3698316.0, + "step": 1210 + }, + { + "entropy": 0.19803793909959494, + "epoch": 0.5136842105263157, + "grad_norm": 8.625, + "learning_rate": 2.508680555555556e-06, + "loss": 0.21513104438781738, + "mean_token_accuracy": 0.9399372972548008, + "num_tokens": 3728552.0, + "step": 1220 + }, + { + "entropy": 0.16103445165790617, + "epoch": 0.5178947368421053, + "grad_norm": 9.6875, + "learning_rate": 2.4869791666666667e-06, + "loss": 0.17716561555862426, + "mean_token_accuracy": 0.9383736155927181, + "num_tokens": 3761109.0, + "step": 1230 + }, + { + "entropy": 0.17295233584009112, + "epoch": 0.5221052631578947, + "grad_norm": 7.3125, + "learning_rate": 2.465277777777778e-06, + "loss": 0.1894593596458435, + "mean_token_accuracy": 0.9400911360979081, + "num_tokens": 3790881.0, + "step": 1240 + }, + { + "entropy": 0.1784422152210027, + "epoch": 0.5263157894736842, + "grad_norm": 7.15625, + "learning_rate": 2.443576388888889e-06, + "loss": 0.1849764585494995, + "mean_token_accuracy": 0.94148775562644, + "num_tokens": 3820940.0, + "step": 1250 + }, + { + "entropy": 0.2000632504466921, + "epoch": 0.5305263157894737, + "grad_norm": 9.375, + "learning_rate": 2.421875e-06, + "loss": 0.20038533210754395, + "mean_token_accuracy": 0.9400728508830071, + "num_tokens": 3849110.0, + "step": 1260 + }, + { + "entropy": 0.1836847463157028, + "epoch": 0.5347368421052632, + "grad_norm": 13.8125, + "learning_rate": 2.4001736111111112e-06, + "loss": 0.1924800157546997, + "mean_token_accuracy": 0.941128908097744, + "num_tokens": 3879147.0, + "step": 1270 + }, + { + "entropy": 0.16738553042523563, + "epoch": 0.5389473684210526, + "grad_norm": 8.9375, + "learning_rate": 2.3784722222222225e-06, + "loss": 0.16631866693496705, + "mean_token_accuracy": 0.9407841734588146, + "num_tokens": 3912281.0, + "step": 1280 + }, + { + "entropy": 0.18249022141098975, + "epoch": 0.5431578947368421, + "grad_norm": 7.5625, + "learning_rate": 2.3567708333333337e-06, + "loss": 0.20991320610046388, + "mean_token_accuracy": 0.9392150454223156, + "num_tokens": 3942679.0, + "step": 1290 + }, + { + "entropy": 0.19449416091665625, + "epoch": 0.5473684210526316, + "grad_norm": 5.96875, + "learning_rate": 2.3350694444444445e-06, + "loss": 0.20127761363983154, + "mean_token_accuracy": 0.9419910401105881, + "num_tokens": 3973714.0, + "step": 1300 + }, + { + "entropy": 0.18408903060480952, + "epoch": 0.5515789473684211, + "grad_norm": 6.5, + "learning_rate": 2.3133680555555558e-06, + "loss": 0.18167479038238527, + "mean_token_accuracy": 0.940599375218153, + "num_tokens": 4003672.0, + "step": 1310 + }, + { + "entropy": 0.17429547207430005, + "epoch": 0.5557894736842105, + "grad_norm": 6.0, + "learning_rate": 2.2916666666666666e-06, + "loss": 0.17235063314437865, + "mean_token_accuracy": 0.9425551310181618, + "num_tokens": 4033615.0, + "step": 1320 + }, + { + "entropy": 0.19979104902595282, + "epoch": 0.56, + "grad_norm": 7.28125, + "learning_rate": 2.2699652777777783e-06, + "loss": 0.19923921823501586, + "mean_token_accuracy": 0.9354177258908749, + "num_tokens": 4063157.0, + "step": 1330 + }, + { + "entropy": 0.172834698157385, + "epoch": 0.5642105263157895, + "grad_norm": 9.0625, + "learning_rate": 2.248263888888889e-06, + "loss": 0.16350815296173096, + "mean_token_accuracy": 0.9458058536052704, + "num_tokens": 4096030.0, + "step": 1340 + }, + { + "entropy": 0.16383765279315413, + "epoch": 0.5684210526315789, + "grad_norm": 8.5625, + "learning_rate": 2.2265625000000003e-06, + "loss": 0.16651760339736937, + "mean_token_accuracy": 0.9458472564816475, + "num_tokens": 4128179.0, + "step": 1350 + }, + { + "entropy": 0.17017353922128678, + "epoch": 0.5726315789473684, + "grad_norm": 7.40625, + "learning_rate": 2.204861111111111e-06, + "loss": 0.19026347398757934, + "mean_token_accuracy": 0.9381780102849007, + "num_tokens": 4155994.0, + "step": 1360 + }, + { + "entropy": 0.1771279716398567, + "epoch": 0.5768421052631579, + "grad_norm": 5.71875, + "learning_rate": 2.1831597222222224e-06, + "loss": 0.19147398471832275, + "mean_token_accuracy": 0.9430553078651428, + "num_tokens": 4186351.0, + "step": 1370 + }, + { + "entropy": 0.19348430414684117, + "epoch": 0.5810526315789474, + "grad_norm": 7.125, + "learning_rate": 2.1614583333333336e-06, + "loss": 0.1967188000679016, + "mean_token_accuracy": 0.9417100548744202, + "num_tokens": 4217267.0, + "step": 1380 + }, + { + "entropy": 0.1781235427595675, + "epoch": 0.5852631578947368, + "grad_norm": 9.6875, + "learning_rate": 2.1397569444444445e-06, + "loss": 0.1788840651512146, + "mean_token_accuracy": 0.9439761303365231, + "num_tokens": 4245898.0, + "step": 1390 + }, + { + "entropy": 0.18975053504109382, + "epoch": 0.5894736842105263, + "grad_norm": 6.90625, + "learning_rate": 2.1180555555555557e-06, + "loss": 0.20416486263275146, + "mean_token_accuracy": 0.9330022357404232, + "num_tokens": 4276408.0, + "step": 1400 + }, + { + "entropy": 0.15169767290353775, + "epoch": 0.5936842105263158, + "grad_norm": 8.125, + "learning_rate": 2.096354166666667e-06, + "loss": 0.1466461420059204, + "mean_token_accuracy": 0.9468090102076531, + "num_tokens": 4306229.0, + "step": 1410 + }, + { + "entropy": 0.17125667606014758, + "epoch": 0.5978947368421053, + "grad_norm": 11.5625, + "learning_rate": 2.074652777777778e-06, + "loss": 0.17265563011169432, + "mean_token_accuracy": 0.9404310546815395, + "num_tokens": 4336858.0, + "step": 1420 + }, + { + "entropy": 0.18555003069341183, + "epoch": 0.6021052631578947, + "grad_norm": 10.4375, + "learning_rate": 2.052951388888889e-06, + "loss": 0.21136150360107422, + "mean_token_accuracy": 0.9409567311406135, + "num_tokens": 4369258.0, + "step": 1430 + }, + { + "entropy": 0.1963237697724253, + "epoch": 0.6063157894736843, + "grad_norm": 7.78125, + "learning_rate": 2.0312500000000002e-06, + "loss": 0.20975203514099122, + "mean_token_accuracy": 0.9345369815826416, + "num_tokens": 4397776.0, + "step": 1440 + }, + { + "entropy": 0.17916312967427075, + "epoch": 0.6105263157894737, + "grad_norm": 9.375, + "learning_rate": 2.009548611111111e-06, + "loss": 0.18616671562194825, + "mean_token_accuracy": 0.9408248156309128, + "num_tokens": 4427110.0, + "step": 1450 + }, + { + "entropy": 0.16703516799025236, + "epoch": 0.6147368421052631, + "grad_norm": 8.125, + "learning_rate": 1.9878472222222223e-06, + "loss": 0.16765722036361694, + "mean_token_accuracy": 0.9442552007734776, + "num_tokens": 4457255.0, + "step": 1460 + }, + { + "entropy": 0.15452561462298037, + "epoch": 0.6189473684210526, + "grad_norm": 11.875, + "learning_rate": 1.9661458333333335e-06, + "loss": 0.17154412269592284, + "mean_token_accuracy": 0.9427635096013546, + "num_tokens": 4489436.0, + "step": 1470 + }, + { + "entropy": 0.15665105115622283, + "epoch": 0.6231578947368421, + "grad_norm": 11.0625, + "learning_rate": 1.944444444444445e-06, + "loss": 0.14704623222351074, + "mean_token_accuracy": 0.9460532791912556, + "num_tokens": 4519132.0, + "step": 1480 + }, + { + "entropy": 0.16253583596553653, + "epoch": 0.6273684210526316, + "grad_norm": 6.21875, + "learning_rate": 1.9227430555555556e-06, + "loss": 0.1866912603378296, + "mean_token_accuracy": 0.9407590955495835, + "num_tokens": 4550041.0, + "step": 1490 + }, + { + "entropy": 0.16826875344850123, + "epoch": 0.631578947368421, + "grad_norm": 7.40625, + "learning_rate": 1.9010416666666666e-06, + "loss": 0.1752256989479065, + "mean_token_accuracy": 0.9387107148766518, + "num_tokens": 4578389.0, + "step": 1500 + }, + { + "entropy": 0.19358136910013854, + "epoch": 0.6357894736842106, + "grad_norm": 5.625, + "learning_rate": 1.879340277777778e-06, + "loss": 0.21114234924316405, + "mean_token_accuracy": 0.9343519538640976, + "num_tokens": 4607459.0, + "step": 1510 + }, + { + "entropy": 0.17116468152962624, + "epoch": 0.64, + "grad_norm": 7.75, + "learning_rate": 1.8576388888888891e-06, + "loss": 0.17217620611190795, + "mean_token_accuracy": 0.9416831895709038, + "num_tokens": 4638613.0, + "step": 1520 + }, + { + "entropy": 0.2053681869059801, + "epoch": 0.6442105263157895, + "grad_norm": 6.9375, + "learning_rate": 1.8359375000000002e-06, + "loss": 0.22673625946044923, + "mean_token_accuracy": 0.9339988075196743, + "num_tokens": 4666917.0, + "step": 1530 + }, + { + "entropy": 0.1771626771427691, + "epoch": 0.6484210526315789, + "grad_norm": 6.875, + "learning_rate": 1.8142361111111112e-06, + "loss": 0.17564224004745482, + "mean_token_accuracy": 0.9413302429020405, + "num_tokens": 4696949.0, + "step": 1540 + }, + { + "entropy": 0.16223793958779426, + "epoch": 0.6526315789473685, + "grad_norm": 7.4375, + "learning_rate": 1.7925347222222222e-06, + "loss": 0.1730146288871765, + "mean_token_accuracy": 0.9461438663303852, + "num_tokens": 4726446.0, + "step": 1550 + }, + { + "entropy": 0.1785475114127621, + "epoch": 0.6568421052631579, + "grad_norm": 9.4375, + "learning_rate": 1.7708333333333337e-06, + "loss": 0.17630915641784667, + "mean_token_accuracy": 0.941747710108757, + "num_tokens": 4757299.0, + "step": 1560 + }, + { + "entropy": 0.1837959503289312, + "epoch": 0.6610526315789473, + "grad_norm": 6.9375, + "learning_rate": 1.7491319444444447e-06, + "loss": 0.21238627433776855, + "mean_token_accuracy": 0.9363793157041073, + "num_tokens": 4786004.0, + "step": 1570 + }, + { + "entropy": 0.17369262645952405, + "epoch": 0.6652631578947369, + "grad_norm": 8.3125, + "learning_rate": 1.7274305555555557e-06, + "loss": 0.17695680856704712, + "mean_token_accuracy": 0.9470903754234314, + "num_tokens": 4817544.0, + "step": 1580 + }, + { + "entropy": 0.20245264389086515, + "epoch": 0.6694736842105263, + "grad_norm": 7.125, + "learning_rate": 1.7057291666666668e-06, + "loss": 0.2123798370361328, + "mean_token_accuracy": 0.9319963820278645, + "num_tokens": 4847565.0, + "step": 1590 + }, + { + "entropy": 0.16917385840788485, + "epoch": 0.6736842105263158, + "grad_norm": 7.15625, + "learning_rate": 1.684027777777778e-06, + "loss": 0.15817831754684447, + "mean_token_accuracy": 0.9451489493250846, + "num_tokens": 4879212.0, + "step": 1600 + }, + { + "entropy": 0.18626669934019446, + "epoch": 0.6778947368421052, + "grad_norm": 9.25, + "learning_rate": 1.662326388888889e-06, + "loss": 0.1963236451148987, + "mean_token_accuracy": 0.9376043990254402, + "num_tokens": 4911216.0, + "step": 1610 + }, + { + "entropy": 0.1777785701211542, + "epoch": 0.6821052631578948, + "grad_norm": 6.375, + "learning_rate": 1.640625e-06, + "loss": 0.186847722530365, + "mean_token_accuracy": 0.9369994647800922, + "num_tokens": 4943694.0, + "step": 1620 + }, + { + "entropy": 0.19736593994311988, + "epoch": 0.6863157894736842, + "grad_norm": 12.5625, + "learning_rate": 1.618923611111111e-06, + "loss": 0.22199268341064454, + "mean_token_accuracy": 0.9357027292251587, + "num_tokens": 4972032.0, + "step": 1630 + }, + { + "entropy": 0.1932901387102902, + "epoch": 0.6905263157894737, + "grad_norm": 9.9375, + "learning_rate": 1.5972222222222221e-06, + "loss": 0.19801300764083862, + "mean_token_accuracy": 0.9328659147024154, + "num_tokens": 5001985.0, + "step": 1640 + }, + { + "entropy": 0.18727553612552583, + "epoch": 0.6947368421052632, + "grad_norm": 7.25, + "learning_rate": 1.5755208333333336e-06, + "loss": 0.200944185256958, + "mean_token_accuracy": 0.9315911903977394, + "num_tokens": 5031619.0, + "step": 1650 + }, + { + "entropy": 0.18988265902735293, + "epoch": 0.6989473684210527, + "grad_norm": 7.375, + "learning_rate": 1.5538194444444446e-06, + "loss": 0.18995364904403686, + "mean_token_accuracy": 0.9380945399403572, + "num_tokens": 5061525.0, + "step": 1660 + }, + { + "entropy": 0.1723124712705612, + "epoch": 0.7031578947368421, + "grad_norm": 10.0625, + "learning_rate": 1.5321180555555556e-06, + "loss": 0.18223432302474976, + "mean_token_accuracy": 0.936246433109045, + "num_tokens": 5092491.0, + "step": 1670 + }, + { + "entropy": 0.1763459252426401, + "epoch": 0.7073684210526315, + "grad_norm": 12.625, + "learning_rate": 1.5104166666666667e-06, + "loss": 0.19533677101135255, + "mean_token_accuracy": 0.9401971742510795, + "num_tokens": 5123692.0, + "step": 1680 + }, + { + "entropy": 0.17391538694500924, + "epoch": 0.7115789473684211, + "grad_norm": 5.34375, + "learning_rate": 1.488715277777778e-06, + "loss": 0.1833168625831604, + "mean_token_accuracy": 0.9375192701816559, + "num_tokens": 5154327.0, + "step": 1690 + }, + { + "entropy": 0.19447861842345446, + "epoch": 0.7157894736842105, + "grad_norm": 6.34375, + "learning_rate": 1.4670138888888892e-06, + "loss": 0.2035297393798828, + "mean_token_accuracy": 0.9407950587570667, + "num_tokens": 5185441.0, + "step": 1700 + }, + { + "entropy": 0.17104214280843735, + "epoch": 0.72, + "grad_norm": 10.125, + "learning_rate": 1.4453125000000002e-06, + "loss": 0.19187886714935304, + "mean_token_accuracy": 0.9413264997303485, + "num_tokens": 5214618.0, + "step": 1710 + }, + { + "entropy": 0.164881079364568, + "epoch": 0.7242105263157895, + "grad_norm": 10.625, + "learning_rate": 1.4236111111111112e-06, + "loss": 0.17205849885940552, + "mean_token_accuracy": 0.9436387285590172, + "num_tokens": 5244872.0, + "step": 1720 + }, + { + "entropy": 0.18138507837429643, + "epoch": 0.728421052631579, + "grad_norm": 8.4375, + "learning_rate": 1.4019097222222223e-06, + "loss": 0.18944886922836304, + "mean_token_accuracy": 0.9419176429510117, + "num_tokens": 5274811.0, + "step": 1730 + }, + { + "entropy": 0.15619251895695924, + "epoch": 0.7326315789473684, + "grad_norm": 6.25, + "learning_rate": 1.3802083333333335e-06, + "loss": 0.15303416252136232, + "mean_token_accuracy": 0.9456774339079856, + "num_tokens": 5305936.0, + "step": 1740 + }, + { + "entropy": 0.17621302837505937, + "epoch": 0.7368421052631579, + "grad_norm": 8.125, + "learning_rate": 1.3585069444444445e-06, + "loss": 0.16394418478012085, + "mean_token_accuracy": 0.9439542882144452, + "num_tokens": 5336619.0, + "step": 1750 + }, + { + "entropy": 0.1895997554762289, + "epoch": 0.7410526315789474, + "grad_norm": 9.0, + "learning_rate": 1.3368055555555556e-06, + "loss": 0.20419509410858155, + "mean_token_accuracy": 0.9383566424250602, + "num_tokens": 5366166.0, + "step": 1760 + }, + { + "entropy": 0.15430729234358295, + "epoch": 0.7452631578947368, + "grad_norm": 7.96875, + "learning_rate": 1.3151041666666666e-06, + "loss": 0.14866267442703246, + "mean_token_accuracy": 0.9463915720582008, + "num_tokens": 5395903.0, + "step": 1770 + }, + { + "entropy": 0.1961900666821748, + "epoch": 0.7494736842105263, + "grad_norm": 7.53125, + "learning_rate": 1.293402777777778e-06, + "loss": 0.20117769241333008, + "mean_token_accuracy": 0.9342163056135178, + "num_tokens": 5427462.0, + "step": 1780 + }, + { + "entropy": 0.180325382668525, + "epoch": 0.7536842105263157, + "grad_norm": 10.875, + "learning_rate": 1.271701388888889e-06, + "loss": 0.19998364448547362, + "mean_token_accuracy": 0.9385084345936775, + "num_tokens": 5456165.0, + "step": 1790 + }, + { + "entropy": 0.18382872603833675, + "epoch": 0.7578947368421053, + "grad_norm": 6.96875, + "learning_rate": 1.25e-06, + "loss": 0.18795562982559205, + "mean_token_accuracy": 0.9401052162051201, + "num_tokens": 5484297.0, + "step": 1800 + }, + { + "entropy": 0.1735987264662981, + "epoch": 0.7621052631578947, + "grad_norm": 6.75, + "learning_rate": 1.2282986111111111e-06, + "loss": 0.18496283292770385, + "mean_token_accuracy": 0.9364599175751209, + "num_tokens": 5512121.0, + "step": 1810 + }, + { + "entropy": 0.15811273446306587, + "epoch": 0.7663157894736842, + "grad_norm": 8.375, + "learning_rate": 1.2065972222222224e-06, + "loss": 0.17440826892852784, + "mean_token_accuracy": 0.9474552102386952, + "num_tokens": 5540912.0, + "step": 1820 + }, + { + "entropy": 0.19428172940388322, + "epoch": 0.7705263157894737, + "grad_norm": 7.09375, + "learning_rate": 1.1848958333333334e-06, + "loss": 0.2328346014022827, + "mean_token_accuracy": 0.9367604151368141, + "num_tokens": 5570904.0, + "step": 1830 + }, + { + "entropy": 0.19383207242935896, + "epoch": 0.7747368421052632, + "grad_norm": 10.125, + "learning_rate": 1.1631944444444446e-06, + "loss": 0.20834689140319823, + "mean_token_accuracy": 0.9364984571933747, + "num_tokens": 5601902.0, + "step": 1840 + }, + { + "entropy": 0.1639871869701892, + "epoch": 0.7789473684210526, + "grad_norm": 7.21875, + "learning_rate": 1.1414930555555557e-06, + "loss": 0.15975459814071655, + "mean_token_accuracy": 0.9448182679712772, + "num_tokens": 5633013.0, + "step": 1850 + }, + { + "entropy": 0.17671056441031396, + "epoch": 0.783157894736842, + "grad_norm": 6.90625, + "learning_rate": 1.1197916666666667e-06, + "loss": 0.19243409633636474, + "mean_token_accuracy": 0.9421222470700741, + "num_tokens": 5665644.0, + "step": 1860 + }, + { + "entropy": 0.20545928478240966, + "epoch": 0.7873684210526316, + "grad_norm": 10.875, + "learning_rate": 1.098090277777778e-06, + "loss": 0.22782745361328124, + "mean_token_accuracy": 0.9283310487866402, + "num_tokens": 5696733.0, + "step": 1870 + }, + { + "entropy": 0.1771316953469068, + "epoch": 0.791578947368421, + "grad_norm": 12.0, + "learning_rate": 1.076388888888889e-06, + "loss": 0.20005078315734864, + "mean_token_accuracy": 0.9354067780077457, + "num_tokens": 5726809.0, + "step": 1880 + }, + { + "entropy": 0.1829876006115228, + "epoch": 0.7957894736842105, + "grad_norm": 4.375, + "learning_rate": 1.0546875e-06, + "loss": 0.17984312772750854, + "mean_token_accuracy": 0.9406855225563049, + "num_tokens": 5757443.0, + "step": 1890 + }, + { + "entropy": 0.17131042117252945, + "epoch": 0.8, + "grad_norm": 5.5625, + "learning_rate": 1.032986111111111e-06, + "loss": 0.187677800655365, + "mean_token_accuracy": 0.9410165570676327, + "num_tokens": 5787336.0, + "step": 1900 + }, + { + "entropy": 0.15520585421472788, + "epoch": 0.8042105263157895, + "grad_norm": 8.0625, + "learning_rate": 1.0112847222222223e-06, + "loss": 0.15040594339370728, + "mean_token_accuracy": 0.949649342149496, + "num_tokens": 5818737.0, + "step": 1910 + }, + { + "entropy": 0.19390389914624392, + "epoch": 0.8084210526315789, + "grad_norm": 7.875, + "learning_rate": 9.895833333333333e-07, + "loss": 0.19968183040618898, + "mean_token_accuracy": 0.9386158093810082, + "num_tokens": 5848788.0, + "step": 1920 + }, + { + "entropy": 0.16451029586605728, + "epoch": 0.8126315789473684, + "grad_norm": 8.4375, + "learning_rate": 9.678819444444446e-07, + "loss": 0.17298288345336915, + "mean_token_accuracy": 0.9406896255910396, + "num_tokens": 5878423.0, + "step": 1930 + }, + { + "entropy": 0.17370661203749477, + "epoch": 0.8168421052631579, + "grad_norm": 5.59375, + "learning_rate": 9.461805555555556e-07, + "loss": 0.17794467210769654, + "mean_token_accuracy": 0.9431041710078716, + "num_tokens": 5908453.0, + "step": 1940 + }, + { + "entropy": 0.1777374588418752, + "epoch": 0.8210526315789474, + "grad_norm": 11.875, + "learning_rate": 9.244791666666668e-07, + "loss": 0.20215635299682616, + "mean_token_accuracy": 0.9381940357387066, + "num_tokens": 5940889.0, + "step": 1950 + }, + { + "entropy": 0.18253268958069385, + "epoch": 0.8252631578947368, + "grad_norm": 5.5, + "learning_rate": 9.027777777777779e-07, + "loss": 0.17334474325180055, + "mean_token_accuracy": 0.9461725540459156, + "num_tokens": 5971947.0, + "step": 1960 + }, + { + "entropy": 0.1783546233084053, + "epoch": 0.8294736842105264, + "grad_norm": 6.65625, + "learning_rate": 8.81076388888889e-07, + "loss": 0.2018296718597412, + "mean_token_accuracy": 0.9374721601605416, + "num_tokens": 6004716.0, + "step": 1970 + }, + { + "entropy": 0.17445771326310933, + "epoch": 0.8336842105263158, + "grad_norm": 7.0, + "learning_rate": 8.59375e-07, + "loss": 0.16818779706954956, + "mean_token_accuracy": 0.9427588351070881, + "num_tokens": 6036766.0, + "step": 1980 + }, + { + "entropy": 0.18418881273828447, + "epoch": 0.8378947368421052, + "grad_norm": 8.375, + "learning_rate": 8.376736111111112e-07, + "loss": 0.21462113857269288, + "mean_token_accuracy": 0.937469869852066, + "num_tokens": 6065012.0, + "step": 1990 + }, + { + "entropy": 0.16037586382590235, + "epoch": 0.8421052631578947, + "grad_norm": 5.3125, + "learning_rate": 8.159722222222223e-07, + "loss": 0.17553248405456542, + "mean_token_accuracy": 0.9418002314865589, + "num_tokens": 6095994.0, + "step": 2000 + }, + { + "entropy": 0.18084662132896484, + "epoch": 0.8463157894736842, + "grad_norm": 5.34375, + "learning_rate": 7.942708333333333e-07, + "loss": 0.17438472509384156, + "mean_token_accuracy": 0.9446314513683319, + "num_tokens": 6126311.0, + "step": 2010 + }, + { + "entropy": 0.17917919345200062, + "epoch": 0.8505263157894737, + "grad_norm": 8.1875, + "learning_rate": 7.725694444444446e-07, + "loss": 0.18416545391082764, + "mean_token_accuracy": 0.9393374055624009, + "num_tokens": 6155779.0, + "step": 2020 + }, + { + "entropy": 0.17618270772509276, + "epoch": 0.8547368421052631, + "grad_norm": 11.6875, + "learning_rate": 7.508680555555556e-07, + "loss": 0.18657455444335938, + "mean_token_accuracy": 0.9405391819775104, + "num_tokens": 6185940.0, + "step": 2030 + }, + { + "entropy": 0.1900980792939663, + "epoch": 0.8589473684210527, + "grad_norm": 6.34375, + "learning_rate": 7.291666666666667e-07, + "loss": 0.22004635334014894, + "mean_token_accuracy": 0.9368100091814995, + "num_tokens": 6216119.0, + "step": 2040 + }, + { + "entropy": 0.1798396656755358, + "epoch": 0.8631578947368421, + "grad_norm": 7.75, + "learning_rate": 7.074652777777778e-07, + "loss": 0.17679187059402465, + "mean_token_accuracy": 0.9427471734583378, + "num_tokens": 6248190.0, + "step": 2050 + }, + { + "entropy": 0.1862468993291259, + "epoch": 0.8673684210526316, + "grad_norm": 8.25, + "learning_rate": 6.85763888888889e-07, + "loss": 0.19130966663360596, + "mean_token_accuracy": 0.9378809235990048, + "num_tokens": 6282715.0, + "step": 2060 + }, + { + "entropy": 0.18561029192060233, + "epoch": 0.871578947368421, + "grad_norm": 11.25, + "learning_rate": 6.640625e-07, + "loss": 0.18389569520950316, + "mean_token_accuracy": 0.9425614275038242, + "num_tokens": 6312639.0, + "step": 2070 + }, + { + "entropy": 0.17955731307156383, + "epoch": 0.8757894736842106, + "grad_norm": 5.75, + "learning_rate": 6.423611111111111e-07, + "loss": 0.18569419384002686, + "mean_token_accuracy": 0.9404567658901215, + "num_tokens": 6342390.0, + "step": 2080 + }, + { + "entropy": 0.16996940840035676, + "epoch": 0.88, + "grad_norm": 7.53125, + "learning_rate": 6.206597222222223e-07, + "loss": 0.18246359825134278, + "mean_token_accuracy": 0.9428071282804013, + "num_tokens": 6374191.0, + "step": 2090 + }, + { + "entropy": 0.19498055870644748, + "epoch": 0.8842105263157894, + "grad_norm": 7.0625, + "learning_rate": 5.989583333333335e-07, + "loss": 0.20460844039916992, + "mean_token_accuracy": 0.9390896797180176, + "num_tokens": 6403082.0, + "step": 2100 + }, + { + "entropy": 0.15782833809498698, + "epoch": 0.888421052631579, + "grad_norm": 8.8125, + "learning_rate": 5.772569444444445e-07, + "loss": 0.17403767108917237, + "mean_token_accuracy": 0.9430192783474922, + "num_tokens": 6434824.0, + "step": 2110 + }, + { + "entropy": 0.16636770479381086, + "epoch": 0.8926315789473684, + "grad_norm": 4.90625, + "learning_rate": 5.555555555555555e-07, + "loss": 0.16270433664321898, + "mean_token_accuracy": 0.9464202426373959, + "num_tokens": 6464051.0, + "step": 2120 + }, + { + "entropy": 0.18715114260558038, + "epoch": 0.8968421052631579, + "grad_norm": 8.5625, + "learning_rate": 5.338541666666667e-07, + "loss": 0.22732558250427246, + "mean_token_accuracy": 0.9373150266706943, + "num_tokens": 6493853.0, + "step": 2130 + }, + { + "entropy": 0.17378719956614078, + "epoch": 0.9010526315789473, + "grad_norm": 7.28125, + "learning_rate": 5.121527777777778e-07, + "loss": 0.1807337999343872, + "mean_token_accuracy": 0.9400206096470356, + "num_tokens": 6527705.0, + "step": 2140 + }, + { + "entropy": 0.18033494814299048, + "epoch": 0.9052631578947369, + "grad_norm": 5.875, + "learning_rate": 4.904513888888889e-07, + "loss": 0.19068191051483155, + "mean_token_accuracy": 0.937873587757349, + "num_tokens": 6555761.0, + "step": 2150 + }, + { + "entropy": 0.175303529528901, + "epoch": 0.9094736842105263, + "grad_norm": 8.125, + "learning_rate": 4.6875000000000006e-07, + "loss": 0.183641254901886, + "mean_token_accuracy": 0.9391517236828804, + "num_tokens": 6585723.0, + "step": 2160 + }, + { + "entropy": 0.17232977109961212, + "epoch": 0.9136842105263158, + "grad_norm": 3.890625, + "learning_rate": 4.4704861111111115e-07, + "loss": 0.18852951526641845, + "mean_token_accuracy": 0.940208625793457, + "num_tokens": 6615288.0, + "step": 2170 + }, + { + "entropy": 0.16042230552993714, + "epoch": 0.9178947368421052, + "grad_norm": 8.625, + "learning_rate": 4.253472222222223e-07, + "loss": 0.16894659996032715, + "mean_token_accuracy": 0.9419491074979305, + "num_tokens": 6644603.0, + "step": 2180 + }, + { + "entropy": 0.18052552677690983, + "epoch": 0.9221052631578948, + "grad_norm": 9.1875, + "learning_rate": 4.0364583333333337e-07, + "loss": 0.18952767848968505, + "mean_token_accuracy": 0.9380584709346295, + "num_tokens": 6673545.0, + "step": 2190 + }, + { + "entropy": 0.19028411931358277, + "epoch": 0.9263157894736842, + "grad_norm": 10.5625, + "learning_rate": 3.819444444444445e-07, + "loss": 0.18890902996063233, + "mean_token_accuracy": 0.9400756865739822, + "num_tokens": 6703299.0, + "step": 2200 + }, + { + "entropy": 0.16922345554921775, + "epoch": 0.9305263157894736, + "grad_norm": 6.84375, + "learning_rate": 3.6024305555555554e-07, + "loss": 0.17599536180496217, + "mean_token_accuracy": 0.9455938622355461, + "num_tokens": 6732719.0, + "step": 2210 + }, + { + "entropy": 0.15744123989716172, + "epoch": 0.9347368421052632, + "grad_norm": 6.5, + "learning_rate": 3.3854166666666667e-07, + "loss": 0.17910586595535277, + "mean_token_accuracy": 0.9452849693596364, + "num_tokens": 6765329.0, + "step": 2220 + }, + { + "entropy": 0.15953006697818636, + "epoch": 0.9389473684210526, + "grad_norm": 10.5, + "learning_rate": 3.168402777777778e-07, + "loss": 0.16083180904388428, + "mean_token_accuracy": 0.9451860629022122, + "num_tokens": 6796269.0, + "step": 2230 + }, + { + "entropy": 0.19982436632271855, + "epoch": 0.9431578947368421, + "grad_norm": 4.03125, + "learning_rate": 2.951388888888889e-07, + "loss": 0.1909162998199463, + "mean_token_accuracy": 0.9365056939423084, + "num_tokens": 6826464.0, + "step": 2240 + }, + { + "entropy": 0.1634247657377273, + "epoch": 0.9473684210526315, + "grad_norm": 7.75, + "learning_rate": 2.7343750000000003e-07, + "loss": 0.1653286933898926, + "mean_token_accuracy": 0.9445300824940205, + "num_tokens": 6857440.0, + "step": 2250 + }, + { + "entropy": 0.19435078646056353, + "epoch": 0.9515789473684211, + "grad_norm": 6.90625, + "learning_rate": 2.5173611111111117e-07, + "loss": 0.20611786842346191, + "mean_token_accuracy": 0.9351051561534405, + "num_tokens": 6887333.0, + "step": 2260 + }, + { + "entropy": 0.19693100345321, + "epoch": 0.9557894736842105, + "grad_norm": 8.6875, + "learning_rate": 2.3003472222222225e-07, + "loss": 0.1984722852706909, + "mean_token_accuracy": 0.937918345630169, + "num_tokens": 6917794.0, + "step": 2270 + }, + { + "entropy": 0.1728912627324462, + "epoch": 0.96, + "grad_norm": 4.875, + "learning_rate": 2.0833333333333333e-07, + "loss": 0.16558315753936767, + "mean_token_accuracy": 0.9425304509699345, + "num_tokens": 6947614.0, + "step": 2280 + }, + { + "entropy": 0.17391241467557847, + "epoch": 0.9642105263157895, + "grad_norm": 8.0625, + "learning_rate": 1.8663194444444444e-07, + "loss": 0.1844017267227173, + "mean_token_accuracy": 0.9372891336679459, + "num_tokens": 6978068.0, + "step": 2290 + }, + { + "entropy": 0.17354375659488142, + "epoch": 0.968421052631579, + "grad_norm": 7.5, + "learning_rate": 1.6493055555555558e-07, + "loss": 0.17987858057022094, + "mean_token_accuracy": 0.938211838901043, + "num_tokens": 7010536.0, + "step": 2300 + }, + { + "entropy": 0.2068662981968373, + "epoch": 0.9726315789473684, + "grad_norm": 10.0, + "learning_rate": 1.4322916666666666e-07, + "loss": 0.23096110820770263, + "mean_token_accuracy": 0.9326581478118896, + "num_tokens": 7042095.0, + "step": 2310 + }, + { + "entropy": 0.16978850068990142, + "epoch": 0.9768421052631578, + "grad_norm": 8.4375, + "learning_rate": 1.215277777777778e-07, + "loss": 0.17138477563858032, + "mean_token_accuracy": 0.9445147432386876, + "num_tokens": 7072071.0, + "step": 2320 + }, + { + "entropy": 0.17203839742578567, + "epoch": 0.9810526315789474, + "grad_norm": 8.5625, + "learning_rate": 9.982638888888888e-08, + "loss": 0.17553606033325195, + "mean_token_accuracy": 0.9403647750616073, + "num_tokens": 7102822.0, + "step": 2330 + }, + { + "entropy": 0.2015716886613518, + "epoch": 0.9852631578947368, + "grad_norm": 6.46875, + "learning_rate": 7.8125e-08, + "loss": 0.2199774980545044, + "mean_token_accuracy": 0.9335592687129974, + "num_tokens": 7132670.0, + "step": 2340 + }, + { + "entropy": 0.17184048132039606, + "epoch": 0.9894736842105263, + "grad_norm": 10.25, + "learning_rate": 5.6423611111111116e-08, + "loss": 0.19170069694519043, + "mean_token_accuracy": 0.9456814132630825, + "num_tokens": 7163095.0, + "step": 2350 + }, + { + "entropy": 0.18851842815056444, + "epoch": 0.9936842105263158, + "grad_norm": 7.40625, + "learning_rate": 3.472222222222222e-08, + "loss": 0.18714648485183716, + "mean_token_accuracy": 0.9426751539111138, + "num_tokens": 7191956.0, + "step": 2360 + }, + { + "entropy": 0.17078252806095406, + "epoch": 0.9978947368421053, + "grad_norm": 9.5, + "learning_rate": 1.3020833333333333e-08, + "loss": 0.16565700769424438, + "mean_token_accuracy": 0.9381283052265644, + "num_tokens": 7221933.0, + "step": 2370 + } + ], + "logging_steps": 10, + "max_steps": 2375, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 7.608549619134874e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2375/training_args.bin b/checkpoint-2375/training_args.bin new file mode 100644 index 0000000..79984c8 --- /dev/null +++ b/checkpoint-2375/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dff8e567c2d962375693a532e5305e36f6a51c2b316d666c005b3bfe2d70466 +size 5777 diff --git a/checkpoint-2551/chat_template.jinja b/checkpoint-2551/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-2551/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-2551/config.json b/checkpoint-2551/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-2551/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2551/generation_config.json b/checkpoint-2551/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-2551/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-2551/model.safetensors b/checkpoint-2551/model.safetensors new file mode 100644 index 0000000..a158a5a --- /dev/null +++ b/checkpoint-2551/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f77a976b0308720d09d1989a48aee5169ffdcf716f71404832899518b7211c1d +size 3441185608 diff --git a/checkpoint-2551/optimizer.pt b/checkpoint-2551/optimizer.pt new file mode 100644 index 0000000..6399c20 --- /dev/null +++ b/checkpoint-2551/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75cf16cf2a3e37985328f27f6832a03a72d37f25b80a0296c458bc9b296d4936 +size 6882572207 diff --git a/checkpoint-2551/rng_state.pth b/checkpoint-2551/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-2551/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-2551/scheduler.pt b/checkpoint-2551/scheduler.pt new file mode 100644 index 0000000..f92cfc3 --- /dev/null +++ b/checkpoint-2551/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2c66f3faa37cf2bb890385165a08aaa2407ad1ea91ebb077248f7a98cab673a +size 1465 diff --git a/checkpoint-2551/tokenizer.json b/checkpoint-2551/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-2551/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-2551/tokenizer_config.json b/checkpoint-2551/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-2551/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2551/trainer_state.json b/checkpoint-2551/trainer_state.json new file mode 100644 index 0000000..a29123d --- /dev/null +++ b/checkpoint-2551/trainer_state.json @@ -0,0 +1,2584 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 2551, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.40051139052957296, + "epoch": 0.0039204155640497895, + "grad_norm": 30.875, + "learning_rate": 7.058823529411766e-07, + "loss": 1.316241455078125, + "mean_token_accuracy": 0.7522591009736062, + "num_tokens": 130734.0, + "step": 10 + }, + { + "entropy": 0.39800101406872274, + "epoch": 0.007840831128099579, + "grad_norm": 28.875, + "learning_rate": 1.4901960784313726e-06, + "loss": 1.2585247993469237, + "mean_token_accuracy": 0.7558266721665859, + "num_tokens": 267547.0, + "step": 20 + }, + { + "entropy": 0.4350921854376793, + "epoch": 0.011761246692149369, + "grad_norm": 19.25, + "learning_rate": 2.274509803921569e-06, + "loss": 1.2427661895751954, + "mean_token_accuracy": 0.7490861490368843, + "num_tokens": 399201.0, + "step": 30 + }, + { + "entropy": 0.4768921874463558, + "epoch": 0.015681662256199158, + "grad_norm": 12.4375, + "learning_rate": 3.058823529411765e-06, + "loss": 1.1601675987243651, + "mean_token_accuracy": 0.7548606485128403, + "num_tokens": 527876.0, + "step": 40 + }, + { + "entropy": 0.5394699670374393, + "epoch": 0.019602077820248948, + "grad_norm": 10.125, + "learning_rate": 3.843137254901962e-06, + "loss": 1.0834686279296875, + "mean_token_accuracy": 0.7578018061816693, + "num_tokens": 658209.0, + "step": 50 + }, + { + "entropy": 0.5797790199518204, + "epoch": 0.023522493384298737, + "grad_norm": 6.28125, + "learning_rate": 4.627450980392157e-06, + "loss": 0.9067214965820313, + "mean_token_accuracy": 0.7733228243887424, + "num_tokens": 791743.0, + "step": 60 + }, + { + "entropy": 0.6502218697220087, + "epoch": 0.027442908948348527, + "grad_norm": 5.0, + "learning_rate": 5.411764705882353e-06, + "loss": 0.8242039680480957, + "mean_token_accuracy": 0.7773927368223668, + "num_tokens": 922196.0, + "step": 70 + }, + { + "entropy": 0.6219119645655156, + "epoch": 0.031363324512398316, + "grad_norm": 2.953125, + "learning_rate": 6.19607843137255e-06, + "loss": 0.7194723129272461, + "mean_token_accuracy": 0.7994848638772964, + "num_tokens": 1053869.0, + "step": 80 + }, + { + "entropy": 0.6228452675044537, + "epoch": 0.035283740076448106, + "grad_norm": 2.796875, + "learning_rate": 6.9803921568627454e-06, + "loss": 0.6950749397277832, + "mean_token_accuracy": 0.8034804306924344, + "num_tokens": 1179999.0, + "step": 90 + }, + { + "entropy": 0.6416549410670995, + "epoch": 0.039204155640497895, + "grad_norm": 2.890625, + "learning_rate": 7.764705882352941e-06, + "loss": 0.6787714958190918, + "mean_token_accuracy": 0.8052642524242402, + "num_tokens": 1313342.0, + "step": 100 + }, + { + "entropy": 0.6458045944571496, + "epoch": 0.043124571204547685, + "grad_norm": 2.078125, + "learning_rate": 8.549019607843138e-06, + "loss": 0.6732481956481934, + "mean_token_accuracy": 0.803642563521862, + "num_tokens": 1444756.0, + "step": 110 + }, + { + "entropy": 0.634775460511446, + "epoch": 0.047044986768597474, + "grad_norm": 1.9375, + "learning_rate": 9.333333333333334e-06, + "loss": 0.6695634841918945, + "mean_token_accuracy": 0.8045761771500111, + "num_tokens": 1577527.0, + "step": 120 + }, + { + "entropy": 0.6200754787772894, + "epoch": 0.050965402332647264, + "grad_norm": 1.9765625, + "learning_rate": 1.011764705882353e-05, + "loss": 0.6515310764312744, + "mean_token_accuracy": 0.8107773900032044, + "num_tokens": 1712989.0, + "step": 130 + }, + { + "entropy": 0.6283363737165928, + "epoch": 0.05488581789669705, + "grad_norm": 1.859375, + "learning_rate": 1.0901960784313726e-05, + "loss": 0.6463139057159424, + "mean_token_accuracy": 0.8106771603226661, + "num_tokens": 1842532.0, + "step": 140 + }, + { + "entropy": 0.6225632183253765, + "epoch": 0.058806233460746836, + "grad_norm": 1.90625, + "learning_rate": 1.1686274509803922e-05, + "loss": 0.6356926918029785, + "mean_token_accuracy": 0.8131030358374118, + "num_tokens": 1973776.0, + "step": 150 + }, + { + "entropy": 0.5999462876468897, + "epoch": 0.06272664902479663, + "grad_norm": 1.9375, + "learning_rate": 1.2470588235294119e-05, + "loss": 0.6088289737701416, + "mean_token_accuracy": 0.8192408412694931, + "num_tokens": 2106467.0, + "step": 160 + }, + { + "entropy": 0.6009281285107135, + "epoch": 0.06664706458884642, + "grad_norm": 1.96875, + "learning_rate": 1.3254901960784314e-05, + "loss": 0.6100459575653077, + "mean_token_accuracy": 0.8174788504838943, + "num_tokens": 2243425.0, + "step": 170 + }, + { + "entropy": 0.6414772845804692, + "epoch": 0.07056748015289621, + "grad_norm": 2.0625, + "learning_rate": 1.403921568627451e-05, + "loss": 0.6445958614349365, + "mean_token_accuracy": 0.8082262150943279, + "num_tokens": 2376987.0, + "step": 180 + }, + { + "entropy": 0.6355918630957603, + "epoch": 0.074487895716946, + "grad_norm": 1.921875, + "learning_rate": 1.4823529411764707e-05, + "loss": 0.6472668170928955, + "mean_token_accuracy": 0.8071325562894345, + "num_tokens": 2510548.0, + "step": 190 + }, + { + "entropy": 0.6236784052103758, + "epoch": 0.07840831128099579, + "grad_norm": 1.875, + "learning_rate": 1.5607843137254904e-05, + "loss": 0.6356116771697998, + "mean_token_accuracy": 0.8135113798081874, + "num_tokens": 2643661.0, + "step": 200 + }, + { + "entropy": 0.6072400402277708, + "epoch": 0.08232872684504558, + "grad_norm": 1.84375, + "learning_rate": 1.63921568627451e-05, + "loss": 0.6156791687011719, + "mean_token_accuracy": 0.8167861931025981, + "num_tokens": 2774529.0, + "step": 210 + }, + { + "entropy": 0.5829322092235089, + "epoch": 0.08624914240909537, + "grad_norm": 1.8203125, + "learning_rate": 1.7176470588235293e-05, + "loss": 0.5909051418304443, + "mean_token_accuracy": 0.8222801245748996, + "num_tokens": 2908058.0, + "step": 220 + }, + { + "entropy": 0.6099963016808033, + "epoch": 0.09016955797314516, + "grad_norm": 1.6875, + "learning_rate": 1.796078431372549e-05, + "loss": 0.6119081974029541, + "mean_token_accuracy": 0.8179662935435772, + "num_tokens": 3040096.0, + "step": 230 + }, + { + "entropy": 0.6330130327492952, + "epoch": 0.09408997353719495, + "grad_norm": 1.96875, + "learning_rate": 1.8745098039215686e-05, + "loss": 0.6453900814056397, + "mean_token_accuracy": 0.8088094264268875, + "num_tokens": 3171172.0, + "step": 240 + }, + { + "entropy": 0.6220353674143553, + "epoch": 0.09801038910124474, + "grad_norm": 1.9609375, + "learning_rate": 1.9529411764705885e-05, + "loss": 0.6278114318847656, + "mean_token_accuracy": 0.8124852173030377, + "num_tokens": 3304081.0, + "step": 250 + }, + { + "entropy": 0.6234287895262242, + "epoch": 0.10193080466529453, + "grad_norm": 1.8359375, + "learning_rate": 1.996515679442509e-05, + "loss": 0.6276582717895508, + "mean_token_accuracy": 0.8146361976861953, + "num_tokens": 3438063.0, + "step": 260 + }, + { + "entropy": 0.6037415701895952, + "epoch": 0.10585122022934432, + "grad_norm": 1.8125, + "learning_rate": 1.9878048780487806e-05, + "loss": 0.6147459506988525, + "mean_token_accuracy": 0.8160658605396748, + "num_tokens": 3572684.0, + "step": 270 + }, + { + "entropy": 0.6154889557510614, + "epoch": 0.1097716357933941, + "grad_norm": 1.828125, + "learning_rate": 1.9790940766550523e-05, + "loss": 0.620861291885376, + "mean_token_accuracy": 0.8154690660536289, + "num_tokens": 3706498.0, + "step": 280 + }, + { + "entropy": 0.5835129704326392, + "epoch": 0.11369205135744388, + "grad_norm": 1.828125, + "learning_rate": 1.970383275261324e-05, + "loss": 0.5846771240234375, + "mean_token_accuracy": 0.8243817768990993, + "num_tokens": 3837023.0, + "step": 290 + }, + { + "entropy": 0.5867879837751389, + "epoch": 0.11761246692149367, + "grad_norm": 1.828125, + "learning_rate": 1.961672473867596e-05, + "loss": 0.5936897277832032, + "mean_token_accuracy": 0.8227211274206638, + "num_tokens": 3966412.0, + "step": 300 + }, + { + "entropy": 0.6125645771622658, + "epoch": 0.12153288248554346, + "grad_norm": 1.6953125, + "learning_rate": 1.9529616724738677e-05, + "loss": 0.6198267459869384, + "mean_token_accuracy": 0.8152951590716839, + "num_tokens": 4096711.0, + "step": 310 + }, + { + "entropy": 0.610775976255536, + "epoch": 0.12545329804959326, + "grad_norm": 1.9296875, + "learning_rate": 1.9442508710801397e-05, + "loss": 0.6171989917755127, + "mean_token_accuracy": 0.815869303047657, + "num_tokens": 4228066.0, + "step": 320 + }, + { + "entropy": 0.6040709633380175, + "epoch": 0.12937371361364305, + "grad_norm": 1.9609375, + "learning_rate": 1.9355400696864114e-05, + "loss": 0.6103555202484131, + "mean_token_accuracy": 0.8194533243775368, + "num_tokens": 4357664.0, + "step": 330 + }, + { + "entropy": 0.6143804207444191, + "epoch": 0.13329412917769284, + "grad_norm": 1.8828125, + "learning_rate": 1.926829268292683e-05, + "loss": 0.6221739768981933, + "mean_token_accuracy": 0.8153439976274968, + "num_tokens": 4487688.0, + "step": 340 + }, + { + "entropy": 0.6031508389860392, + "epoch": 0.13721454474174263, + "grad_norm": 1.84375, + "learning_rate": 1.9181184668989547e-05, + "loss": 0.6023824214935303, + "mean_token_accuracy": 0.8190992563962937, + "num_tokens": 4619172.0, + "step": 350 + }, + { + "entropy": 0.6009325047954917, + "epoch": 0.14113496030579242, + "grad_norm": 1.8125, + "learning_rate": 1.9094076655052267e-05, + "loss": 0.6117077827453613, + "mean_token_accuracy": 0.8167035676538944, + "num_tokens": 4749078.0, + "step": 360 + }, + { + "entropy": 0.6013668723404407, + "epoch": 0.1450553758698422, + "grad_norm": 1.6875, + "learning_rate": 1.9006968641114984e-05, + "loss": 0.5986335277557373, + "mean_token_accuracy": 0.818621464818716, + "num_tokens": 4881903.0, + "step": 370 + }, + { + "entropy": 0.5910887397825718, + "epoch": 0.148975791433892, + "grad_norm": 1.7578125, + "learning_rate": 1.89198606271777e-05, + "loss": 0.5908828735351562, + "mean_token_accuracy": 0.8213229507207871, + "num_tokens": 5015453.0, + "step": 380 + }, + { + "entropy": 0.5584426306188106, + "epoch": 0.1528962069979418, + "grad_norm": 1.7265625, + "learning_rate": 1.8832752613240418e-05, + "loss": 0.5678494930267334, + "mean_token_accuracy": 0.8277932897210121, + "num_tokens": 5147133.0, + "step": 390 + }, + { + "entropy": 0.5676757667213679, + "epoch": 0.15681662256199158, + "grad_norm": 1.6640625, + "learning_rate": 1.8745644599303138e-05, + "loss": 0.5685997009277344, + "mean_token_accuracy": 0.8280104413628578, + "num_tokens": 5278668.0, + "step": 400 + }, + { + "entropy": 0.5922414932399989, + "epoch": 0.16073703812604137, + "grad_norm": 1.765625, + "learning_rate": 1.8658536585365855e-05, + "loss": 0.5975120067596436, + "mean_token_accuracy": 0.8212826780974865, + "num_tokens": 5411982.0, + "step": 410 + }, + { + "entropy": 0.6230888426303863, + "epoch": 0.16465745369009116, + "grad_norm": 1.84375, + "learning_rate": 1.8571428571428575e-05, + "loss": 0.6278081417083741, + "mean_token_accuracy": 0.8109267845749855, + "num_tokens": 5546647.0, + "step": 420 + }, + { + "entropy": 0.5828145634382963, + "epoch": 0.16857786925414095, + "grad_norm": 1.765625, + "learning_rate": 1.8484320557491292e-05, + "loss": 0.5854227542877197, + "mean_token_accuracy": 0.8228457011282444, + "num_tokens": 5679428.0, + "step": 430 + }, + { + "entropy": 0.5902278333902359, + "epoch": 0.17249828481819074, + "grad_norm": 1.75, + "learning_rate": 1.839721254355401e-05, + "loss": 0.5893653869628906, + "mean_token_accuracy": 0.8216007687151432, + "num_tokens": 5811806.0, + "step": 440 + }, + { + "entropy": 0.5815000183880329, + "epoch": 0.17641870038224053, + "grad_norm": 1.828125, + "learning_rate": 1.8310104529616726e-05, + "loss": 0.5896779060363769, + "mean_token_accuracy": 0.8225190363824367, + "num_tokens": 5945059.0, + "step": 450 + }, + { + "entropy": 0.6150704674422741, + "epoch": 0.18033911594629032, + "grad_norm": 1.828125, + "learning_rate": 1.8222996515679442e-05, + "loss": 0.6127326965332032, + "mean_token_accuracy": 0.8161274991929531, + "num_tokens": 6075109.0, + "step": 460 + }, + { + "entropy": 0.5994422011077404, + "epoch": 0.1842595315103401, + "grad_norm": 1.578125, + "learning_rate": 1.8135888501742163e-05, + "loss": 0.6054869174957276, + "mean_token_accuracy": 0.8187170140445232, + "num_tokens": 6206390.0, + "step": 470 + }, + { + "entropy": 0.5805528864264489, + "epoch": 0.1881799470743899, + "grad_norm": 1.875, + "learning_rate": 1.804878048780488e-05, + "loss": 0.5855868339538575, + "mean_token_accuracy": 0.8244414009153843, + "num_tokens": 6343008.0, + "step": 480 + }, + { + "entropy": 0.591498001664877, + "epoch": 0.1921003626384397, + "grad_norm": 1.7578125, + "learning_rate": 1.7961672473867596e-05, + "loss": 0.5893340587615967, + "mean_token_accuracy": 0.8216389425098896, + "num_tokens": 6474582.0, + "step": 490 + }, + { + "entropy": 0.626478412002325, + "epoch": 0.19602077820248948, + "grad_norm": 1.921875, + "learning_rate": 1.7874564459930313e-05, + "loss": 0.6263772487640381, + "mean_token_accuracy": 0.8120981454849243, + "num_tokens": 6607442.0, + "step": 500 + }, + { + "entropy": 0.5900808859616518, + "epoch": 0.19994119376653927, + "grad_norm": 1.5546875, + "learning_rate": 1.7787456445993033e-05, + "loss": 0.6009780883789062, + "mean_token_accuracy": 0.819120641052723, + "num_tokens": 6734350.0, + "step": 510 + }, + { + "entropy": 0.600034212693572, + "epoch": 0.20386160933058906, + "grad_norm": 1.78125, + "learning_rate": 1.770034843205575e-05, + "loss": 0.5948196887969971, + "mean_token_accuracy": 0.8216341696679592, + "num_tokens": 6869288.0, + "step": 520 + }, + { + "entropy": 0.5796573795378208, + "epoch": 0.20778202489463884, + "grad_norm": 1.9296875, + "learning_rate": 1.761324041811847e-05, + "loss": 0.5877546787261962, + "mean_token_accuracy": 0.8229651033878327, + "num_tokens": 7002603.0, + "step": 530 + }, + { + "entropy": 0.5981475539505482, + "epoch": 0.21170244045868863, + "grad_norm": 1.734375, + "learning_rate": 1.7526132404181187e-05, + "loss": 0.5986037254333496, + "mean_token_accuracy": 0.8196288175880909, + "num_tokens": 7135386.0, + "step": 540 + }, + { + "entropy": 0.5765955623239278, + "epoch": 0.21562285602273842, + "grad_norm": 1.6484375, + "learning_rate": 1.7439024390243904e-05, + "loss": 0.5820430278778076, + "mean_token_accuracy": 0.8261640995740891, + "num_tokens": 7269113.0, + "step": 550 + }, + { + "entropy": 0.5835400529205799, + "epoch": 0.2195432715867882, + "grad_norm": 1.96875, + "learning_rate": 1.735191637630662e-05, + "loss": 0.5847792148590087, + "mean_token_accuracy": 0.8227986313402653, + "num_tokens": 7397199.0, + "step": 560 + }, + { + "entropy": 0.5926254410296679, + "epoch": 0.22346368715083798, + "grad_norm": 1.6640625, + "learning_rate": 1.7264808362369338e-05, + "loss": 0.5946429252624512, + "mean_token_accuracy": 0.8213822573423386, + "num_tokens": 7528883.0, + "step": 570 + }, + { + "entropy": 0.5802333429455757, + "epoch": 0.22738410271488776, + "grad_norm": 1.8203125, + "learning_rate": 1.7177700348432058e-05, + "loss": 0.5808559417724609, + "mean_token_accuracy": 0.8252004392445087, + "num_tokens": 7661460.0, + "step": 580 + }, + { + "entropy": 0.5792893666774035, + "epoch": 0.23130451827893755, + "grad_norm": 1.78125, + "learning_rate": 1.7090592334494775e-05, + "loss": 0.5844778060913086, + "mean_token_accuracy": 0.8232058539986611, + "num_tokens": 7792654.0, + "step": 590 + }, + { + "entropy": 0.5875658553093672, + "epoch": 0.23522493384298734, + "grad_norm": 1.6328125, + "learning_rate": 1.700348432055749e-05, + "loss": 0.5934580802917481, + "mean_token_accuracy": 0.8228402160108089, + "num_tokens": 7929712.0, + "step": 600 + }, + { + "entropy": 0.6018010344356298, + "epoch": 0.23914534940703713, + "grad_norm": 1.8125, + "learning_rate": 1.691637630662021e-05, + "loss": 0.6018301010131836, + "mean_token_accuracy": 0.8196375787258148, + "num_tokens": 8062790.0, + "step": 610 + }, + { + "entropy": 0.5656840804964304, + "epoch": 0.24306576497108692, + "grad_norm": 1.65625, + "learning_rate": 1.682926829268293e-05, + "loss": 0.5681605339050293, + "mean_token_accuracy": 0.829612398892641, + "num_tokens": 8195902.0, + "step": 620 + }, + { + "entropy": 0.590726625546813, + "epoch": 0.2469861805351367, + "grad_norm": 1.7265625, + "learning_rate": 1.6742160278745645e-05, + "loss": 0.5950279712677002, + "mean_token_accuracy": 0.8214106187224388, + "num_tokens": 8326346.0, + "step": 630 + }, + { + "entropy": 0.5998687874525785, + "epoch": 0.25090659609918653, + "grad_norm": 1.734375, + "learning_rate": 1.6655052264808366e-05, + "loss": 0.5983600616455078, + "mean_token_accuracy": 0.8189966283738613, + "num_tokens": 8460029.0, + "step": 640 + }, + { + "entropy": 0.5715964876115323, + "epoch": 0.2548270116632363, + "grad_norm": 1.8359375, + "learning_rate": 1.6567944250871082e-05, + "loss": 0.5704405307769775, + "mean_token_accuracy": 0.8268976680934429, + "num_tokens": 8593771.0, + "step": 650 + }, + { + "entropy": 0.5904807798564434, + "epoch": 0.2587474272272861, + "grad_norm": 1.7265625, + "learning_rate": 1.64808362369338e-05, + "loss": 0.5930946350097657, + "mean_token_accuracy": 0.8214319236576557, + "num_tokens": 8727138.0, + "step": 660 + }, + { + "entropy": 0.587051372230053, + "epoch": 0.26266784279133587, + "grad_norm": 1.9296875, + "learning_rate": 1.6393728222996516e-05, + "loss": 0.5872443675994873, + "mean_token_accuracy": 0.8230736367404461, + "num_tokens": 8859295.0, + "step": 670 + }, + { + "entropy": 0.5858452804386616, + "epoch": 0.2665882583553857, + "grad_norm": 1.6640625, + "learning_rate": 1.6306620209059233e-05, + "loss": 0.5850666522979736, + "mean_token_accuracy": 0.824101684987545, + "num_tokens": 8990881.0, + "step": 680 + }, + { + "entropy": 0.5871569372713565, + "epoch": 0.27050867391943545, + "grad_norm": 1.640625, + "learning_rate": 1.6219512195121953e-05, + "loss": 0.594059944152832, + "mean_token_accuracy": 0.8213366828858852, + "num_tokens": 9123499.0, + "step": 690 + }, + { + "entropy": 0.5705580234527587, + "epoch": 0.27442908948348527, + "grad_norm": 1.828125, + "learning_rate": 1.613240418118467e-05, + "loss": 0.578093433380127, + "mean_token_accuracy": 0.8268906474113464, + "num_tokens": 9257724.0, + "step": 700 + }, + { + "entropy": 0.5944200098514557, + "epoch": 0.27834950504753503, + "grad_norm": 1.8671875, + "learning_rate": 1.604529616724739e-05, + "loss": 0.5930802345275878, + "mean_token_accuracy": 0.8196114718914032, + "num_tokens": 9393701.0, + "step": 710 + }, + { + "entropy": 0.5565787792205811, + "epoch": 0.28226992061158485, + "grad_norm": 1.7265625, + "learning_rate": 1.5958188153310107e-05, + "loss": 0.556602954864502, + "mean_token_accuracy": 0.8305394992232322, + "num_tokens": 9525747.0, + "step": 720 + }, + { + "entropy": 0.5937954898923635, + "epoch": 0.2861903361756346, + "grad_norm": 1.78125, + "learning_rate": 1.5871080139372824e-05, + "loss": 0.6040976524353028, + "mean_token_accuracy": 0.819646991044283, + "num_tokens": 9656394.0, + "step": 730 + }, + { + "entropy": 0.5655720047652721, + "epoch": 0.2901107517396844, + "grad_norm": 1.515625, + "learning_rate": 1.578397212543554e-05, + "loss": 0.5677313804626465, + "mean_token_accuracy": 0.8287642747163773, + "num_tokens": 9786846.0, + "step": 740 + }, + { + "entropy": 0.5690872885286808, + "epoch": 0.2940311673037342, + "grad_norm": 1.59375, + "learning_rate": 1.569686411149826e-05, + "loss": 0.577194356918335, + "mean_token_accuracy": 0.8268992677330971, + "num_tokens": 9920679.0, + "step": 750 + }, + { + "entropy": 0.5751310177147388, + "epoch": 0.297951582867784, + "grad_norm": 1.609375, + "learning_rate": 1.5609756097560978e-05, + "loss": 0.5761964797973633, + "mean_token_accuracy": 0.8257504008710385, + "num_tokens": 10052844.0, + "step": 760 + }, + { + "entropy": 0.5573539979755878, + "epoch": 0.30187199843183377, + "grad_norm": 1.71875, + "learning_rate": 1.5522648083623694e-05, + "loss": 0.5600615978240967, + "mean_token_accuracy": 0.8300629191100597, + "num_tokens": 10183471.0, + "step": 770 + }, + { + "entropy": 0.5975829754024744, + "epoch": 0.3057924139958836, + "grad_norm": 1.703125, + "learning_rate": 1.543554006968641e-05, + "loss": 0.5992860317230224, + "mean_token_accuracy": 0.8206060849130153, + "num_tokens": 10313392.0, + "step": 780 + }, + { + "entropy": 0.5773690041154623, + "epoch": 0.30971282955993334, + "grad_norm": 1.75, + "learning_rate": 1.5348432055749128e-05, + "loss": 0.5855609893798828, + "mean_token_accuracy": 0.8234818607568741, + "num_tokens": 10448285.0, + "step": 790 + }, + { + "entropy": 0.5769198387861252, + "epoch": 0.31363324512398316, + "grad_norm": 1.6484375, + "learning_rate": 1.5261324041811848e-05, + "loss": 0.5776193618774415, + "mean_token_accuracy": 0.8255582615733147, + "num_tokens": 10584236.0, + "step": 800 + }, + { + "entropy": 0.5881247483193874, + "epoch": 0.3175536606880329, + "grad_norm": 1.7734375, + "learning_rate": 1.5174216027874567e-05, + "loss": 0.593212080001831, + "mean_token_accuracy": 0.8212029553949833, + "num_tokens": 10716890.0, + "step": 810 + }, + { + "entropy": 0.5945972129702568, + "epoch": 0.32147407625208274, + "grad_norm": 1.6171875, + "learning_rate": 1.5087108013937284e-05, + "loss": 0.5877634525299072, + "mean_token_accuracy": 0.8238732725381851, + "num_tokens": 10852266.0, + "step": 820 + }, + { + "entropy": 0.5811539199203253, + "epoch": 0.3253944918161325, + "grad_norm": 1.796875, + "learning_rate": 1.5000000000000002e-05, + "loss": 0.5907205104827881, + "mean_token_accuracy": 0.8204894207417965, + "num_tokens": 10985262.0, + "step": 830 + }, + { + "entropy": 0.5849005732685327, + "epoch": 0.3293149073801823, + "grad_norm": 1.5625, + "learning_rate": 1.4912891986062719e-05, + "loss": 0.5833800792694092, + "mean_token_accuracy": 0.8241490855813026, + "num_tokens": 11112681.0, + "step": 840 + }, + { + "entropy": 0.5829326704144477, + "epoch": 0.3332353229442321, + "grad_norm": 1.78125, + "learning_rate": 1.4825783972125436e-05, + "loss": 0.5943526268005371, + "mean_token_accuracy": 0.8208344720304013, + "num_tokens": 11240261.0, + "step": 850 + }, + { + "entropy": 0.6181501217186451, + "epoch": 0.3371557385082819, + "grad_norm": 1.7109375, + "learning_rate": 1.4738675958188156e-05, + "loss": 0.6180446147918701, + "mean_token_accuracy": 0.8148257359862328, + "num_tokens": 11370906.0, + "step": 860 + }, + { + "entropy": 0.5804374283179641, + "epoch": 0.34107615407233166, + "grad_norm": 1.65625, + "learning_rate": 1.4651567944250873e-05, + "loss": 0.5808406829833984, + "mean_token_accuracy": 0.8237656883895397, + "num_tokens": 11502338.0, + "step": 870 + }, + { + "entropy": 0.5756821602582931, + "epoch": 0.3449965696363815, + "grad_norm": 1.703125, + "learning_rate": 1.456445993031359e-05, + "loss": 0.5798084735870361, + "mean_token_accuracy": 0.8261652231216431, + "num_tokens": 11632165.0, + "step": 880 + }, + { + "entropy": 0.6068328361958265, + "epoch": 0.34891698520043124, + "grad_norm": 1.671875, + "learning_rate": 1.4477351916376308e-05, + "loss": 0.6065092563629151, + "mean_token_accuracy": 0.8196589723229408, + "num_tokens": 11763511.0, + "step": 890 + }, + { + "entropy": 0.5891778867691755, + "epoch": 0.35283740076448106, + "grad_norm": 1.609375, + "learning_rate": 1.4390243902439025e-05, + "loss": 0.592728567123413, + "mean_token_accuracy": 0.8230046413838863, + "num_tokens": 11895820.0, + "step": 900 + }, + { + "entropy": 0.6052472297102213, + "epoch": 0.3567578163285308, + "grad_norm": 1.7265625, + "learning_rate": 1.4303135888501742e-05, + "loss": 0.6157879829406738, + "mean_token_accuracy": 0.8158482946455479, + "num_tokens": 12025859.0, + "step": 910 + }, + { + "entropy": 0.5880816575139761, + "epoch": 0.36067823189258064, + "grad_norm": 1.828125, + "learning_rate": 1.4216027874564462e-05, + "loss": 0.5871949672698975, + "mean_token_accuracy": 0.8238230250775814, + "num_tokens": 12154614.0, + "step": 920 + }, + { + "entropy": 0.5858425028622151, + "epoch": 0.3645986474566304, + "grad_norm": 1.65625, + "learning_rate": 1.4128919860627179e-05, + "loss": 0.5850693225860596, + "mean_token_accuracy": 0.8225606501102447, + "num_tokens": 12289271.0, + "step": 930 + }, + { + "entropy": 0.6006566017866135, + "epoch": 0.3685190630206802, + "grad_norm": 1.765625, + "learning_rate": 1.4041811846689897e-05, + "loss": 0.6035278797149658, + "mean_token_accuracy": 0.8171994499862194, + "num_tokens": 12419706.0, + "step": 940 + }, + { + "entropy": 0.5940211407840252, + "epoch": 0.37243947858473, + "grad_norm": 1.9375, + "learning_rate": 1.3954703832752614e-05, + "loss": 0.5872697830200195, + "mean_token_accuracy": 0.8215577825903893, + "num_tokens": 12551888.0, + "step": 950 + }, + { + "entropy": 0.5769729625433684, + "epoch": 0.3763598941487798, + "grad_norm": 1.78125, + "learning_rate": 1.3867595818815331e-05, + "loss": 0.5793991088867188, + "mean_token_accuracy": 0.8250952236354351, + "num_tokens": 12685739.0, + "step": 960 + }, + { + "entropy": 0.5762162335216999, + "epoch": 0.38028030971282956, + "grad_norm": 1.640625, + "learning_rate": 1.378048780487805e-05, + "loss": 0.5773720741271973, + "mean_token_accuracy": 0.8255156740546227, + "num_tokens": 12817700.0, + "step": 970 + }, + { + "entropy": 0.5669316282495857, + "epoch": 0.3842007252768794, + "grad_norm": 1.78125, + "learning_rate": 1.3693379790940768e-05, + "loss": 0.5728450775146484, + "mean_token_accuracy": 0.8277040965855121, + "num_tokens": 12949817.0, + "step": 980 + }, + { + "entropy": 0.5594307694584131, + "epoch": 0.38812114084092914, + "grad_norm": 1.796875, + "learning_rate": 1.3606271777003486e-05, + "loss": 0.5640110015869141, + "mean_token_accuracy": 0.8282136105000972, + "num_tokens": 13081117.0, + "step": 990 + }, + { + "entropy": 0.5791688058525324, + "epoch": 0.39204155640497895, + "grad_norm": 1.7578125, + "learning_rate": 1.3519163763066203e-05, + "loss": 0.5831320285797119, + "mean_token_accuracy": 0.8250723823904991, + "num_tokens": 13210883.0, + "step": 1000 + }, + { + "entropy": 0.5651333026587964, + "epoch": 0.3959619719690287, + "grad_norm": 1.5859375, + "learning_rate": 1.343205574912892e-05, + "loss": 0.5608913898468018, + "mean_token_accuracy": 0.8293542221188546, + "num_tokens": 13341155.0, + "step": 1010 + }, + { + "entropy": 0.5587001539766788, + "epoch": 0.39988238753307853, + "grad_norm": 1.59375, + "learning_rate": 1.3344947735191639e-05, + "loss": 0.5611968040466309, + "mean_token_accuracy": 0.8305698707699776, + "num_tokens": 13474505.0, + "step": 1020 + }, + { + "entropy": 0.5640604499727487, + "epoch": 0.4038028030971283, + "grad_norm": 1.796875, + "learning_rate": 1.3257839721254357e-05, + "loss": 0.565390157699585, + "mean_token_accuracy": 0.8282499298453331, + "num_tokens": 13607139.0, + "step": 1030 + }, + { + "entropy": 0.5637519735842943, + "epoch": 0.4077232186611781, + "grad_norm": 1.6484375, + "learning_rate": 1.3170731707317076e-05, + "loss": 0.5695384502410888, + "mean_token_accuracy": 0.8287955388426781, + "num_tokens": 13738326.0, + "step": 1040 + }, + { + "entropy": 0.580963845923543, + "epoch": 0.4116436342252279, + "grad_norm": 1.765625, + "learning_rate": 1.3083623693379792e-05, + "loss": 0.5769538402557373, + "mean_token_accuracy": 0.8235732421278954, + "num_tokens": 13868452.0, + "step": 1050 + }, + { + "entropy": 0.5678158435970545, + "epoch": 0.4155640497892777, + "grad_norm": 1.8125, + "learning_rate": 1.299651567944251e-05, + "loss": 0.5710372924804688, + "mean_token_accuracy": 0.8272775359451771, + "num_tokens": 14002979.0, + "step": 1060 + }, + { + "entropy": 0.5668239049613476, + "epoch": 0.41948446535332745, + "grad_norm": 1.6328125, + "learning_rate": 1.2909407665505226e-05, + "loss": 0.5655869483947754, + "mean_token_accuracy": 0.8278685718774795, + "num_tokens": 14138341.0, + "step": 1070 + }, + { + "entropy": 0.5467250619083643, + "epoch": 0.42340488091737727, + "grad_norm": 1.75, + "learning_rate": 1.2822299651567945e-05, + "loss": 0.5479381561279297, + "mean_token_accuracy": 0.8330328531563282, + "num_tokens": 14266623.0, + "step": 1080 + }, + { + "entropy": 0.574169309064746, + "epoch": 0.42732529648142703, + "grad_norm": 1.578125, + "learning_rate": 1.2735191637630663e-05, + "loss": 0.580225658416748, + "mean_token_accuracy": 0.8270016670227051, + "num_tokens": 14397016.0, + "step": 1090 + }, + { + "entropy": 0.5842641271650791, + "epoch": 0.43124571204547685, + "grad_norm": 1.6171875, + "learning_rate": 1.2648083623693382e-05, + "loss": 0.5883337020874023, + "mean_token_accuracy": 0.8239847645163536, + "num_tokens": 14530946.0, + "step": 1100 + }, + { + "entropy": 0.5795733086764813, + "epoch": 0.4351661276095266, + "grad_norm": 1.71875, + "learning_rate": 1.2560975609756098e-05, + "loss": 0.5835409164428711, + "mean_token_accuracy": 0.8237806595861912, + "num_tokens": 14668493.0, + "step": 1110 + }, + { + "entropy": 0.5661398351192475, + "epoch": 0.4390865431735764, + "grad_norm": 1.8203125, + "learning_rate": 1.2473867595818815e-05, + "loss": 0.5688445091247558, + "mean_token_accuracy": 0.8283659070730209, + "num_tokens": 14798023.0, + "step": 1120 + }, + { + "entropy": 0.5716561190783978, + "epoch": 0.4430069587376262, + "grad_norm": 1.734375, + "learning_rate": 1.2386759581881534e-05, + "loss": 0.5737256526947021, + "mean_token_accuracy": 0.8261031933128834, + "num_tokens": 14927026.0, + "step": 1130 + }, + { + "entropy": 0.5815631907433272, + "epoch": 0.44692737430167595, + "grad_norm": 1.796875, + "learning_rate": 1.2299651567944252e-05, + "loss": 0.5841750144958496, + "mean_token_accuracy": 0.8246684789657592, + "num_tokens": 15059756.0, + "step": 1140 + }, + { + "entropy": 0.5747593600302935, + "epoch": 0.45084778986572577, + "grad_norm": 1.5546875, + "learning_rate": 1.2212543554006971e-05, + "loss": 0.5716405391693116, + "mean_token_accuracy": 0.8256537966430187, + "num_tokens": 15192135.0, + "step": 1150 + }, + { + "entropy": 0.5864708483219147, + "epoch": 0.45476820542977553, + "grad_norm": 1.875, + "learning_rate": 1.2125435540069688e-05, + "loss": 0.5913478851318359, + "mean_token_accuracy": 0.8215075552463531, + "num_tokens": 15326137.0, + "step": 1160 + }, + { + "entropy": 0.5958302663639188, + "epoch": 0.45868862099382535, + "grad_norm": 1.6796875, + "learning_rate": 1.2038327526132404e-05, + "loss": 0.5945512294769287, + "mean_token_accuracy": 0.8203706957399846, + "num_tokens": 15455625.0, + "step": 1170 + }, + { + "entropy": 0.5705973919481039, + "epoch": 0.4626090365578751, + "grad_norm": 1.8203125, + "learning_rate": 1.1951219512195123e-05, + "loss": 0.574383544921875, + "mean_token_accuracy": 0.8264761112630368, + "num_tokens": 15589324.0, + "step": 1180 + }, + { + "entropy": 0.5563835114240646, + "epoch": 0.4665294521219249, + "grad_norm": 1.640625, + "learning_rate": 1.186411149825784e-05, + "loss": 0.5601149559020996, + "mean_token_accuracy": 0.8305692337453365, + "num_tokens": 15725979.0, + "step": 1190 + }, + { + "entropy": 0.5868437562137843, + "epoch": 0.4704498676859747, + "grad_norm": 1.7109375, + "learning_rate": 1.177700348432056e-05, + "loss": 0.583247709274292, + "mean_token_accuracy": 0.8226650767028332, + "num_tokens": 15857159.0, + "step": 1200 + }, + { + "entropy": 0.5893880043178796, + "epoch": 0.4743702832500245, + "grad_norm": 1.671875, + "learning_rate": 1.1689895470383277e-05, + "loss": 0.6006449222564697, + "mean_token_accuracy": 0.8204010248184204, + "num_tokens": 15992142.0, + "step": 1210 + }, + { + "entropy": 0.5774647582322359, + "epoch": 0.47829069881407427, + "grad_norm": 1.734375, + "learning_rate": 1.1602787456445994e-05, + "loss": 0.5778764247894287, + "mean_token_accuracy": 0.8252091869711876, + "num_tokens": 16125065.0, + "step": 1220 + }, + { + "entropy": 0.5712914764881134, + "epoch": 0.4822111143781241, + "grad_norm": 1.828125, + "learning_rate": 1.1515679442508712e-05, + "loss": 0.569354248046875, + "mean_token_accuracy": 0.8279658198356629, + "num_tokens": 16254336.0, + "step": 1230 + }, + { + "entropy": 0.5910291790962219, + "epoch": 0.48613152994217385, + "grad_norm": 1.65625, + "learning_rate": 1.1428571428571429e-05, + "loss": 0.5976007461547852, + "mean_token_accuracy": 0.8213621146976948, + "num_tokens": 16387933.0, + "step": 1240 + }, + { + "entropy": 0.5892755780369043, + "epoch": 0.49005194550622366, + "grad_norm": 1.890625, + "learning_rate": 1.1341463414634146e-05, + "loss": 0.5938167572021484, + "mean_token_accuracy": 0.8218666344881058, + "num_tokens": 16519028.0, + "step": 1250 + }, + { + "entropy": 0.6019801579415798, + "epoch": 0.4939723610702734, + "grad_norm": 2.109375, + "learning_rate": 1.1254355400696866e-05, + "loss": 0.601860523223877, + "mean_token_accuracy": 0.8193305231630802, + "num_tokens": 16648075.0, + "step": 1260 + }, + { + "entropy": 0.577896298468113, + "epoch": 0.49789277663432324, + "grad_norm": 1.5859375, + "learning_rate": 1.1167247386759583e-05, + "loss": 0.5790606498718261, + "mean_token_accuracy": 0.8256713755428791, + "num_tokens": 16784827.0, + "step": 1270 + }, + { + "entropy": 0.5828588411211968, + "epoch": 0.5018131921983731, + "grad_norm": 1.8671875, + "learning_rate": 1.1080139372822301e-05, + "loss": 0.5897656440734863, + "mean_token_accuracy": 0.8219093471765518, + "num_tokens": 16914566.0, + "step": 1280 + }, + { + "entropy": 0.5996304292231798, + "epoch": 0.5057336077624228, + "grad_norm": 1.625, + "learning_rate": 1.0993031358885018e-05, + "loss": 0.6056152820587158, + "mean_token_accuracy": 0.8185230061411858, + "num_tokens": 17047396.0, + "step": 1290 + }, + { + "entropy": 0.5672778323292732, + "epoch": 0.5096540233264726, + "grad_norm": 1.7109375, + "learning_rate": 1.0905923344947735e-05, + "loss": 0.5606307983398438, + "mean_token_accuracy": 0.8289205953478813, + "num_tokens": 17173863.0, + "step": 1300 + }, + { + "entropy": 0.574262236058712, + "epoch": 0.5135744388905223, + "grad_norm": 1.546875, + "learning_rate": 1.0818815331010455e-05, + "loss": 0.5738330364227295, + "mean_token_accuracy": 0.8259080529212952, + "num_tokens": 17305917.0, + "step": 1310 + }, + { + "entropy": 0.5749511100351811, + "epoch": 0.5174948544545722, + "grad_norm": 1.671875, + "learning_rate": 1.0731707317073172e-05, + "loss": 0.5709137916564941, + "mean_token_accuracy": 0.8254921354353428, + "num_tokens": 17436774.0, + "step": 1320 + }, + { + "entropy": 0.595110259950161, + "epoch": 0.521415270018622, + "grad_norm": 1.6484375, + "learning_rate": 1.0644599303135889e-05, + "loss": 0.5969788551330566, + "mean_token_accuracy": 0.8186600834131241, + "num_tokens": 17569314.0, + "step": 1330 + }, + { + "entropy": 0.5713474582880735, + "epoch": 0.5253356855826717, + "grad_norm": 1.7421875, + "learning_rate": 1.0557491289198607e-05, + "loss": 0.5697742462158203, + "mean_token_accuracy": 0.8268165580928326, + "num_tokens": 17703511.0, + "step": 1340 + }, + { + "entropy": 0.5673128481954336, + "epoch": 0.5292561011467215, + "grad_norm": 1.8828125, + "learning_rate": 1.0470383275261324e-05, + "loss": 0.5756454944610596, + "mean_token_accuracy": 0.8265850961208343, + "num_tokens": 17837601.0, + "step": 1350 + }, + { + "entropy": 0.5455819856375456, + "epoch": 0.5331765167107714, + "grad_norm": 1.5390625, + "learning_rate": 1.0383275261324041e-05, + "loss": 0.5512715339660644, + "mean_token_accuracy": 0.8321440435945988, + "num_tokens": 17968783.0, + "step": 1360 + }, + { + "entropy": 0.5659995820373297, + "epoch": 0.5370969322748211, + "grad_norm": 1.890625, + "learning_rate": 1.0296167247386761e-05, + "loss": 0.5727234363555909, + "mean_token_accuracy": 0.8270003162324429, + "num_tokens": 18100467.0, + "step": 1370 + }, + { + "entropy": 0.5558266244828701, + "epoch": 0.5410173478388709, + "grad_norm": 1.71875, + "learning_rate": 1.0209059233449478e-05, + "loss": 0.5511263847351074, + "mean_token_accuracy": 0.8310491070151329, + "num_tokens": 18236883.0, + "step": 1380 + }, + { + "entropy": 0.5848776459693908, + "epoch": 0.5449377634029207, + "grad_norm": 1.78125, + "learning_rate": 1.0121951219512197e-05, + "loss": 0.5840703964233398, + "mean_token_accuracy": 0.8221600718796254, + "num_tokens": 18366506.0, + "step": 1390 + }, + { + "entropy": 0.5772740695625543, + "epoch": 0.5488581789669705, + "grad_norm": 1.6171875, + "learning_rate": 1.0034843205574913e-05, + "loss": 0.5818367004394531, + "mean_token_accuracy": 0.8251958817243576, + "num_tokens": 18499076.0, + "step": 1400 + }, + { + "entropy": 0.5877921745181084, + "epoch": 0.5527785945310203, + "grad_norm": 1.71875, + "learning_rate": 9.947735191637632e-06, + "loss": 0.5859257221221924, + "mean_token_accuracy": 0.8229536414146423, + "num_tokens": 18627838.0, + "step": 1410 + }, + { + "entropy": 0.5713149573653936, + "epoch": 0.5566990100950701, + "grad_norm": 1.8046875, + "learning_rate": 9.860627177700349e-06, + "loss": 0.5759402751922608, + "mean_token_accuracy": 0.8259151518344879, + "num_tokens": 18757141.0, + "step": 1420 + }, + { + "entropy": 0.5910889457911253, + "epoch": 0.5606194256591198, + "grad_norm": 1.7421875, + "learning_rate": 9.773519163763067e-06, + "loss": 0.5957850933074951, + "mean_token_accuracy": 0.8201118834316731, + "num_tokens": 18888358.0, + "step": 1430 + }, + { + "entropy": 0.5661411985754967, + "epoch": 0.5645398412231697, + "grad_norm": 1.6015625, + "learning_rate": 9.686411149825786e-06, + "loss": 0.5634143829345704, + "mean_token_accuracy": 0.8272467255592346, + "num_tokens": 19020119.0, + "step": 1440 + }, + { + "entropy": 0.5582704734057188, + "epoch": 0.5684602567872195, + "grad_norm": 1.6484375, + "learning_rate": 9.599303135888503e-06, + "loss": 0.5596834659576416, + "mean_token_accuracy": 0.831222715228796, + "num_tokens": 19152625.0, + "step": 1450 + }, + { + "entropy": 0.5848473913967609, + "epoch": 0.5723806723512692, + "grad_norm": 1.5625, + "learning_rate": 9.51219512195122e-06, + "loss": 0.5914738655090332, + "mean_token_accuracy": 0.821570199728012, + "num_tokens": 19282646.0, + "step": 1460 + }, + { + "entropy": 0.5610104382038117, + "epoch": 0.576301087915319, + "grad_norm": 1.625, + "learning_rate": 9.425087108013938e-06, + "loss": 0.5633234500885009, + "mean_token_accuracy": 0.8292761743068695, + "num_tokens": 19418710.0, + "step": 1470 + }, + { + "entropy": 0.5768874280154705, + "epoch": 0.5802215034793688, + "grad_norm": 1.6875, + "learning_rate": 9.337979094076656e-06, + "loss": 0.5723718166351318, + "mean_token_accuracy": 0.827151071280241, + "num_tokens": 19548708.0, + "step": 1480 + }, + { + "entropy": 0.5698943838477135, + "epoch": 0.5841419190434186, + "grad_norm": 1.8359375, + "learning_rate": 9.250871080139373e-06, + "loss": 0.570045804977417, + "mean_token_accuracy": 0.8272446699440479, + "num_tokens": 19684081.0, + "step": 1490 + }, + { + "entropy": 0.5802167691290379, + "epoch": 0.5880623346074684, + "grad_norm": 1.65625, + "learning_rate": 9.163763066202092e-06, + "loss": 0.5790982246398926, + "mean_token_accuracy": 0.8250758126378059, + "num_tokens": 19817358.0, + "step": 1500 + }, + { + "entropy": 0.5764854367822408, + "epoch": 0.5919827501715181, + "grad_norm": 1.6640625, + "learning_rate": 9.076655052264809e-06, + "loss": 0.5897452354431152, + "mean_token_accuracy": 0.8235625132918358, + "num_tokens": 19947516.0, + "step": 1510 + }, + { + "entropy": 0.568101118132472, + "epoch": 0.595903165735568, + "grad_norm": 1.7734375, + "learning_rate": 8.989547038327527e-06, + "loss": 0.5667146205902099, + "mean_token_accuracy": 0.8266186200082302, + "num_tokens": 20080882.0, + "step": 1520 + }, + { + "entropy": 0.5726208575069904, + "epoch": 0.5998235812996178, + "grad_norm": 1.6171875, + "learning_rate": 8.902439024390244e-06, + "loss": 0.5737858772277832, + "mean_token_accuracy": 0.8270730949938297, + "num_tokens": 20208040.0, + "step": 1530 + }, + { + "entropy": 0.5912529457360506, + "epoch": 0.6037439968636675, + "grad_norm": 1.671875, + "learning_rate": 8.815331010452962e-06, + "loss": 0.5949665069580078, + "mean_token_accuracy": 0.821034874767065, + "num_tokens": 20342523.0, + "step": 1540 + }, + { + "entropy": 0.5770879618823528, + "epoch": 0.6076644124277173, + "grad_norm": 1.609375, + "learning_rate": 8.728222996515681e-06, + "loss": 0.5765514373779297, + "mean_token_accuracy": 0.8256359219551086, + "num_tokens": 20475073.0, + "step": 1550 + }, + { + "entropy": 0.5801304239779711, + "epoch": 0.6115848279917672, + "grad_norm": 1.8671875, + "learning_rate": 8.641114982578398e-06, + "loss": 0.5835923194885254, + "mean_token_accuracy": 0.8234978429973125, + "num_tokens": 20607544.0, + "step": 1560 + }, + { + "entropy": 0.5704087316989899, + "epoch": 0.6155052435558169, + "grad_norm": 1.8203125, + "learning_rate": 8.554006968641115e-06, + "loss": 0.5746907234191895, + "mean_token_accuracy": 0.826793709397316, + "num_tokens": 20740547.0, + "step": 1570 + }, + { + "entropy": 0.5753558877855539, + "epoch": 0.6194256591198667, + "grad_norm": 1.7890625, + "learning_rate": 8.466898954703833e-06, + "loss": 0.5788108825683593, + "mean_token_accuracy": 0.8253151409327983, + "num_tokens": 20872233.0, + "step": 1580 + }, + { + "entropy": 0.58878487162292, + "epoch": 0.6233460746839165, + "grad_norm": 1.640625, + "learning_rate": 8.379790940766552e-06, + "loss": 0.5929996490478515, + "mean_token_accuracy": 0.8213138461112977, + "num_tokens": 21004889.0, + "step": 1590 + }, + { + "entropy": 0.5900854453444481, + "epoch": 0.6272664902479663, + "grad_norm": 1.8125, + "learning_rate": 8.292682926829268e-06, + "loss": 0.5887657642364502, + "mean_token_accuracy": 0.8229175060987473, + "num_tokens": 21136824.0, + "step": 1600 + }, + { + "entropy": 0.5752101615071297, + "epoch": 0.6311869058120161, + "grad_norm": 1.65625, + "learning_rate": 8.205574912891987e-06, + "loss": 0.5777340412139893, + "mean_token_accuracy": 0.8268123485147953, + "num_tokens": 21270819.0, + "step": 1610 + }, + { + "entropy": 0.5686212468892335, + "epoch": 0.6351073213760658, + "grad_norm": 1.625, + "learning_rate": 8.118466898954704e-06, + "loss": 0.5700377941131591, + "mean_token_accuracy": 0.8275717034935951, + "num_tokens": 21403507.0, + "step": 1620 + }, + { + "entropy": 0.588944623246789, + "epoch": 0.6390277369401156, + "grad_norm": 1.7109375, + "learning_rate": 8.031358885017422e-06, + "loss": 0.5895246028900146, + "mean_token_accuracy": 0.8222635351121426, + "num_tokens": 21539521.0, + "step": 1630 + }, + { + "entropy": 0.5585341576486826, + "epoch": 0.6429481525041655, + "grad_norm": 1.6875, + "learning_rate": 7.94425087108014e-06, + "loss": 0.5642005443572998, + "mean_token_accuracy": 0.8300835333764554, + "num_tokens": 21672827.0, + "step": 1640 + }, + { + "entropy": 0.601011986285448, + "epoch": 0.6468685680682152, + "grad_norm": 1.703125, + "learning_rate": 7.857142857142858e-06, + "loss": 0.6064223766326904, + "mean_token_accuracy": 0.8166609443724155, + "num_tokens": 21803538.0, + "step": 1650 + }, + { + "entropy": 0.5758274313062429, + "epoch": 0.650788983632265, + "grad_norm": 1.5625, + "learning_rate": 7.770034843205574e-06, + "loss": 0.5752018928527832, + "mean_token_accuracy": 0.8261359445750713, + "num_tokens": 21937475.0, + "step": 1660 + }, + { + "entropy": 0.5863972809165716, + "epoch": 0.6547093991963148, + "grad_norm": 1.9140625, + "learning_rate": 7.682926829268293e-06, + "loss": 0.5884019374847412, + "mean_token_accuracy": 0.8222149938344956, + "num_tokens": 22076290.0, + "step": 1670 + }, + { + "entropy": 0.5837476711720229, + "epoch": 0.6586298147603646, + "grad_norm": 1.625, + "learning_rate": 7.595818815331011e-06, + "loss": 0.5863306522369385, + "mean_token_accuracy": 0.8223751485347748, + "num_tokens": 22210848.0, + "step": 1680 + }, + { + "entropy": 0.5637796241790056, + "epoch": 0.6625502303244144, + "grad_norm": 1.6953125, + "learning_rate": 7.508710801393729e-06, + "loss": 0.5643290996551513, + "mean_token_accuracy": 0.8271290838718415, + "num_tokens": 22342981.0, + "step": 1690 + }, + { + "entropy": 0.6036053825169801, + "epoch": 0.6664706458884642, + "grad_norm": 1.7109375, + "learning_rate": 7.421602787456447e-06, + "loss": 0.6125275135040283, + "mean_token_accuracy": 0.8163446709513664, + "num_tokens": 22473724.0, + "step": 1700 + }, + { + "entropy": 0.5783885445445776, + "epoch": 0.6703910614525139, + "grad_norm": 1.609375, + "learning_rate": 7.334494773519164e-06, + "loss": 0.5844399452209472, + "mean_token_accuracy": 0.8247374482452869, + "num_tokens": 22603403.0, + "step": 1710 + }, + { + "entropy": 0.5773308865725995, + "epoch": 0.6743114770165638, + "grad_norm": 1.546875, + "learning_rate": 7.247386759581882e-06, + "loss": 0.5766704082489014, + "mean_token_accuracy": 0.8254517287015914, + "num_tokens": 22737181.0, + "step": 1720 + }, + { + "entropy": 0.5680316600948572, + "epoch": 0.6782318925806136, + "grad_norm": 1.75, + "learning_rate": 7.1602787456446e-06, + "loss": 0.5672269821166992, + "mean_token_accuracy": 0.8279304966330528, + "num_tokens": 22869837.0, + "step": 1730 + }, + { + "entropy": 0.5941996563225984, + "epoch": 0.6821523081446633, + "grad_norm": 1.7578125, + "learning_rate": 7.0731707317073175e-06, + "loss": 0.5911007404327393, + "mean_token_accuracy": 0.8221167460083961, + "num_tokens": 23003947.0, + "step": 1740 + }, + { + "entropy": 0.5591667950153351, + "epoch": 0.6860727237087131, + "grad_norm": 1.71875, + "learning_rate": 6.986062717770036e-06, + "loss": 0.5607988834381104, + "mean_token_accuracy": 0.8297918803989888, + "num_tokens": 23134580.0, + "step": 1750 + }, + { + "entropy": 0.5676139583811164, + "epoch": 0.689993139272763, + "grad_norm": 1.6875, + "learning_rate": 6.898954703832753e-06, + "loss": 0.5650452613830567, + "mean_token_accuracy": 0.8275837257504464, + "num_tokens": 23264614.0, + "step": 1760 + }, + { + "entropy": 0.5673956066370011, + "epoch": 0.6939135548368127, + "grad_norm": 1.7265625, + "learning_rate": 6.8118466898954705e-06, + "loss": 0.5670190334320069, + "mean_token_accuracy": 0.8293615274131299, + "num_tokens": 23396741.0, + "step": 1770 + }, + { + "entropy": 0.5742564305663109, + "epoch": 0.6978339704008625, + "grad_norm": 1.6875, + "learning_rate": 6.724738675958189e-06, + "loss": 0.5740202903747559, + "mean_token_accuracy": 0.8251322150230408, + "num_tokens": 23527918.0, + "step": 1780 + }, + { + "entropy": 0.5919697143137455, + "epoch": 0.7017543859649122, + "grad_norm": 1.8046875, + "learning_rate": 6.637630662020907e-06, + "loss": 0.5907740116119384, + "mean_token_accuracy": 0.8220984056591988, + "num_tokens": 23657724.0, + "step": 1790 + }, + { + "entropy": 0.5903078105300665, + "epoch": 0.7056748015289621, + "grad_norm": 1.7109375, + "learning_rate": 6.5505226480836235e-06, + "loss": 0.5937789440155029, + "mean_token_accuracy": 0.8217935189604759, + "num_tokens": 23786805.0, + "step": 1800 + }, + { + "entropy": 0.5733942896127701, + "epoch": 0.7095952170930119, + "grad_norm": 1.6171875, + "learning_rate": 6.463414634146342e-06, + "loss": 0.5705701828002929, + "mean_token_accuracy": 0.8282980337738991, + "num_tokens": 23923037.0, + "step": 1810 + }, + { + "entropy": 0.5554268516600132, + "epoch": 0.7135156326570616, + "grad_norm": 1.7890625, + "learning_rate": 6.37630662020906e-06, + "loss": 0.5606462955474854, + "mean_token_accuracy": 0.8297950372099876, + "num_tokens": 24053827.0, + "step": 1820 + }, + { + "entropy": 0.6039480961859226, + "epoch": 0.7174360482211114, + "grad_norm": 1.703125, + "learning_rate": 6.289198606271778e-06, + "loss": 0.6047250747680664, + "mean_token_accuracy": 0.8179130852222443, + "num_tokens": 24186062.0, + "step": 1830 + }, + { + "entropy": 0.570480278134346, + "epoch": 0.7213564637851613, + "grad_norm": 1.6796875, + "learning_rate": 6.202090592334495e-06, + "loss": 0.5673944473266601, + "mean_token_accuracy": 0.8284930318593979, + "num_tokens": 24315993.0, + "step": 1840 + }, + { + "entropy": 0.5931850384920836, + "epoch": 0.725276879349211, + "grad_norm": 1.78125, + "learning_rate": 6.114982578397213e-06, + "loss": 0.5964553833007813, + "mean_token_accuracy": 0.8212155938148499, + "num_tokens": 24446858.0, + "step": 1850 + }, + { + "entropy": 0.6047229062765836, + "epoch": 0.7291972949132608, + "grad_norm": 1.828125, + "learning_rate": 6.027874564459931e-06, + "loss": 0.6088790416717529, + "mean_token_accuracy": 0.817912295460701, + "num_tokens": 24580558.0, + "step": 1860 + }, + { + "entropy": 0.5825779817998409, + "epoch": 0.7331177104773106, + "grad_norm": 1.859375, + "learning_rate": 5.940766550522649e-06, + "loss": 0.5805138111114502, + "mean_token_accuracy": 0.8254112429916859, + "num_tokens": 24718156.0, + "step": 1870 + }, + { + "entropy": 0.5747309617698193, + "epoch": 0.7370381260413604, + "grad_norm": 1.6953125, + "learning_rate": 5.853658536585366e-06, + "loss": 0.5785604953765869, + "mean_token_accuracy": 0.8248820044100285, + "num_tokens": 24853874.0, + "step": 1880 + }, + { + "entropy": 0.601536936685443, + "epoch": 0.7409585416054102, + "grad_norm": 1.8671875, + "learning_rate": 5.766550522648084e-06, + "loss": 0.6086882591247559, + "mean_token_accuracy": 0.8180929891765117, + "num_tokens": 24983767.0, + "step": 1890 + }, + { + "entropy": 0.5778564881533385, + "epoch": 0.74487895716946, + "grad_norm": 1.6484375, + "learning_rate": 5.679442508710802e-06, + "loss": 0.5777944087982178, + "mean_token_accuracy": 0.8248727723956109, + "num_tokens": 25112481.0, + "step": 1900 + }, + { + "entropy": 0.5561993703246116, + "epoch": 0.7487993727335097, + "grad_norm": 1.5859375, + "learning_rate": 5.59233449477352e-06, + "loss": 0.5545031547546386, + "mean_token_accuracy": 0.8313787803053856, + "num_tokens": 25244621.0, + "step": 1910 + }, + { + "entropy": 0.5606953276321291, + "epoch": 0.7527197882975596, + "grad_norm": 1.640625, + "learning_rate": 5.505226480836237e-06, + "loss": 0.5607188224792481, + "mean_token_accuracy": 0.8297309316694736, + "num_tokens": 25376819.0, + "step": 1920 + }, + { + "entropy": 0.5783195950090885, + "epoch": 0.7566402038616094, + "grad_norm": 1.78125, + "learning_rate": 5.418118466898955e-06, + "loss": 0.57740797996521, + "mean_token_accuracy": 0.8247730396687984, + "num_tokens": 25509601.0, + "step": 1930 + }, + { + "entropy": 0.5782700140029192, + "epoch": 0.7605606194256591, + "grad_norm": 1.6796875, + "learning_rate": 5.331010452961673e-06, + "loss": 0.5782852649688721, + "mean_token_accuracy": 0.824606055766344, + "num_tokens": 25643050.0, + "step": 1940 + }, + { + "entropy": 0.561905774474144, + "epoch": 0.7644810349897089, + "grad_norm": 1.6171875, + "learning_rate": 5.243902439024391e-06, + "loss": 0.5643265724182129, + "mean_token_accuracy": 0.8287928692996502, + "num_tokens": 25769796.0, + "step": 1950 + }, + { + "entropy": 0.5690379086881876, + "epoch": 0.7684014505537587, + "grad_norm": 1.6796875, + "learning_rate": 5.156794425087108e-06, + "loss": 0.5731242179870606, + "mean_token_accuracy": 0.8275946989655495, + "num_tokens": 25904632.0, + "step": 1960 + }, + { + "entropy": 0.5504785589873791, + "epoch": 0.7723218661178085, + "grad_norm": 1.53125, + "learning_rate": 5.0696864111498264e-06, + "loss": 0.5516434192657471, + "mean_token_accuracy": 0.8325082875788212, + "num_tokens": 26035719.0, + "step": 1970 + }, + { + "entropy": 0.5725247742608189, + "epoch": 0.7762422816818583, + "grad_norm": 1.734375, + "learning_rate": 4.982578397212544e-06, + "loss": 0.5785993576049805, + "mean_token_accuracy": 0.8263410687446594, + "num_tokens": 26167472.0, + "step": 1980 + }, + { + "entropy": 0.5720619566738605, + "epoch": 0.780162697245908, + "grad_norm": 1.5390625, + "learning_rate": 4.895470383275262e-06, + "loss": 0.5713248252868652, + "mean_token_accuracy": 0.8277824930846691, + "num_tokens": 26301163.0, + "step": 1990 + }, + { + "entropy": 0.5604069098830223, + "epoch": 0.7840831128099579, + "grad_norm": 1.6875, + "learning_rate": 4.8083623693379794e-06, + "loss": 0.5647254467010498, + "mean_token_accuracy": 0.8302777230739593, + "num_tokens": 26430868.0, + "step": 2000 + }, + { + "entropy": 0.5837776899337769, + "epoch": 0.7880035283740077, + "grad_norm": 1.7734375, + "learning_rate": 4.721254355400697e-06, + "loss": 0.5858076095581055, + "mean_token_accuracy": 0.8225198201835155, + "num_tokens": 26560775.0, + "step": 2010 + }, + { + "entropy": 0.5733296349644661, + "epoch": 0.7919239439380574, + "grad_norm": 1.625, + "learning_rate": 4.634146341463416e-06, + "loss": 0.5758630275726319, + "mean_token_accuracy": 0.8253522992134095, + "num_tokens": 26693234.0, + "step": 2020 + }, + { + "entropy": 0.557684974372387, + "epoch": 0.7958443595021072, + "grad_norm": 1.7109375, + "learning_rate": 4.5470383275261325e-06, + "loss": 0.5551248073577881, + "mean_token_accuracy": 0.8318465322256088, + "num_tokens": 26828863.0, + "step": 2030 + }, + { + "entropy": 0.5628693040460349, + "epoch": 0.7997647750661571, + "grad_norm": 1.75, + "learning_rate": 4.45993031358885e-06, + "loss": 0.571585750579834, + "mean_token_accuracy": 0.8270311810076236, + "num_tokens": 26960662.0, + "step": 2040 + }, + { + "entropy": 0.5897116485983134, + "epoch": 0.8036851906302068, + "grad_norm": 1.7421875, + "learning_rate": 4.372822299651569e-06, + "loss": 0.5966588497161865, + "mean_token_accuracy": 0.8230571210384369, + "num_tokens": 27091359.0, + "step": 2050 + }, + { + "entropy": 0.5869602940976619, + "epoch": 0.8076056061942566, + "grad_norm": 1.7109375, + "learning_rate": 4.2857142857142855e-06, + "loss": 0.588222885131836, + "mean_token_accuracy": 0.8202385693788529, + "num_tokens": 27225435.0, + "step": 2060 + }, + { + "entropy": 0.6028570268303156, + "epoch": 0.8115260217583063, + "grad_norm": 1.6328125, + "learning_rate": 4.198606271777004e-06, + "loss": 0.606553316116333, + "mean_token_accuracy": 0.8188532285392285, + "num_tokens": 27358034.0, + "step": 2070 + }, + { + "entropy": 0.5583208829164505, + "epoch": 0.8154464373223562, + "grad_norm": 1.78125, + "learning_rate": 4.111498257839722e-06, + "loss": 0.5582189083099365, + "mean_token_accuracy": 0.8306586474180222, + "num_tokens": 27490146.0, + "step": 2080 + }, + { + "entropy": 0.5879612069576978, + "epoch": 0.819366852886406, + "grad_norm": 1.8359375, + "learning_rate": 4.024390243902439e-06, + "loss": 0.5868856906890869, + "mean_token_accuracy": 0.821238712221384, + "num_tokens": 27621710.0, + "step": 2090 + }, + { + "entropy": 0.5645837895572186, + "epoch": 0.8232872684504557, + "grad_norm": 1.6640625, + "learning_rate": 3.937282229965157e-06, + "loss": 0.5639263153076172, + "mean_token_accuracy": 0.8281869366765022, + "num_tokens": 27758436.0, + "step": 2100 + }, + { + "entropy": 0.5650495100766421, + "epoch": 0.8272076840145055, + "grad_norm": 1.71875, + "learning_rate": 3.850174216027875e-06, + "loss": 0.5659061908721924, + "mean_token_accuracy": 0.8303346544504165, + "num_tokens": 27886659.0, + "step": 2110 + }, + { + "entropy": 0.5669478211551905, + "epoch": 0.8311280995785554, + "grad_norm": 1.875, + "learning_rate": 3.7630662020905927e-06, + "loss": 0.5676323890686035, + "mean_token_accuracy": 0.8264915093779563, + "num_tokens": 28020869.0, + "step": 2120 + }, + { + "entropy": 0.5540701054036618, + "epoch": 0.8350485151426051, + "grad_norm": 1.71875, + "learning_rate": 3.67595818815331e-06, + "loss": 0.5579257011413574, + "mean_token_accuracy": 0.8317541219294071, + "num_tokens": 28152491.0, + "step": 2130 + }, + { + "entropy": 0.5800126396119595, + "epoch": 0.8389689307066549, + "grad_norm": 1.8125, + "learning_rate": 3.588850174216028e-06, + "loss": 0.5853462219238281, + "mean_token_accuracy": 0.8237727522850037, + "num_tokens": 28284698.0, + "step": 2140 + }, + { + "entropy": 0.6118997160345316, + "epoch": 0.8428893462707047, + "grad_norm": 1.6796875, + "learning_rate": 3.501742160278746e-06, + "loss": 0.6129735469818115, + "mean_token_accuracy": 0.8170387588441372, + "num_tokens": 28413374.0, + "step": 2150 + }, + { + "entropy": 0.5741523541510105, + "epoch": 0.8468097618347545, + "grad_norm": 1.703125, + "learning_rate": 3.414634146341464e-06, + "loss": 0.5773540019989014, + "mean_token_accuracy": 0.8257443487644196, + "num_tokens": 28545962.0, + "step": 2160 + }, + { + "entropy": 0.5787751715630293, + "epoch": 0.8507301773988043, + "grad_norm": 1.6796875, + "learning_rate": 3.3275261324041815e-06, + "loss": 0.5790078163146972, + "mean_token_accuracy": 0.825350683927536, + "num_tokens": 28680445.0, + "step": 2170 + }, + { + "entropy": 0.5716471575200558, + "epoch": 0.8546505929628541, + "grad_norm": 1.6953125, + "learning_rate": 3.240418118466899e-06, + "loss": 0.5730648994445801, + "mean_token_accuracy": 0.8268438093364239, + "num_tokens": 28811694.0, + "step": 2180 + }, + { + "entropy": 0.5720145717263222, + "epoch": 0.8585710085269038, + "grad_norm": 1.84375, + "learning_rate": 3.1533101045296173e-06, + "loss": 0.5759499549865723, + "mean_token_accuracy": 0.8254265673458576, + "num_tokens": 28945890.0, + "step": 2190 + }, + { + "entropy": 0.5667183842509985, + "epoch": 0.8624914240909537, + "grad_norm": 1.796875, + "learning_rate": 3.0662020905923345e-06, + "loss": 0.567453670501709, + "mean_token_accuracy": 0.8277363143861294, + "num_tokens": 29075294.0, + "step": 2200 + }, + { + "entropy": 0.583325557038188, + "epoch": 0.8664118396550035, + "grad_norm": 1.7109375, + "learning_rate": 2.9790940766550526e-06, + "loss": 0.5870438098907471, + "mean_token_accuracy": 0.8234334781765937, + "num_tokens": 29207413.0, + "step": 2210 + }, + { + "entropy": 0.5614024806767702, + "epoch": 0.8703322552190532, + "grad_norm": 1.65625, + "learning_rate": 2.8919860627177703e-06, + "loss": 0.5655649185180665, + "mean_token_accuracy": 0.8296878322958946, + "num_tokens": 29340755.0, + "step": 2220 + }, + { + "entropy": 0.5752765364944935, + "epoch": 0.874252670783103, + "grad_norm": 1.65625, + "learning_rate": 2.8048780487804884e-06, + "loss": 0.5743560314178466, + "mean_token_accuracy": 0.8251941919326782, + "num_tokens": 29474374.0, + "step": 2230 + }, + { + "entropy": 0.5772768836468458, + "epoch": 0.8781730863471529, + "grad_norm": 1.5234375, + "learning_rate": 2.7177700348432056e-06, + "loss": 0.5725958347320557, + "mean_token_accuracy": 0.826750586181879, + "num_tokens": 29607009.0, + "step": 2240 + }, + { + "entropy": 0.5837304752320052, + "epoch": 0.8820935019112026, + "grad_norm": 1.875, + "learning_rate": 2.6306620209059237e-06, + "loss": 0.588666296005249, + "mean_token_accuracy": 0.8247058235108853, + "num_tokens": 29738318.0, + "step": 2250 + }, + { + "entropy": 0.5540932007133961, + "epoch": 0.8860139174752524, + "grad_norm": 1.8359375, + "learning_rate": 2.5435540069686414e-06, + "loss": 0.5564557552337647, + "mean_token_accuracy": 0.8303055338561535, + "num_tokens": 29871530.0, + "step": 2260 + }, + { + "entropy": 0.5647958315908909, + "epoch": 0.8899343330393021, + "grad_norm": 1.90625, + "learning_rate": 2.456445993031359e-06, + "loss": 0.5653763771057129, + "mean_token_accuracy": 0.8286120660603047, + "num_tokens": 30004318.0, + "step": 2270 + }, + { + "entropy": 0.5644253201782703, + "epoch": 0.8938547486033519, + "grad_norm": 1.734375, + "learning_rate": 2.3693379790940767e-06, + "loss": 0.5634883880615235, + "mean_token_accuracy": 0.8275229759514332, + "num_tokens": 30136805.0, + "step": 2280 + }, + { + "entropy": 0.5744159776717425, + "epoch": 0.8977751641674018, + "grad_norm": 1.7421875, + "learning_rate": 2.282229965156795e-06, + "loss": 0.5716060161590576, + "mean_token_accuracy": 0.82558439001441, + "num_tokens": 30267921.0, + "step": 2290 + }, + { + "entropy": 0.5650353617966175, + "epoch": 0.9016955797314515, + "grad_norm": 1.6953125, + "learning_rate": 2.1951219512195125e-06, + "loss": 0.5693279266357422, + "mean_token_accuracy": 0.8278473079204559, + "num_tokens": 30402696.0, + "step": 2300 + }, + { + "entropy": 0.5965178959071636, + "epoch": 0.9056159952955013, + "grad_norm": 1.71875, + "learning_rate": 2.10801393728223e-06, + "loss": 0.6028908729553223, + "mean_token_accuracy": 0.8189698673784733, + "num_tokens": 30536391.0, + "step": 2310 + }, + { + "entropy": 0.601612939313054, + "epoch": 0.9095364108595511, + "grad_norm": 1.796875, + "learning_rate": 2.020905923344948e-06, + "loss": 0.6042677402496338, + "mean_token_accuracy": 0.8197429470717907, + "num_tokens": 30669243.0, + "step": 2320 + }, + { + "entropy": 0.5706041093915701, + "epoch": 0.9134568264236009, + "grad_norm": 1.8984375, + "learning_rate": 1.9337979094076655e-06, + "loss": 0.5693963527679443, + "mean_token_accuracy": 0.8271980971097946, + "num_tokens": 30800574.0, + "step": 2330 + }, + { + "entropy": 0.5640899043530225, + "epoch": 0.9173772419876507, + "grad_norm": 1.6484375, + "learning_rate": 1.8466898954703836e-06, + "loss": 0.5705442905426026, + "mean_token_accuracy": 0.8264196418225765, + "num_tokens": 30933674.0, + "step": 2340 + }, + { + "entropy": 0.5657230116426945, + "epoch": 0.9212976575517005, + "grad_norm": 1.609375, + "learning_rate": 1.7595818815331012e-06, + "loss": 0.570911979675293, + "mean_token_accuracy": 0.8267294079065323, + "num_tokens": 31065102.0, + "step": 2350 + }, + { + "entropy": 0.562781137228012, + "epoch": 0.9252180731157502, + "grad_norm": 1.796875, + "learning_rate": 1.6724738675958191e-06, + "loss": 0.5703943729400635, + "mean_token_accuracy": 0.8268945284187794, + "num_tokens": 31195291.0, + "step": 2360 + }, + { + "entropy": 0.5673436559736729, + "epoch": 0.9291384886798001, + "grad_norm": 1.609375, + "learning_rate": 1.5853658536585368e-06, + "loss": 0.5722959995269775, + "mean_token_accuracy": 0.8286142982542515, + "num_tokens": 31327802.0, + "step": 2370 + }, + { + "entropy": 0.5538178488612175, + "epoch": 0.9330589042438499, + "grad_norm": 1.859375, + "learning_rate": 1.4982578397212545e-06, + "loss": 0.5541855812072753, + "mean_token_accuracy": 0.8326897613704205, + "num_tokens": 31454999.0, + "step": 2380 + }, + { + "entropy": 0.5489677041769028, + "epoch": 0.9369793198078996, + "grad_norm": 1.5625, + "learning_rate": 1.4111498257839723e-06, + "loss": 0.5547789573669434, + "mean_token_accuracy": 0.8312770992517471, + "num_tokens": 31592105.0, + "step": 2390 + }, + { + "entropy": 0.5815275024622679, + "epoch": 0.9408997353719494, + "grad_norm": 1.8671875, + "learning_rate": 1.32404181184669e-06, + "loss": 0.589658784866333, + "mean_token_accuracy": 0.8223450765013695, + "num_tokens": 31720785.0, + "step": 2400 + }, + { + "entropy": 0.5828426022082567, + "epoch": 0.9448201509359992, + "grad_norm": 1.71875, + "learning_rate": 1.2369337979094077e-06, + "loss": 0.5806317806243897, + "mean_token_accuracy": 0.8260537378489972, + "num_tokens": 31852738.0, + "step": 2410 + }, + { + "entropy": 0.5780322037637233, + "epoch": 0.948740566500049, + "grad_norm": 1.671875, + "learning_rate": 1.1498257839721255e-06, + "loss": 0.573900556564331, + "mean_token_accuracy": 0.8251914605498314, + "num_tokens": 31985785.0, + "step": 2420 + }, + { + "entropy": 0.5561736188828945, + "epoch": 0.9526609820640988, + "grad_norm": 1.734375, + "learning_rate": 1.0627177700348432e-06, + "loss": 0.5585554599761963, + "mean_token_accuracy": 0.8310537829995155, + "num_tokens": 32116876.0, + "step": 2430 + }, + { + "entropy": 0.5960801500827074, + "epoch": 0.9565813976281485, + "grad_norm": 1.8515625, + "learning_rate": 9.75609756097561e-07, + "loss": 0.6023911476135254, + "mean_token_accuracy": 0.8196251414716244, + "num_tokens": 32251738.0, + "step": 2440 + }, + { + "entropy": 0.5717530060559511, + "epoch": 0.9605018131921984, + "grad_norm": 1.4609375, + "learning_rate": 8.885017421602789e-07, + "loss": 0.5710067749023438, + "mean_token_accuracy": 0.8258462838828564, + "num_tokens": 32386153.0, + "step": 2450 + }, + { + "entropy": 0.5853737752884627, + "epoch": 0.9644222287562482, + "grad_norm": 1.875, + "learning_rate": 8.013937282229965e-07, + "loss": 0.59298734664917, + "mean_token_accuracy": 0.8215095445513725, + "num_tokens": 32521937.0, + "step": 2460 + }, + { + "entropy": 0.5701922554522753, + "epoch": 0.9683426443202979, + "grad_norm": 1.7578125, + "learning_rate": 7.142857142857143e-07, + "loss": 0.5723202228546143, + "mean_token_accuracy": 0.8279052451252937, + "num_tokens": 32653667.0, + "step": 2470 + }, + { + "entropy": 0.5777422614395619, + "epoch": 0.9722630598843477, + "grad_norm": 1.6875, + "learning_rate": 6.271777003484321e-07, + "loss": 0.581749153137207, + "mean_token_accuracy": 0.8253626257181168, + "num_tokens": 32787099.0, + "step": 2480 + }, + { + "entropy": 0.5758198279887438, + "epoch": 0.9761834754483976, + "grad_norm": 1.6015625, + "learning_rate": 5.400696864111499e-07, + "loss": 0.5754932403564453, + "mean_token_accuracy": 0.8263446964323521, + "num_tokens": 32919330.0, + "step": 2490 + }, + { + "entropy": 0.5611678268760443, + "epoch": 0.9801038910124473, + "grad_norm": 1.703125, + "learning_rate": 4.5296167247386764e-07, + "loss": 0.5594530582427979, + "mean_token_accuracy": 0.8305567562580108, + "num_tokens": 33052864.0, + "step": 2500 + }, + { + "entropy": 0.5629599597305059, + "epoch": 0.9840243065764971, + "grad_norm": 1.7578125, + "learning_rate": 3.6585365853658536e-07, + "loss": 0.5671555042266846, + "mean_token_accuracy": 0.8285158857703209, + "num_tokens": 33187097.0, + "step": 2510 + }, + { + "entropy": 0.5846845716238022, + "epoch": 0.9879447221405468, + "grad_norm": 1.640625, + "learning_rate": 2.7874564459930313e-07, + "loss": 0.5863873481750488, + "mean_token_accuracy": 0.8224268153309822, + "num_tokens": 33324336.0, + "step": 2520 + }, + { + "entropy": 0.5745269916951656, + "epoch": 0.9918651377045967, + "grad_norm": 1.8046875, + "learning_rate": 1.916376306620209e-07, + "loss": 0.5790841579437256, + "mean_token_accuracy": 0.8244781382381916, + "num_tokens": 33457098.0, + "step": 2530 + }, + { + "entropy": 0.5654200822114944, + "epoch": 0.9957855532686465, + "grad_norm": 1.6640625, + "learning_rate": 1.045296167247387e-07, + "loss": 0.5639691829681397, + "mean_token_accuracy": 0.8280748799443245, + "num_tokens": 33590704.0, + "step": 2540 + }, + { + "entropy": 0.5788687650114298, + "epoch": 0.9997059688326962, + "grad_norm": 1.7265625, + "learning_rate": 1.7421602787456446e-08, + "loss": 0.5817639350891113, + "mean_token_accuracy": 0.8252955496311187, + "num_tokens": 33724217.0, + "step": 2550 + } + ], + "logging_steps": 10, + "max_steps": 2551, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3.2075681983043174e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2551/training_args.bin b/checkpoint-2551/training_args.bin new file mode 100644 index 0000000..48a55f1 --- /dev/null +++ b/checkpoint-2551/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf5271d53e5ecf5b0e849ec9b9f8933cba573d9ea8200a6a32b9985bb49d32a6 +size 5713 diff --git a/checkpoint-3125/chat_template.jinja b/checkpoint-3125/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-3125/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-3125/config.json b/checkpoint-3125/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-3125/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-3125/generation_config.json b/checkpoint-3125/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-3125/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-3125/model.safetensors b/checkpoint-3125/model.safetensors new file mode 100644 index 0000000..95a51f7 --- /dev/null +++ b/checkpoint-3125/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14b0fd351e37275309a2c94d55333a16a5faca6fbf327aa7fb4431ebccbfed50 +size 3441185608 diff --git a/checkpoint-3125/optimizer.pt b/checkpoint-3125/optimizer.pt new file mode 100644 index 0000000..c49f95c --- /dev/null +++ b/checkpoint-3125/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd38e7062bd3fce5a8b5906d2b5b0ef73ffa049892fa69c57b43c43a46cf8c51 +size 6882572207 diff --git a/checkpoint-3125/rng_state.pth b/checkpoint-3125/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-3125/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-3125/scheduler.pt b/checkpoint-3125/scheduler.pt new file mode 100644 index 0000000..c99d7ea --- /dev/null +++ b/checkpoint-3125/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69ba192991ebd4efd6d159e8ee8fa290726aa6c790ff3ef407a41d087cea6c58 +size 1465 diff --git a/checkpoint-3125/tokenizer.json b/checkpoint-3125/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-3125/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-3125/tokenizer_config.json b/checkpoint-3125/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/checkpoint-3125/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-3125/trainer_state.json b/checkpoint-3125/trainer_state.json new file mode 100644 index 0000000..d306494 --- /dev/null +++ b/checkpoint-3125/trainer_state.json @@ -0,0 +1,3154 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 3125, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.2839297465980053, + "epoch": 0.00320038404608553, + "grad_norm": 73.0, + "learning_rate": 5.76923076923077e-07, + "loss": 1.9901968002319337, + "mean_token_accuracy": 0.6237266078591347, + "num_tokens": 33961.0, + "step": 10 + }, + { + "entropy": 1.2595087066292763, + "epoch": 0.00640076809217106, + "grad_norm": 64.5, + "learning_rate": 1.217948717948718e-06, + "loss": 1.8751590728759766, + "mean_token_accuracy": 0.6309439569711686, + "num_tokens": 71819.0, + "step": 20 + }, + { + "entropy": 1.2931475669145585, + "epoch": 0.00960115213825659, + "grad_norm": 22.625, + "learning_rate": 1.8589743589743592e-06, + "loss": 1.7506515502929687, + "mean_token_accuracy": 0.6388503693044185, + "num_tokens": 112770.0, + "step": 30 + }, + { + "entropy": 1.2734985046088696, + "epoch": 0.01280153618434212, + "grad_norm": 23.75, + "learning_rate": 2.5e-06, + "loss": 1.6540241241455078, + "mean_token_accuracy": 0.657498425245285, + "num_tokens": 146669.0, + "step": 40 + }, + { + "entropy": 1.287195574492216, + "epoch": 0.016001920230427652, + "grad_norm": 13.3125, + "learning_rate": 3.141025641025641e-06, + "loss": 1.5025008201599122, + "mean_token_accuracy": 0.6669101666659116, + "num_tokens": 183443.0, + "step": 50 + }, + { + "entropy": 1.3111741445958613, + "epoch": 0.01920230427651318, + "grad_norm": 6.46875, + "learning_rate": 3.782051282051282e-06, + "loss": 1.485818862915039, + "mean_token_accuracy": 0.685324776172638, + "num_tokens": 219455.0, + "step": 60 + }, + { + "entropy": 1.2205842301249503, + "epoch": 0.022402688322598712, + "grad_norm": 4.8125, + "learning_rate": 4.423076923076924e-06, + "loss": 1.3609664916992188, + "mean_token_accuracy": 0.7222387321293354, + "num_tokens": 253077.0, + "step": 70 + }, + { + "entropy": 1.2416748367249966, + "epoch": 0.02560307236868424, + "grad_norm": 5.15625, + "learning_rate": 5.064102564102565e-06, + "loss": 1.3642989158630372, + "mean_token_accuracy": 0.7237147346138955, + "num_tokens": 287333.0, + "step": 80 + }, + { + "entropy": 1.2347557865083219, + "epoch": 0.028803456414769772, + "grad_norm": 4.4375, + "learning_rate": 5.705128205128206e-06, + "loss": 1.3442096710205078, + "mean_token_accuracy": 0.7320359282195568, + "num_tokens": 319994.0, + "step": 90 + }, + { + "entropy": 1.298496885597706, + "epoch": 0.032003840460855304, + "grad_norm": 4.71875, + "learning_rate": 6.3461538461538466e-06, + "loss": 1.3953542709350586, + "mean_token_accuracy": 0.720856224745512, + "num_tokens": 353798.0, + "step": 100 + }, + { + "entropy": 1.283773996680975, + "epoch": 0.035204224506940836, + "grad_norm": 3.78125, + "learning_rate": 6.9871794871794876e-06, + "loss": 1.3582657814025878, + "mean_token_accuracy": 0.7274449437856674, + "num_tokens": 387975.0, + "step": 110 + }, + { + "entropy": 1.2386538445949555, + "epoch": 0.03840460855302636, + "grad_norm": 3.96875, + "learning_rate": 7.6282051282051286e-06, + "loss": 1.2965816497802733, + "mean_token_accuracy": 0.7260297447443008, + "num_tokens": 426086.0, + "step": 120 + }, + { + "entropy": 1.2428459793329238, + "epoch": 0.04160499259911189, + "grad_norm": 4.6875, + "learning_rate": 8.26923076923077e-06, + "loss": 1.3305319786071776, + "mean_token_accuracy": 0.7313816711306572, + "num_tokens": 459435.0, + "step": 130 + }, + { + "entropy": 1.2068071104586124, + "epoch": 0.044805376645197424, + "grad_norm": 4.59375, + "learning_rate": 8.910256410256411e-06, + "loss": 1.246315860748291, + "mean_token_accuracy": 0.7372891046106815, + "num_tokens": 490867.0, + "step": 140 + }, + { + "entropy": 1.2453301914036274, + "epoch": 0.048005760691282956, + "grad_norm": 3.84375, + "learning_rate": 9.551282051282053e-06, + "loss": 1.2801061630249024, + "mean_token_accuracy": 0.7317046545445919, + "num_tokens": 524633.0, + "step": 150 + }, + { + "entropy": 1.2111680828034879, + "epoch": 0.05120614473736848, + "grad_norm": 3.375, + "learning_rate": 1.0192307692307692e-05, + "loss": 1.2264927864074706, + "mean_token_accuracy": 0.7354309558868408, + "num_tokens": 558869.0, + "step": 160 + }, + { + "entropy": 1.2417247883975506, + "epoch": 0.05440652878345401, + "grad_norm": 3.8125, + "learning_rate": 1.0833333333333334e-05, + "loss": 1.296145248413086, + "mean_token_accuracy": 0.7288267895579338, + "num_tokens": 594200.0, + "step": 170 + }, + { + "entropy": 1.2903451286256313, + "epoch": 0.057606912829539544, + "grad_norm": 3.953125, + "learning_rate": 1.1474358974358974e-05, + "loss": 1.391792392730713, + "mean_token_accuracy": 0.7263986520469189, + "num_tokens": 628749.0, + "step": 180 + }, + { + "entropy": 1.1508908517658711, + "epoch": 0.060807296875625076, + "grad_norm": 4.59375, + "learning_rate": 1.2115384615384615e-05, + "loss": 1.170935821533203, + "mean_token_accuracy": 0.7482860818505287, + "num_tokens": 660930.0, + "step": 190 + }, + { + "entropy": 1.1952558353543281, + "epoch": 0.06400768092171061, + "grad_norm": 3.953125, + "learning_rate": 1.2756410256410257e-05, + "loss": 1.2448080062866211, + "mean_token_accuracy": 0.7362503379583358, + "num_tokens": 696715.0, + "step": 200 + }, + { + "entropy": 1.2021137841045857, + "epoch": 0.06720806496779613, + "grad_norm": 4.28125, + "learning_rate": 1.3397435897435897e-05, + "loss": 1.2594975471496581, + "mean_token_accuracy": 0.7392350442707538, + "num_tokens": 730982.0, + "step": 210 + }, + { + "entropy": 1.1502027772367, + "epoch": 0.07040844901388167, + "grad_norm": 4.0, + "learning_rate": 1.403846153846154e-05, + "loss": 1.2263619422912597, + "mean_token_accuracy": 0.7455356031656265, + "num_tokens": 765369.0, + "step": 220 + }, + { + "entropy": 1.2468636624515057, + "epoch": 0.0736088330599672, + "grad_norm": 3.875, + "learning_rate": 1.467948717948718e-05, + "loss": 1.317853832244873, + "mean_token_accuracy": 0.7300475873053074, + "num_tokens": 800884.0, + "step": 230 + }, + { + "entropy": 1.1318104140460492, + "epoch": 0.07680921710605272, + "grad_norm": 3.234375, + "learning_rate": 1.5320512820512823e-05, + "loss": 1.1992506980895996, + "mean_token_accuracy": 0.7450634054839611, + "num_tokens": 837605.0, + "step": 240 + }, + { + "entropy": 1.1415463611483574, + "epoch": 0.08000960115213826, + "grad_norm": 3.640625, + "learning_rate": 1.5961538461538465e-05, + "loss": 1.1817726135253905, + "mean_token_accuracy": 0.7463984578847885, + "num_tokens": 876274.0, + "step": 250 + }, + { + "entropy": 1.2368000820279121, + "epoch": 0.08320998519822378, + "grad_norm": 3.546875, + "learning_rate": 1.6602564102564103e-05, + "loss": 1.297637367248535, + "mean_token_accuracy": 0.7309112548828125, + "num_tokens": 911704.0, + "step": 260 + }, + { + "entropy": 1.1496265470981597, + "epoch": 0.08641036924430932, + "grad_norm": 3.875, + "learning_rate": 1.7243589743589745e-05, + "loss": 1.2197935104370117, + "mean_token_accuracy": 0.7450173661112786, + "num_tokens": 947611.0, + "step": 270 + }, + { + "entropy": 1.1289438650012016, + "epoch": 0.08961075329039485, + "grad_norm": 4.5, + "learning_rate": 1.7884615384615387e-05, + "loss": 1.2045019149780274, + "mean_token_accuracy": 0.745516513288021, + "num_tokens": 984535.0, + "step": 280 + }, + { + "entropy": 1.1794409573078155, + "epoch": 0.09281113733648037, + "grad_norm": 3.71875, + "learning_rate": 1.852564102564103e-05, + "loss": 1.211575698852539, + "mean_token_accuracy": 0.7406493924558163, + "num_tokens": 1018018.0, + "step": 290 + }, + { + "entropy": 1.1385122388601303, + "epoch": 0.09601152138256591, + "grad_norm": 4.53125, + "learning_rate": 1.916666666666667e-05, + "loss": 1.1774465560913085, + "mean_token_accuracy": 0.7501121394336223, + "num_tokens": 1051431.0, + "step": 300 + }, + { + "entropy": 1.1633801862597466, + "epoch": 0.09921190542865144, + "grad_norm": 3.359375, + "learning_rate": 1.980769230769231e-05, + "loss": 1.2100112915039063, + "mean_token_accuracy": 0.7403162129223346, + "num_tokens": 1086575.0, + "step": 310 + }, + { + "entropy": 1.1853495672345162, + "epoch": 0.10241228947473696, + "grad_norm": 3.546875, + "learning_rate": 1.9950231070031995e-05, + "loss": 1.234422206878662, + "mean_token_accuracy": 0.7415597856044769, + "num_tokens": 1122816.0, + "step": 320 + }, + { + "entropy": 1.1737073637545108, + "epoch": 0.1056126735208225, + "grad_norm": 3.671875, + "learning_rate": 1.9879132598649133e-05, + "loss": 1.2083673477172852, + "mean_token_accuracy": 0.7407836645841599, + "num_tokens": 1158910.0, + "step": 330 + }, + { + "entropy": 1.2292249217629432, + "epoch": 0.10881305756690803, + "grad_norm": 3.875, + "learning_rate": 1.9808034127266264e-05, + "loss": 1.269577407836914, + "mean_token_accuracy": 0.7345698416233063, + "num_tokens": 1196019.0, + "step": 340 + }, + { + "entropy": 1.18642889931798, + "epoch": 0.11201344161299356, + "grad_norm": 3.578125, + "learning_rate": 1.97369356558834e-05, + "loss": 1.2697922706604003, + "mean_token_accuracy": 0.741942162066698, + "num_tokens": 1229987.0, + "step": 350 + }, + { + "entropy": 1.1096315152943135, + "epoch": 0.11521382565907909, + "grad_norm": 3.9375, + "learning_rate": 1.9665837184500536e-05, + "loss": 1.1929906845092773, + "mean_token_accuracy": 0.7551799714565277, + "num_tokens": 1263509.0, + "step": 360 + }, + { + "entropy": 1.1739558205008507, + "epoch": 0.11841420970516461, + "grad_norm": 3.8125, + "learning_rate": 1.959473871311767e-05, + "loss": 1.2080945014953612, + "mean_token_accuracy": 0.7452260315418243, + "num_tokens": 1301070.0, + "step": 370 + }, + { + "entropy": 1.11664487272501, + "epoch": 0.12161459375125015, + "grad_norm": 4.15625, + "learning_rate": 1.9523640241734804e-05, + "loss": 1.189216423034668, + "mean_token_accuracy": 0.7564452636986971, + "num_tokens": 1333449.0, + "step": 380 + }, + { + "entropy": 1.1409478083252906, + "epoch": 0.12481497779733568, + "grad_norm": 3.34375, + "learning_rate": 1.9452541770351938e-05, + "loss": 1.2088672637939453, + "mean_token_accuracy": 0.7491289660334587, + "num_tokens": 1368459.0, + "step": 390 + }, + { + "entropy": 1.2013175174593926, + "epoch": 0.12801536184342122, + "grad_norm": 4.5625, + "learning_rate": 1.9381443298969072e-05, + "loss": 1.26448335647583, + "mean_token_accuracy": 0.7352619163691998, + "num_tokens": 1406859.0, + "step": 400 + }, + { + "entropy": 1.1907868802547454, + "epoch": 0.13121574588950674, + "grad_norm": 3.015625, + "learning_rate": 1.931034482758621e-05, + "loss": 1.2475446701049804, + "mean_token_accuracy": 0.7453009270131588, + "num_tokens": 1442179.0, + "step": 410 + }, + { + "entropy": 1.0694843657314776, + "epoch": 0.13441612993559227, + "grad_norm": 3.59375, + "learning_rate": 1.9239246356203344e-05, + "loss": 1.1248330116271972, + "mean_token_accuracy": 0.756801488250494, + "num_tokens": 1481135.0, + "step": 420 + }, + { + "entropy": 1.1419598631560803, + "epoch": 0.1376165139816778, + "grad_norm": 3.71875, + "learning_rate": 1.916814788482048e-05, + "loss": 1.2056102752685547, + "mean_token_accuracy": 0.7461046256124974, + "num_tokens": 1518329.0, + "step": 430 + }, + { + "entropy": 1.1788324914872645, + "epoch": 0.14081689802776334, + "grad_norm": 3.421875, + "learning_rate": 1.9097049413437613e-05, + "loss": 1.2716377258300782, + "mean_token_accuracy": 0.7431618466973304, + "num_tokens": 1554359.0, + "step": 440 + }, + { + "entropy": 1.2097309075295926, + "epoch": 0.14401728207384887, + "grad_norm": 3.984375, + "learning_rate": 1.9025950942054747e-05, + "loss": 1.246030616760254, + "mean_token_accuracy": 0.7390106923878192, + "num_tokens": 1591000.0, + "step": 450 + }, + { + "entropy": 1.2036252733319999, + "epoch": 0.1472176661199344, + "grad_norm": 3.21875, + "learning_rate": 1.8954852470671884e-05, + "loss": 1.2812904357910155, + "mean_token_accuracy": 0.7348502896726131, + "num_tokens": 1627205.0, + "step": 460 + }, + { + "entropy": 1.1779422886669635, + "epoch": 0.15041805016601992, + "grad_norm": 3.234375, + "learning_rate": 1.8883753999289015e-05, + "loss": 1.2817980766296386, + "mean_token_accuracy": 0.7440513700246811, + "num_tokens": 1663831.0, + "step": 470 + }, + { + "entropy": 1.1881464742124082, + "epoch": 0.15361843421210544, + "grad_norm": 3.03125, + "learning_rate": 1.8812655527906153e-05, + "loss": 1.2011470794677734, + "mean_token_accuracy": 0.7412468865513802, + "num_tokens": 1701847.0, + "step": 480 + }, + { + "entropy": 1.1195977296680213, + "epoch": 0.156818818258191, + "grad_norm": 3.328125, + "learning_rate": 1.8741557056523287e-05, + "loss": 1.1337278366088868, + "mean_token_accuracy": 0.7538985311985016, + "num_tokens": 1740516.0, + "step": 490 + }, + { + "entropy": 1.2133471183478832, + "epoch": 0.16001920230427652, + "grad_norm": 3.40625, + "learning_rate": 1.867045858514042e-05, + "loss": 1.2486845970153808, + "mean_token_accuracy": 0.7390272334218025, + "num_tokens": 1778065.0, + "step": 500 + }, + { + "entropy": 1.1336760543286801, + "epoch": 0.16321958635036204, + "grad_norm": 3.53125, + "learning_rate": 1.8599360113757556e-05, + "loss": 1.1790239334106445, + "mean_token_accuracy": 0.7476340506225825, + "num_tokens": 1817855.0, + "step": 510 + }, + { + "entropy": 1.193576630949974, + "epoch": 0.16641997039644757, + "grad_norm": 3.4375, + "learning_rate": 1.852826164237469e-05, + "loss": 1.2632747650146485, + "mean_token_accuracy": 0.7385985501110554, + "num_tokens": 1853124.0, + "step": 520 + }, + { + "entropy": 1.1418399840593338, + "epoch": 0.1696203544425331, + "grad_norm": 3.640625, + "learning_rate": 1.8457163170991824e-05, + "loss": 1.194384479522705, + "mean_token_accuracy": 0.7507250174880028, + "num_tokens": 1887889.0, + "step": 530 + }, + { + "entropy": 1.1441261656582355, + "epoch": 0.17282073848861865, + "grad_norm": 2.96875, + "learning_rate": 1.838606469960896e-05, + "loss": 1.1474828720092773, + "mean_token_accuracy": 0.7484897166490555, + "num_tokens": 1925355.0, + "step": 540 + }, + { + "entropy": 1.1772041961550712, + "epoch": 0.17602112253470417, + "grad_norm": 2.859375, + "learning_rate": 1.8314966228226096e-05, + "loss": 1.2581243515014648, + "mean_token_accuracy": 0.7401691168546677, + "num_tokens": 1962021.0, + "step": 550 + }, + { + "entropy": 1.154813179373741, + "epoch": 0.1792215065807897, + "grad_norm": 3.140625, + "learning_rate": 1.824386775684323e-05, + "loss": 1.2013533592224122, + "mean_token_accuracy": 0.7461690090596675, + "num_tokens": 1999387.0, + "step": 560 + }, + { + "entropy": 1.1134828917682171, + "epoch": 0.18242189062687522, + "grad_norm": 4.0625, + "learning_rate": 1.8172769285460364e-05, + "loss": 1.155961513519287, + "mean_token_accuracy": 0.7565418593585491, + "num_tokens": 2033366.0, + "step": 570 + }, + { + "entropy": 1.129069809615612, + "epoch": 0.18562227467296075, + "grad_norm": 3.3125, + "learning_rate": 1.81016708140775e-05, + "loss": 1.202240562438965, + "mean_token_accuracy": 0.7467389106750488, + "num_tokens": 2068012.0, + "step": 580 + }, + { + "entropy": 1.1413575001060963, + "epoch": 0.1888226587190463, + "grad_norm": 2.859375, + "learning_rate": 1.8030572342694636e-05, + "loss": 1.1733809471130372, + "mean_token_accuracy": 0.7480724595487118, + "num_tokens": 2103208.0, + "step": 590 + }, + { + "entropy": 1.1593846715986729, + "epoch": 0.19202304276513182, + "grad_norm": 3.65625, + "learning_rate": 1.7959473871311767e-05, + "loss": 1.2416004180908202, + "mean_token_accuracy": 0.7419706009328365, + "num_tokens": 2139071.0, + "step": 600 + }, + { + "entropy": 1.2044602517038583, + "epoch": 0.19522342681121735, + "grad_norm": 2.640625, + "learning_rate": 1.7888375399928905e-05, + "loss": 1.2507460594177247, + "mean_token_accuracy": 0.7382100090384484, + "num_tokens": 2178499.0, + "step": 610 + }, + { + "entropy": 1.096942011639476, + "epoch": 0.19842381085730287, + "grad_norm": 3.359375, + "learning_rate": 1.7817276928546035e-05, + "loss": 1.1269610404968262, + "mean_token_accuracy": 0.7565630108118058, + "num_tokens": 2214448.0, + "step": 620 + }, + { + "entropy": 1.1488182917237282, + "epoch": 0.2016241949033884, + "grad_norm": 4.0625, + "learning_rate": 1.7746178457163173e-05, + "loss": 1.1636892318725587, + "mean_token_accuracy": 0.7449747450649739, + "num_tokens": 2249587.0, + "step": 630 + }, + { + "entropy": 1.072244780510664, + "epoch": 0.20482457894947392, + "grad_norm": 2.875, + "learning_rate": 1.7675079985780307e-05, + "loss": 1.1519594192504883, + "mean_token_accuracy": 0.7612439051270485, + "num_tokens": 2285832.0, + "step": 640 + }, + { + "entropy": 1.137603597342968, + "epoch": 0.20802496299555948, + "grad_norm": 3.65625, + "learning_rate": 1.760398151439744e-05, + "loss": 1.2033388137817382, + "mean_token_accuracy": 0.750407163798809, + "num_tokens": 2323267.0, + "step": 650 + }, + { + "entropy": 1.1386778496205807, + "epoch": 0.211225347041645, + "grad_norm": 3.90625, + "learning_rate": 1.7532883043014576e-05, + "loss": 1.2194600105285645, + "mean_token_accuracy": 0.7539191976189613, + "num_tokens": 2356234.0, + "step": 660 + }, + { + "entropy": 1.171764300018549, + "epoch": 0.21442573108773053, + "grad_norm": 3.734375, + "learning_rate": 1.746178457163171e-05, + "loss": 1.2068530082702638, + "mean_token_accuracy": 0.7431434363126754, + "num_tokens": 2394092.0, + "step": 670 + }, + { + "entropy": 1.1694385975599288, + "epoch": 0.21762611513381605, + "grad_norm": 3.609375, + "learning_rate": 1.7390686100248847e-05, + "loss": 1.2267550468444823, + "mean_token_accuracy": 0.7448949955403805, + "num_tokens": 2429083.0, + "step": 680 + }, + { + "entropy": 1.0907738648355008, + "epoch": 0.22082649917990158, + "grad_norm": 3.0625, + "learning_rate": 1.731958762886598e-05, + "loss": 1.139704990386963, + "mean_token_accuracy": 0.7525829285383224, + "num_tokens": 2466650.0, + "step": 690 + }, + { + "entropy": 1.263149094209075, + "epoch": 0.22402688322598713, + "grad_norm": 3.421875, + "learning_rate": 1.7248489157483116e-05, + "loss": 1.3419886589050294, + "mean_token_accuracy": 0.7293422102928162, + "num_tokens": 2502306.0, + "step": 700 + }, + { + "entropy": 1.0920586667954921, + "epoch": 0.22722726727207265, + "grad_norm": 3.484375, + "learning_rate": 1.717739068610025e-05, + "loss": 1.1174333572387696, + "mean_token_accuracy": 0.7606853067874908, + "num_tokens": 2535111.0, + "step": 710 + }, + { + "entropy": 1.1392102960497141, + "epoch": 0.23042765131815818, + "grad_norm": 4.03125, + "learning_rate": 1.7106292214717384e-05, + "loss": 1.2493625640869142, + "mean_token_accuracy": 0.7505488857626915, + "num_tokens": 2569698.0, + "step": 720 + }, + { + "entropy": 1.1885175816714764, + "epoch": 0.2336280353642437, + "grad_norm": 3.046875, + "learning_rate": 1.703519374333452e-05, + "loss": 1.2571531295776368, + "mean_token_accuracy": 0.7408242344856262, + "num_tokens": 2606190.0, + "step": 730 + }, + { + "entropy": 1.1989564672112465, + "epoch": 0.23682841941032923, + "grad_norm": 3.03125, + "learning_rate": 1.6964095271951656e-05, + "loss": 1.2598919868469238, + "mean_token_accuracy": 0.7383935242891312, + "num_tokens": 2643059.0, + "step": 740 + }, + { + "entropy": 1.0843516297638416, + "epoch": 0.24002880345641478, + "grad_norm": 3.265625, + "learning_rate": 1.6892996800568787e-05, + "loss": 1.1271354675292968, + "mean_token_accuracy": 0.7592886902391911, + "num_tokens": 2677827.0, + "step": 750 + }, + { + "entropy": 1.1722411584109067, + "epoch": 0.2432291875025003, + "grad_norm": 4.25, + "learning_rate": 1.6821898329185925e-05, + "loss": 1.2755705833435058, + "mean_token_accuracy": 0.7432561241090297, + "num_tokens": 2711876.0, + "step": 760 + }, + { + "entropy": 1.1528743766248226, + "epoch": 0.24642957154858583, + "grad_norm": 3.125, + "learning_rate": 1.675079985780306e-05, + "loss": 1.1825839042663575, + "mean_token_accuracy": 0.7412553071975708, + "num_tokens": 2753314.0, + "step": 770 + }, + { + "entropy": 1.1880130164325238, + "epoch": 0.24962995559467135, + "grad_norm": 3.453125, + "learning_rate": 1.6679701386420193e-05, + "loss": 1.2241481781005858, + "mean_token_accuracy": 0.7429691925644875, + "num_tokens": 2789807.0, + "step": 780 + }, + { + "entropy": 1.10604536421597, + "epoch": 0.2528303396407569, + "grad_norm": 3.171875, + "learning_rate": 1.6608602915037327e-05, + "loss": 1.133774185180664, + "mean_token_accuracy": 0.7567876607179642, + "num_tokens": 2822248.0, + "step": 790 + }, + { + "entropy": 1.1902866654098034, + "epoch": 0.25603072368684243, + "grad_norm": 3.734375, + "learning_rate": 1.653750444365446e-05, + "loss": 1.2399630546569824, + "mean_token_accuracy": 0.7448862664401531, + "num_tokens": 2858863.0, + "step": 800 + }, + { + "entropy": 1.0581065572798252, + "epoch": 0.25923110773292796, + "grad_norm": 3.625, + "learning_rate": 1.64664059722716e-05, + "loss": 1.1121423721313477, + "mean_token_accuracy": 0.7639019310474395, + "num_tokens": 2892587.0, + "step": 810 + }, + { + "entropy": 1.1235411427915096, + "epoch": 0.2624314917790135, + "grad_norm": 4.0, + "learning_rate": 1.6395307500888733e-05, + "loss": 1.1847190856933594, + "mean_token_accuracy": 0.7517336331307888, + "num_tokens": 2926589.0, + "step": 820 + }, + { + "entropy": 1.1476600162684918, + "epoch": 0.265631875825099, + "grad_norm": 3.375, + "learning_rate": 1.6324209029505868e-05, + "loss": 1.1705162048339843, + "mean_token_accuracy": 0.7479592509567737, + "num_tokens": 2963433.0, + "step": 830 + }, + { + "entropy": 1.1264712318778038, + "epoch": 0.26883225987118453, + "grad_norm": 3.625, + "learning_rate": 1.6253110558123002e-05, + "loss": 1.1919547080993653, + "mean_token_accuracy": 0.7536672279238701, + "num_tokens": 2997829.0, + "step": 840 + }, + { + "entropy": 1.139496796950698, + "epoch": 0.27203264391727006, + "grad_norm": 3.296875, + "learning_rate": 1.6182012086740136e-05, + "loss": 1.1688892364501953, + "mean_token_accuracy": 0.7543122127652169, + "num_tokens": 3032167.0, + "step": 850 + }, + { + "entropy": 1.1148510150611401, + "epoch": 0.2752330279633556, + "grad_norm": 4.40625, + "learning_rate": 1.611091361535727e-05, + "loss": 1.1610650062561034, + "mean_token_accuracy": 0.7572085842490196, + "num_tokens": 3066591.0, + "step": 860 + }, + { + "entropy": 1.1900723941624165, + "epoch": 0.2784334120094411, + "grad_norm": 3.59375, + "learning_rate": 1.6039815143974408e-05, + "loss": 1.2732099533081054, + "mean_token_accuracy": 0.7386304296553134, + "num_tokens": 3104866.0, + "step": 870 + }, + { + "entropy": 1.1151873588562011, + "epoch": 0.2816337960555267, + "grad_norm": 3.609375, + "learning_rate": 1.596871667259154e-05, + "loss": 1.194847583770752, + "mean_token_accuracy": 0.7530852198600769, + "num_tokens": 3140582.0, + "step": 880 + }, + { + "entropy": 1.1318743109703064, + "epoch": 0.2848341801016122, + "grad_norm": 3.5, + "learning_rate": 1.5897618201208676e-05, + "loss": 1.1644593238830567, + "mean_token_accuracy": 0.7507477700710297, + "num_tokens": 3172606.0, + "step": 890 + }, + { + "entropy": 1.1317258846014737, + "epoch": 0.28803456414769774, + "grad_norm": 3.109375, + "learning_rate": 1.582651972982581e-05, + "loss": 1.2329046249389648, + "mean_token_accuracy": 0.7497393228113651, + "num_tokens": 3211067.0, + "step": 900 + }, + { + "entropy": 1.0851800233125686, + "epoch": 0.29123494819378326, + "grad_norm": 2.890625, + "learning_rate": 1.5755421258442945e-05, + "loss": 1.0999431610107422, + "mean_token_accuracy": 0.757631991803646, + "num_tokens": 3245422.0, + "step": 910 + }, + { + "entropy": 1.159644392132759, + "epoch": 0.2944353322398688, + "grad_norm": 2.703125, + "learning_rate": 1.568432278706008e-05, + "loss": 1.2022081375122071, + "mean_token_accuracy": 0.7452364444732666, + "num_tokens": 3281658.0, + "step": 920 + }, + { + "entropy": 1.1171356670558452, + "epoch": 0.2976357162859543, + "grad_norm": 3.53125, + "learning_rate": 1.5613224315677213e-05, + "loss": 1.1644085884094237, + "mean_token_accuracy": 0.7566865622997284, + "num_tokens": 3313665.0, + "step": 930 + }, + { + "entropy": 1.1976551342755557, + "epoch": 0.30083610033203984, + "grad_norm": 2.71875, + "learning_rate": 1.554212584429435e-05, + "loss": 1.2674903869628906, + "mean_token_accuracy": 0.7411434464156628, + "num_tokens": 3351040.0, + "step": 940 + }, + { + "entropy": 1.1759327344596386, + "epoch": 0.30403648437812536, + "grad_norm": 3.140625, + "learning_rate": 1.5471027372911485e-05, + "loss": 1.2738938331604004, + "mean_token_accuracy": 0.7425175003707409, + "num_tokens": 3391557.0, + "step": 950 + }, + { + "entropy": 1.137892946600914, + "epoch": 0.3072368684242109, + "grad_norm": 3.34375, + "learning_rate": 1.539992890152862e-05, + "loss": 1.163702392578125, + "mean_token_accuracy": 0.7516510836780071, + "num_tokens": 3426954.0, + "step": 960 + }, + { + "entropy": 1.1136261202394961, + "epoch": 0.3104372524702964, + "grad_norm": 3.875, + "learning_rate": 1.5328830430145753e-05, + "loss": 1.1672924041748047, + "mean_token_accuracy": 0.7556798778474331, + "num_tokens": 3461984.0, + "step": 970 + }, + { + "entropy": 1.1157550118863582, + "epoch": 0.313637636516382, + "grad_norm": 3.359375, + "learning_rate": 1.5257731958762888e-05, + "loss": 1.177406406402588, + "mean_token_accuracy": 0.7517608553171158, + "num_tokens": 3496620.0, + "step": 980 + }, + { + "entropy": 1.1827008694410324, + "epoch": 0.3168380205624675, + "grad_norm": 3.140625, + "learning_rate": 1.5186633487380022e-05, + "loss": 1.2889988899230957, + "mean_token_accuracy": 0.7372566133737564, + "num_tokens": 3533170.0, + "step": 990 + }, + { + "entropy": 1.1758067298680543, + "epoch": 0.32003840460855304, + "grad_norm": 3.421875, + "learning_rate": 1.5115535015997158e-05, + "loss": 1.2371768951416016, + "mean_token_accuracy": 0.7458655416965485, + "num_tokens": 3567810.0, + "step": 1000 + }, + { + "entropy": 1.1418688822537661, + "epoch": 0.32323878865463856, + "grad_norm": 3.6875, + "learning_rate": 1.5044436544614292e-05, + "loss": 1.186594009399414, + "mean_token_accuracy": 0.7467859581112861, + "num_tokens": 3603353.0, + "step": 1010 + }, + { + "entropy": 1.1797917354851961, + "epoch": 0.3264391727007241, + "grad_norm": 2.96875, + "learning_rate": 1.4973338073231428e-05, + "loss": 1.2142827033996582, + "mean_token_accuracy": 0.7416288331151009, + "num_tokens": 3641475.0, + "step": 1020 + }, + { + "entropy": 1.1130448926240206, + "epoch": 0.3296395567468096, + "grad_norm": 3.109375, + "learning_rate": 1.490223960184856e-05, + "loss": 1.1746935844421387, + "mean_token_accuracy": 0.7544396013021469, + "num_tokens": 3675103.0, + "step": 1030 + }, + { + "entropy": 1.1146599553525447, + "epoch": 0.33283994079289514, + "grad_norm": 3.765625, + "learning_rate": 1.4831141130465696e-05, + "loss": 1.1683859825134277, + "mean_token_accuracy": 0.7539133220911026, + "num_tokens": 3710930.0, + "step": 1040 + }, + { + "entropy": 1.1133470050990582, + "epoch": 0.33604032483898066, + "grad_norm": 3.671875, + "learning_rate": 1.4760042659082832e-05, + "loss": 1.1540699005126953, + "mean_token_accuracy": 0.7554305769503117, + "num_tokens": 3744764.0, + "step": 1050 + }, + { + "entropy": 1.2018603175878524, + "epoch": 0.3392407088850662, + "grad_norm": 3.59375, + "learning_rate": 1.4688944187699965e-05, + "loss": 1.267392635345459, + "mean_token_accuracy": 0.741713160276413, + "num_tokens": 3781290.0, + "step": 1060 + }, + { + "entropy": 1.1392232250422238, + "epoch": 0.3424410929311517, + "grad_norm": 3.359375, + "learning_rate": 1.46178457163171e-05, + "loss": 1.2040764808654785, + "mean_token_accuracy": 0.7544347628951072, + "num_tokens": 3817138.0, + "step": 1070 + }, + { + "entropy": 1.0759797591716052, + "epoch": 0.3456414769772373, + "grad_norm": 3.609375, + "learning_rate": 1.4546747244934237e-05, + "loss": 1.1521276473999023, + "mean_token_accuracy": 0.762228213250637, + "num_tokens": 3852430.0, + "step": 1080 + }, + { + "entropy": 1.0658919643610716, + "epoch": 0.3488418610233228, + "grad_norm": 3.4375, + "learning_rate": 1.4475648773551369e-05, + "loss": 1.103238582611084, + "mean_token_accuracy": 0.7651597328484059, + "num_tokens": 3885416.0, + "step": 1090 + }, + { + "entropy": 1.1337019324302673, + "epoch": 0.35204224506940834, + "grad_norm": 3.234375, + "learning_rate": 1.4404550302168505e-05, + "loss": 1.1847347259521483, + "mean_token_accuracy": 0.7472888924181461, + "num_tokens": 3925885.0, + "step": 1100 + }, + { + "entropy": 1.124852604046464, + "epoch": 0.35524262911549387, + "grad_norm": 3.421875, + "learning_rate": 1.433345183078564e-05, + "loss": 1.166029167175293, + "mean_token_accuracy": 0.752812971919775, + "num_tokens": 3961029.0, + "step": 1110 + }, + { + "entropy": 1.1096541091799736, + "epoch": 0.3584430131615794, + "grad_norm": 3.21875, + "learning_rate": 1.4262353359402773e-05, + "loss": 1.1557273864746094, + "mean_token_accuracy": 0.7548218049108982, + "num_tokens": 3998345.0, + "step": 1120 + }, + { + "entropy": 1.1478759333491326, + "epoch": 0.3616433972076649, + "grad_norm": 3.59375, + "learning_rate": 1.419125488801991e-05, + "loss": 1.2064693450927735, + "mean_token_accuracy": 0.7461572416126728, + "num_tokens": 4032627.0, + "step": 1130 + }, + { + "entropy": 1.1037006203085185, + "epoch": 0.36484378125375044, + "grad_norm": 3.546875, + "learning_rate": 1.4120156416637044e-05, + "loss": 1.148093032836914, + "mean_token_accuracy": 0.7576483316719532, + "num_tokens": 4067935.0, + "step": 1140 + }, + { + "entropy": 1.1473457373678684, + "epoch": 0.36804416529983597, + "grad_norm": 3.375, + "learning_rate": 1.404905794525418e-05, + "loss": 1.2009618759155274, + "mean_token_accuracy": 0.751000577956438, + "num_tokens": 4105088.0, + "step": 1150 + }, + { + "entropy": 1.0734026059508324, + "epoch": 0.3712445493459215, + "grad_norm": 3.453125, + "learning_rate": 1.3977959473871312e-05, + "loss": 1.154836368560791, + "mean_token_accuracy": 0.7619151666760444, + "num_tokens": 4138662.0, + "step": 1160 + }, + { + "entropy": 1.106279893964529, + "epoch": 0.374444933392007, + "grad_norm": 3.265625, + "learning_rate": 1.3906861002488448e-05, + "loss": 1.1779499053955078, + "mean_token_accuracy": 0.7563652314245701, + "num_tokens": 4172413.0, + "step": 1170 + }, + { + "entropy": 1.1056948460638523, + "epoch": 0.3776453174380926, + "grad_norm": 3.5, + "learning_rate": 1.3835762531105584e-05, + "loss": 1.137861728668213, + "mean_token_accuracy": 0.7500715628266335, + "num_tokens": 4210692.0, + "step": 1180 + }, + { + "entropy": 1.0903994772583245, + "epoch": 0.3808457014841781, + "grad_norm": 3.09375, + "learning_rate": 1.3764664059722716e-05, + "loss": 1.1262723922729492, + "mean_token_accuracy": 0.7594648048281669, + "num_tokens": 4245849.0, + "step": 1190 + }, + { + "entropy": 1.1103523924946785, + "epoch": 0.38404608553026365, + "grad_norm": 3.46875, + "learning_rate": 1.3693565588339852e-05, + "loss": 1.1854586601257324, + "mean_token_accuracy": 0.7494940027594567, + "num_tokens": 4285585.0, + "step": 1200 + }, + { + "entropy": 1.2059497661888599, + "epoch": 0.3872464695763492, + "grad_norm": 3.609375, + "learning_rate": 1.3622467116956985e-05, + "loss": 1.2371363639831543, + "mean_token_accuracy": 0.7376365043222904, + "num_tokens": 4321004.0, + "step": 1210 + }, + { + "entropy": 1.0978240944445132, + "epoch": 0.3904468536224347, + "grad_norm": 3.53125, + "learning_rate": 1.355136864557412e-05, + "loss": 1.1326991081237794, + "mean_token_accuracy": 0.7563932791352272, + "num_tokens": 4354913.0, + "step": 1220 + }, + { + "entropy": 1.1807831916958094, + "epoch": 0.3936472376685202, + "grad_norm": 3.546875, + "learning_rate": 1.3480270174191257e-05, + "loss": 1.2626652717590332, + "mean_token_accuracy": 0.7423581592738628, + "num_tokens": 4390839.0, + "step": 1230 + }, + { + "entropy": 1.1470067836344242, + "epoch": 0.39684762171460575, + "grad_norm": 3.25, + "learning_rate": 1.340917170280839e-05, + "loss": 1.2084031105041504, + "mean_token_accuracy": 0.7459581665694713, + "num_tokens": 4430160.0, + "step": 1240 + }, + { + "entropy": 1.1267010770738124, + "epoch": 0.40004800576069127, + "grad_norm": 3.25, + "learning_rate": 1.3338073231425525e-05, + "loss": 1.1781652450561524, + "mean_token_accuracy": 0.7555549241602421, + "num_tokens": 4465773.0, + "step": 1250 + }, + { + "entropy": 1.0811494186520576, + "epoch": 0.4032483898067768, + "grad_norm": 3.28125, + "learning_rate": 1.3266974760042661e-05, + "loss": 1.105443000793457, + "mean_token_accuracy": 0.7561785206198692, + "num_tokens": 4506676.0, + "step": 1260 + }, + { + "entropy": 1.1555583395063878, + "epoch": 0.4064487738528623, + "grad_norm": 3.453125, + "learning_rate": 1.3195876288659795e-05, + "loss": 1.2035257339477539, + "mean_token_accuracy": 0.7430454775691032, + "num_tokens": 4544343.0, + "step": 1270 + }, + { + "entropy": 1.1449554897844791, + "epoch": 0.40964915789894785, + "grad_norm": 2.8125, + "learning_rate": 1.3124777817276931e-05, + "loss": 1.1892909049987792, + "mean_token_accuracy": 0.7470510423183441, + "num_tokens": 4580275.0, + "step": 1280 + }, + { + "entropy": 1.2319265089929103, + "epoch": 0.4128495419450334, + "grad_norm": 3.1875, + "learning_rate": 1.3053679345894064e-05, + "loss": 1.3125531196594238, + "mean_token_accuracy": 0.7335422746837139, + "num_tokens": 4619336.0, + "step": 1290 + }, + { + "entropy": 1.1227019898593427, + "epoch": 0.41604992599111895, + "grad_norm": 4.03125, + "learning_rate": 1.29825808745112e-05, + "loss": 1.1845033645629883, + "mean_token_accuracy": 0.7563824310898781, + "num_tokens": 4652166.0, + "step": 1300 + }, + { + "entropy": 1.0604043878614902, + "epoch": 0.4192503100372045, + "grad_norm": 3.390625, + "learning_rate": 1.2911482403128335e-05, + "loss": 1.100246524810791, + "mean_token_accuracy": 0.7669279538094997, + "num_tokens": 4684045.0, + "step": 1310 + }, + { + "entropy": 1.0832354299724103, + "epoch": 0.42245069408329, + "grad_norm": 3.40625, + "learning_rate": 1.2840383931745468e-05, + "loss": 1.1006074905395509, + "mean_token_accuracy": 0.7616540372371674, + "num_tokens": 4716393.0, + "step": 1320 + }, + { + "entropy": 1.1083704240620136, + "epoch": 0.4256510781293755, + "grad_norm": 3.5, + "learning_rate": 1.2769285460362604e-05, + "loss": 1.169978427886963, + "mean_token_accuracy": 0.7535225711762905, + "num_tokens": 4751437.0, + "step": 1330 + }, + { + "entropy": 1.0894863862544297, + "epoch": 0.42885146217546105, + "grad_norm": 2.96875, + "learning_rate": 1.2698186988979736e-05, + "loss": 1.132776641845703, + "mean_token_accuracy": 0.7577138744294644, + "num_tokens": 4788926.0, + "step": 1340 + }, + { + "entropy": 1.0949448980391026, + "epoch": 0.4320518462215466, + "grad_norm": 3.078125, + "learning_rate": 1.2627088517596872e-05, + "loss": 1.1491366386413575, + "mean_token_accuracy": 0.7596271999180317, + "num_tokens": 4825682.0, + "step": 1350 + }, + { + "entropy": 1.1359346587210895, + "epoch": 0.4352522302676321, + "grad_norm": 2.984375, + "learning_rate": 1.2555990046214008e-05, + "loss": 1.186843204498291, + "mean_token_accuracy": 0.7496764816343784, + "num_tokens": 4860636.0, + "step": 1360 + }, + { + "entropy": 1.1405926086008549, + "epoch": 0.4384526143137176, + "grad_norm": 2.765625, + "learning_rate": 1.2484891574831142e-05, + "loss": 1.2072190284729003, + "mean_token_accuracy": 0.7515955492854118, + "num_tokens": 4895008.0, + "step": 1370 + }, + { + "entropy": 1.188784885033965, + "epoch": 0.44165299835980315, + "grad_norm": 3.046875, + "learning_rate": 1.2413793103448277e-05, + "loss": 1.2465777397155762, + "mean_token_accuracy": 0.7442488387227059, + "num_tokens": 4934544.0, + "step": 1380 + }, + { + "entropy": 1.128134048730135, + "epoch": 0.44485338240588873, + "grad_norm": 3.109375, + "learning_rate": 1.2342694632065411e-05, + "loss": 1.1832526206970215, + "mean_token_accuracy": 0.7498147763311863, + "num_tokens": 4971830.0, + "step": 1390 + }, + { + "entropy": 1.0631931692361831, + "epoch": 0.44805376645197426, + "grad_norm": 3.234375, + "learning_rate": 1.2271596160682547e-05, + "loss": 1.0955133438110352, + "mean_token_accuracy": 0.7623707666993141, + "num_tokens": 5005602.0, + "step": 1400 + }, + { + "entropy": 1.1206502683460713, + "epoch": 0.4512541504980598, + "grad_norm": 3.6875, + "learning_rate": 1.2200497689299681e-05, + "loss": 1.1475549697875977, + "mean_token_accuracy": 0.7556483931839466, + "num_tokens": 5041164.0, + "step": 1410 + }, + { + "entropy": 1.1239325635135173, + "epoch": 0.4544545345441453, + "grad_norm": 3.109375, + "learning_rate": 1.2129399217916815e-05, + "loss": 1.166731357574463, + "mean_token_accuracy": 0.7522053651511669, + "num_tokens": 5078129.0, + "step": 1420 + }, + { + "entropy": 1.0859558291733264, + "epoch": 0.45765491859023083, + "grad_norm": 2.875, + "learning_rate": 1.2058300746533951e-05, + "loss": 1.1344684600830077, + "mean_token_accuracy": 0.7648185789585114, + "num_tokens": 5110323.0, + "step": 1430 + }, + { + "entropy": 1.101494075730443, + "epoch": 0.46085530263631636, + "grad_norm": 2.609375, + "learning_rate": 1.1987202275151084e-05, + "loss": 1.1418500900268556, + "mean_token_accuracy": 0.7596986934542656, + "num_tokens": 5145202.0, + "step": 1440 + }, + { + "entropy": 1.1369270034134389, + "epoch": 0.4640556866824019, + "grad_norm": 3.734375, + "learning_rate": 1.191610380376822e-05, + "loss": 1.1892250061035157, + "mean_token_accuracy": 0.752144105732441, + "num_tokens": 5179225.0, + "step": 1450 + }, + { + "entropy": 1.1221182450652123, + "epoch": 0.4672560707284874, + "grad_norm": 3.4375, + "learning_rate": 1.1845005332385355e-05, + "loss": 1.1399892807006835, + "mean_token_accuracy": 0.7525438450276851, + "num_tokens": 5213701.0, + "step": 1460 + }, + { + "entropy": 1.136232825368643, + "epoch": 0.47045645477457293, + "grad_norm": 2.890625, + "learning_rate": 1.1773906861002488e-05, + "loss": 1.1965130805969237, + "mean_token_accuracy": 0.7524074338376522, + "num_tokens": 5246772.0, + "step": 1470 + }, + { + "entropy": 1.1669704608619214, + "epoch": 0.47365683882065845, + "grad_norm": 3.296875, + "learning_rate": 1.1702808389619624e-05, + "loss": 1.2590208053588867, + "mean_token_accuracy": 0.7439424701035022, + "num_tokens": 5284402.0, + "step": 1480 + }, + { + "entropy": 1.1522224962711334, + "epoch": 0.47685722286674403, + "grad_norm": 3.234375, + "learning_rate": 1.163170991823676e-05, + "loss": 1.2222829818725587, + "mean_token_accuracy": 0.7506221950054168, + "num_tokens": 5318670.0, + "step": 1490 + }, + { + "entropy": 1.1795588433742523, + "epoch": 0.48005760691282956, + "grad_norm": 3.734375, + "learning_rate": 1.1560611446853894e-05, + "loss": 1.2275501251220704, + "mean_token_accuracy": 0.7433199048042297, + "num_tokens": 5353791.0, + "step": 1500 + }, + { + "entropy": 1.1336502090096474, + "epoch": 0.4832579909589151, + "grad_norm": 2.96875, + "learning_rate": 1.1489512975471028e-05, + "loss": 1.1771859169006347, + "mean_token_accuracy": 0.7523209981620311, + "num_tokens": 5387895.0, + "step": 1510 + }, + { + "entropy": 1.1373966462910174, + "epoch": 0.4864583750050006, + "grad_norm": 3.28125, + "learning_rate": 1.1418414504088162e-05, + "loss": 1.210038948059082, + "mean_token_accuracy": 0.7523334570229053, + "num_tokens": 5421253.0, + "step": 1520 + }, + { + "entropy": 1.1127303969115019, + "epoch": 0.48965875905108613, + "grad_norm": 3.203125, + "learning_rate": 1.1347316032705298e-05, + "loss": 1.1646257400512696, + "mean_token_accuracy": 0.7546454407274723, + "num_tokens": 5453927.0, + "step": 1530 + }, + { + "entropy": 1.0216515570878983, + "epoch": 0.49285914309717166, + "grad_norm": 3.734375, + "learning_rate": 1.1276217561322433e-05, + "loss": 1.0620564460754394, + "mean_token_accuracy": 0.7721667498350143, + "num_tokens": 5486994.0, + "step": 1540 + }, + { + "entropy": 1.1356555175036191, + "epoch": 0.4960595271432572, + "grad_norm": 3.390625, + "learning_rate": 1.1205119089939567e-05, + "loss": 1.1835213661193849, + "mean_token_accuracy": 0.7511269651353359, + "num_tokens": 5522667.0, + "step": 1550 + }, + { + "entropy": 1.1677010275423527, + "epoch": 0.4992599111893427, + "grad_norm": 3.25, + "learning_rate": 1.1134020618556703e-05, + "loss": 1.2388330459594727, + "mean_token_accuracy": 0.747014632821083, + "num_tokens": 5559683.0, + "step": 1560 + }, + { + "entropy": 1.0452032934874296, + "epoch": 0.5024602952354282, + "grad_norm": 3.09375, + "learning_rate": 1.1062922147173835e-05, + "loss": 1.0709638595581055, + "mean_token_accuracy": 0.7606437459588051, + "num_tokens": 5596318.0, + "step": 1570 + }, + { + "entropy": 1.0912953674793244, + "epoch": 0.5056606792815138, + "grad_norm": 3.59375, + "learning_rate": 1.0991823675790971e-05, + "loss": 1.1132530212402343, + "mean_token_accuracy": 0.7621871262788773, + "num_tokens": 5633417.0, + "step": 1580 + }, + { + "entropy": 1.0394235443323852, + "epoch": 0.5088610633275993, + "grad_norm": 3.734375, + "learning_rate": 1.0920725204408107e-05, + "loss": 1.0933416366577149, + "mean_token_accuracy": 0.7706859618425369, + "num_tokens": 5667849.0, + "step": 1590 + }, + { + "entropy": 1.0913284711539746, + "epoch": 0.5120614473736849, + "grad_norm": 3.203125, + "learning_rate": 1.084962673302524e-05, + "loss": 1.149476909637451, + "mean_token_accuracy": 0.7546762965619565, + "num_tokens": 5703443.0, + "step": 1600 + }, + { + "entropy": 1.0678091116249562, + "epoch": 0.5152618314197703, + "grad_norm": 3.515625, + "learning_rate": 1.0778528261642376e-05, + "loss": 1.117019271850586, + "mean_token_accuracy": 0.764804369956255, + "num_tokens": 5738612.0, + "step": 1610 + }, + { + "entropy": 1.079986510053277, + "epoch": 0.5184622154658559, + "grad_norm": 3.75, + "learning_rate": 1.070742979025951e-05, + "loss": 1.1276843070983886, + "mean_token_accuracy": 0.761953490972519, + "num_tokens": 5771959.0, + "step": 1620 + }, + { + "entropy": 1.1219593778252601, + "epoch": 0.5216625995119414, + "grad_norm": 3.3125, + "learning_rate": 1.0636331318876646e-05, + "loss": 1.1727846145629883, + "mean_token_accuracy": 0.7556007139384746, + "num_tokens": 5806127.0, + "step": 1630 + }, + { + "entropy": 1.1329836711287498, + "epoch": 0.524862983558027, + "grad_norm": 3.671875, + "learning_rate": 1.056523284749378e-05, + "loss": 1.1895696640014648, + "mean_token_accuracy": 0.7470721893012524, + "num_tokens": 5842096.0, + "step": 1640 + }, + { + "entropy": 1.138121072202921, + "epoch": 0.5280633676041125, + "grad_norm": 3.859375, + "learning_rate": 1.0494134376110914e-05, + "loss": 1.1915018081665039, + "mean_token_accuracy": 0.7498403996229172, + "num_tokens": 5879201.0, + "step": 1650 + }, + { + "entropy": 1.1121512524783612, + "epoch": 0.531263751650198, + "grad_norm": 3.703125, + "learning_rate": 1.042303590472805e-05, + "loss": 1.1451727867126464, + "mean_token_accuracy": 0.7555429771542549, + "num_tokens": 5914614.0, + "step": 1660 + }, + { + "entropy": 1.1279703747481107, + "epoch": 0.5344641356962836, + "grad_norm": 3.6875, + "learning_rate": 1.0351937433345184e-05, + "loss": 1.166687297821045, + "mean_token_accuracy": 0.7522629871964455, + "num_tokens": 5952819.0, + "step": 1670 + }, + { + "entropy": 1.077747032791376, + "epoch": 0.5376645197423691, + "grad_norm": 3.453125, + "learning_rate": 1.0280838961962318e-05, + "loss": 1.161451244354248, + "mean_token_accuracy": 0.7590492330491543, + "num_tokens": 5987954.0, + "step": 1680 + }, + { + "entropy": 1.1237775962799788, + "epoch": 0.5408649037884546, + "grad_norm": 3.90625, + "learning_rate": 1.0209740490579454e-05, + "loss": 1.189088726043701, + "mean_token_accuracy": 0.7514919534325599, + "num_tokens": 6026626.0, + "step": 1690 + }, + { + "entropy": 1.1484537214040755, + "epoch": 0.5440652878345401, + "grad_norm": 3.328125, + "learning_rate": 1.0138642019196587e-05, + "loss": 1.1888233184814454, + "mean_token_accuracy": 0.7529322817921639, + "num_tokens": 6061599.0, + "step": 1700 + }, + { + "entropy": 1.1195942271500825, + "epoch": 0.5472656718806257, + "grad_norm": 3.359375, + "learning_rate": 1.0067543547813723e-05, + "loss": 1.165010643005371, + "mean_token_accuracy": 0.7539791353046894, + "num_tokens": 6099492.0, + "step": 1710 + }, + { + "entropy": 1.105513620376587, + "epoch": 0.5504660559267112, + "grad_norm": 3.46875, + "learning_rate": 9.996445076430857e-06, + "loss": 1.1810117721557618, + "mean_token_accuracy": 0.7567639537155628, + "num_tokens": 6133396.0, + "step": 1720 + }, + { + "entropy": 1.0676775388419628, + "epoch": 0.5536664399727967, + "grad_norm": 2.953125, + "learning_rate": 9.925346605047991e-06, + "loss": 1.0936067581176758, + "mean_token_accuracy": 0.7665429212152958, + "num_tokens": 6170567.0, + "step": 1730 + }, + { + "entropy": 1.0640709735453129, + "epoch": 0.5568668240188822, + "grad_norm": 2.953125, + "learning_rate": 9.854248133665127e-06, + "loss": 1.1169232368469237, + "mean_token_accuracy": 0.7632769830524921, + "num_tokens": 6206392.0, + "step": 1740 + }, + { + "entropy": 1.0739900685846806, + "epoch": 0.5600672080649678, + "grad_norm": 3.359375, + "learning_rate": 9.783149662282261e-06, + "loss": 1.1153389930725097, + "mean_token_accuracy": 0.7636542163789273, + "num_tokens": 6240974.0, + "step": 1750 + }, + { + "entropy": 1.1187460146844388, + "epoch": 0.5632675921110534, + "grad_norm": 3.515625, + "learning_rate": 9.712051190899396e-06, + "loss": 1.213233757019043, + "mean_token_accuracy": 0.7538353092968464, + "num_tokens": 6276403.0, + "step": 1760 + }, + { + "entropy": 1.1634088471531867, + "epoch": 0.5664679761571388, + "grad_norm": 3.0, + "learning_rate": 9.640952719516532e-06, + "loss": 1.222030258178711, + "mean_token_accuracy": 0.7500277526676655, + "num_tokens": 6311555.0, + "step": 1770 + }, + { + "entropy": 1.124206557497382, + "epoch": 0.5696683602032244, + "grad_norm": 3.15625, + "learning_rate": 9.569854248133666e-06, + "loss": 1.166236114501953, + "mean_token_accuracy": 0.7465378858149052, + "num_tokens": 6349569.0, + "step": 1780 + }, + { + "entropy": 1.086308826133609, + "epoch": 0.5728687442493099, + "grad_norm": 3.921875, + "learning_rate": 9.498755776750802e-06, + "loss": 1.1150911331176758, + "mean_token_accuracy": 0.7645099796354771, + "num_tokens": 6381129.0, + "step": 1790 + }, + { + "entropy": 1.0747945971786976, + "epoch": 0.5760691282953955, + "grad_norm": 3.28125, + "learning_rate": 9.427657305367936e-06, + "loss": 1.107960319519043, + "mean_token_accuracy": 0.7637169934809208, + "num_tokens": 6417344.0, + "step": 1800 + }, + { + "entropy": 1.0966923542320728, + "epoch": 0.5792695123414809, + "grad_norm": 3.109375, + "learning_rate": 9.35655883398507e-06, + "loss": 1.1019758224487304, + "mean_token_accuracy": 0.7635716639459134, + "num_tokens": 6453097.0, + "step": 1810 + }, + { + "entropy": 1.0623582422733306, + "epoch": 0.5824698963875665, + "grad_norm": 3.09375, + "learning_rate": 9.285460362602204e-06, + "loss": 1.1345792770385743, + "mean_token_accuracy": 0.7689974352717399, + "num_tokens": 6485206.0, + "step": 1820 + }, + { + "entropy": 1.1150407858192921, + "epoch": 0.585670280433652, + "grad_norm": 3.640625, + "learning_rate": 9.21436189121934e-06, + "loss": 1.1899590492248535, + "mean_token_accuracy": 0.7584320530295372, + "num_tokens": 6518230.0, + "step": 1830 + }, + { + "entropy": 1.0844181418418883, + "epoch": 0.5888706644797376, + "grad_norm": 3.796875, + "learning_rate": 9.143263419836474e-06, + "loss": 1.1547722816467285, + "mean_token_accuracy": 0.7595572479069232, + "num_tokens": 6551894.0, + "step": 1840 + }, + { + "entropy": 1.137659491598606, + "epoch": 0.5920710485258232, + "grad_norm": 4.0625, + "learning_rate": 9.072164948453609e-06, + "loss": 1.215767002105713, + "mean_token_accuracy": 0.7511265553534031, + "num_tokens": 6586711.0, + "step": 1850 + }, + { + "entropy": 1.121223869174719, + "epoch": 0.5952714325719086, + "grad_norm": 3.28125, + "learning_rate": 9.001066477070743e-06, + "loss": 1.1829781532287598, + "mean_token_accuracy": 0.753217040002346, + "num_tokens": 6622616.0, + "step": 1860 + }, + { + "entropy": 1.1693847570568323, + "epoch": 0.5984718166179942, + "grad_norm": 3.546875, + "learning_rate": 8.929968005687877e-06, + "loss": 1.2190765380859374, + "mean_token_accuracy": 0.7442550092935563, + "num_tokens": 6657486.0, + "step": 1870 + }, + { + "entropy": 1.0904726900160313, + "epoch": 0.6016722006640797, + "grad_norm": 2.421875, + "learning_rate": 8.858869534305013e-06, + "loss": 1.1301965713500977, + "mean_token_accuracy": 0.7556341625750065, + "num_tokens": 6696509.0, + "step": 1880 + }, + { + "entropy": 1.085164299607277, + "epoch": 0.6048725847101653, + "grad_norm": 3.25, + "learning_rate": 8.787771062922147e-06, + "loss": 1.1311585426330566, + "mean_token_accuracy": 0.7562328241765499, + "num_tokens": 6733277.0, + "step": 1890 + }, + { + "entropy": 1.1221049219369887, + "epoch": 0.6080729687562507, + "grad_norm": 3.515625, + "learning_rate": 8.716672591539283e-06, + "loss": 1.151298999786377, + "mean_token_accuracy": 0.7528480552136898, + "num_tokens": 6771453.0, + "step": 1900 + }, + { + "entropy": 1.0780573837459086, + "epoch": 0.6112733528023363, + "grad_norm": 2.90625, + "learning_rate": 8.645574120156417e-06, + "loss": 1.1247942924499512, + "mean_token_accuracy": 0.7603120274841786, + "num_tokens": 6804903.0, + "step": 1910 + }, + { + "entropy": 1.2069343857467174, + "epoch": 0.6144737368484218, + "grad_norm": 2.78125, + "learning_rate": 8.574475648773553e-06, + "loss": 1.2467212677001953, + "mean_token_accuracy": 0.7350850224494934, + "num_tokens": 6844790.0, + "step": 1920 + }, + { + "entropy": 1.1031412109732628, + "epoch": 0.6176741208945074, + "grad_norm": 3.03125, + "learning_rate": 8.503377177390687e-06, + "loss": 1.1550896644592286, + "mean_token_accuracy": 0.7549904160201549, + "num_tokens": 6882003.0, + "step": 1930 + }, + { + "entropy": 1.0845932632684707, + "epoch": 0.6208745049405928, + "grad_norm": 3.625, + "learning_rate": 8.432278706007822e-06, + "loss": 1.110933780670166, + "mean_token_accuracy": 0.7586763650178909, + "num_tokens": 6919965.0, + "step": 1940 + }, + { + "entropy": 1.1197873912751675, + "epoch": 0.6240748889866784, + "grad_norm": 3.4375, + "learning_rate": 8.361180234624956e-06, + "loss": 1.185824203491211, + "mean_token_accuracy": 0.7533484481275081, + "num_tokens": 6954264.0, + "step": 1950 + }, + { + "entropy": 1.157302127033472, + "epoch": 0.627275273032764, + "grad_norm": 3.3125, + "learning_rate": 8.29008176324209e-06, + "loss": 1.1991801261901855, + "mean_token_accuracy": 0.7502691283822059, + "num_tokens": 6991391.0, + "step": 1960 + }, + { + "entropy": 1.1476174682378768, + "epoch": 0.6304756570788494, + "grad_norm": 3.1875, + "learning_rate": 8.218983291859226e-06, + "loss": 1.1750798225402832, + "mean_token_accuracy": 0.7515600122511387, + "num_tokens": 7028463.0, + "step": 1970 + }, + { + "entropy": 1.1342898778617383, + "epoch": 0.633676041124935, + "grad_norm": 3.328125, + "learning_rate": 8.14788482047636e-06, + "loss": 1.2121641159057617, + "mean_token_accuracy": 0.7529564268887043, + "num_tokens": 7064064.0, + "step": 1980 + }, + { + "entropy": 1.1010157510638237, + "epoch": 0.6368764251710205, + "grad_norm": 3.75, + "learning_rate": 8.076786349093494e-06, + "loss": 1.1628602981567382, + "mean_token_accuracy": 0.7538565069437027, + "num_tokens": 7098862.0, + "step": 1990 + }, + { + "entropy": 1.1578264623880385, + "epoch": 0.6400768092171061, + "grad_norm": 3.59375, + "learning_rate": 8.005687877710629e-06, + "loss": 1.2299229621887207, + "mean_token_accuracy": 0.7478043004870415, + "num_tokens": 7136374.0, + "step": 2000 + }, + { + "entropy": 1.0698354601860047, + "epoch": 0.6432771932631915, + "grad_norm": 3.390625, + "learning_rate": 7.934589406327765e-06, + "loss": 1.1068886756896972, + "mean_token_accuracy": 0.7641137100756168, + "num_tokens": 7169636.0, + "step": 2010 + }, + { + "entropy": 1.112225916981697, + "epoch": 0.6464775773092771, + "grad_norm": 3.453125, + "learning_rate": 7.863490934944899e-06, + "loss": 1.166696834564209, + "mean_token_accuracy": 0.7522329725325108, + "num_tokens": 7205044.0, + "step": 2020 + }, + { + "entropy": 1.1038395315408707, + "epoch": 0.6496779613553626, + "grad_norm": 3.5, + "learning_rate": 7.792392463562035e-06, + "loss": 1.1359478950500488, + "mean_token_accuracy": 0.756668771058321, + "num_tokens": 7240663.0, + "step": 2030 + }, + { + "entropy": 1.0801053799688816, + "epoch": 0.6528783454014482, + "grad_norm": 3.078125, + "learning_rate": 7.721293992179169e-06, + "loss": 1.119198989868164, + "mean_token_accuracy": 0.7593862563371658, + "num_tokens": 7278637.0, + "step": 2040 + }, + { + "entropy": 1.1753631062805652, + "epoch": 0.6560787294475338, + "grad_norm": 2.90625, + "learning_rate": 7.650195520796303e-06, + "loss": 1.2429065704345703, + "mean_token_accuracy": 0.7447149440646171, + "num_tokens": 7314162.0, + "step": 2050 + }, + { + "entropy": 1.0993228390812875, + "epoch": 0.6592791134936192, + "grad_norm": 3.390625, + "learning_rate": 7.579097049413438e-06, + "loss": 1.1387292861938476, + "mean_token_accuracy": 0.7615065090358257, + "num_tokens": 7349124.0, + "step": 2060 + }, + { + "entropy": 1.1077005062252283, + "epoch": 0.6624794975397048, + "grad_norm": 3.84375, + "learning_rate": 7.507998578030573e-06, + "loss": 1.2013302803039552, + "mean_token_accuracy": 0.7560870915651321, + "num_tokens": 7383405.0, + "step": 2070 + }, + { + "entropy": 1.1307024940848351, + "epoch": 0.6656798815857903, + "grad_norm": 4.28125, + "learning_rate": 7.4369001066477075e-06, + "loss": 1.198493766784668, + "mean_token_accuracy": 0.7485630966722965, + "num_tokens": 7414714.0, + "step": 2080 + }, + { + "entropy": 1.1086504600942135, + "epoch": 0.6688802656318759, + "grad_norm": 3.59375, + "learning_rate": 7.365801635264842e-06, + "loss": 1.1572361946105958, + "mean_token_accuracy": 0.7570925623178482, + "num_tokens": 7451556.0, + "step": 2090 + }, + { + "entropy": 1.0650083281099796, + "epoch": 0.6720806496779613, + "grad_norm": 3.109375, + "learning_rate": 7.294703163881978e-06, + "loss": 1.124489688873291, + "mean_token_accuracy": 0.7592034861445427, + "num_tokens": 7491824.0, + "step": 2100 + }, + { + "entropy": 1.160896249115467, + "epoch": 0.6752810337240469, + "grad_norm": 3.8125, + "learning_rate": 7.223604692499112e-06, + "loss": 1.2408259391784668, + "mean_token_accuracy": 0.7528047002851963, + "num_tokens": 7522213.0, + "step": 2110 + }, + { + "entropy": 1.131654118001461, + "epoch": 0.6784814177701324, + "grad_norm": 3.40625, + "learning_rate": 7.152506221116247e-06, + "loss": 1.1762347221374512, + "mean_token_accuracy": 0.7540035150945187, + "num_tokens": 7556942.0, + "step": 2120 + }, + { + "entropy": 1.1728335734456778, + "epoch": 0.681681801816218, + "grad_norm": 2.828125, + "learning_rate": 7.081407749733381e-06, + "loss": 1.2240229606628419, + "mean_token_accuracy": 0.7472974568605423, + "num_tokens": 7592279.0, + "step": 2130 + }, + { + "entropy": 1.1371040247380733, + "epoch": 0.6848821858623034, + "grad_norm": 3.5, + "learning_rate": 7.010309278350515e-06, + "loss": 1.1692868232727052, + "mean_token_accuracy": 0.7444953367114067, + "num_tokens": 7631064.0, + "step": 2140 + }, + { + "entropy": 1.079392648115754, + "epoch": 0.688082569908389, + "grad_norm": 3.5625, + "learning_rate": 6.939210806967651e-06, + "loss": 1.1272685050964355, + "mean_token_accuracy": 0.7608507804572582, + "num_tokens": 7663853.0, + "step": 2150 + }, + { + "entropy": 1.1790861997753381, + "epoch": 0.6912829539544746, + "grad_norm": 4.0, + "learning_rate": 6.8681123355847855e-06, + "loss": 1.271801471710205, + "mean_token_accuracy": 0.7439754210412503, + "num_tokens": 7699593.0, + "step": 2160 + }, + { + "entropy": 1.1712711922824384, + "epoch": 0.6944833380005601, + "grad_norm": 3.359375, + "learning_rate": 6.79701386420192e-06, + "loss": 1.2302053451538086, + "mean_token_accuracy": 0.7436926513910294, + "num_tokens": 7735279.0, + "step": 2170 + }, + { + "entropy": 1.0742683559656143, + "epoch": 0.6976837220466456, + "grad_norm": 3.28125, + "learning_rate": 6.725915392819055e-06, + "loss": 1.1179491043090821, + "mean_token_accuracy": 0.7624569363892079, + "num_tokens": 7771538.0, + "step": 2180 + }, + { + "entropy": 1.1455774445086717, + "epoch": 0.7008841060927311, + "grad_norm": 3.703125, + "learning_rate": 6.65481692143619e-06, + "loss": 1.210339641571045, + "mean_token_accuracy": 0.750506728887558, + "num_tokens": 7805334.0, + "step": 2190 + }, + { + "entropy": 1.1600065805017947, + "epoch": 0.7040844901388167, + "grad_norm": 3.296875, + "learning_rate": 6.583718450053325e-06, + "loss": 1.2067691802978515, + "mean_token_accuracy": 0.7510503888130188, + "num_tokens": 7838406.0, + "step": 2200 + }, + { + "entropy": 1.1392403941601514, + "epoch": 0.7072848741849022, + "grad_norm": 3.40625, + "learning_rate": 6.512619978670459e-06, + "loss": 1.162048053741455, + "mean_token_accuracy": 0.7528949834406375, + "num_tokens": 7871227.0, + "step": 2210 + }, + { + "entropy": 1.0721043154597283, + "epoch": 0.7104852582309877, + "grad_norm": 3.03125, + "learning_rate": 6.441521507287593e-06, + "loss": 1.1128012657165527, + "mean_token_accuracy": 0.7630540691316128, + "num_tokens": 7904703.0, + "step": 2220 + }, + { + "entropy": 1.0537261202931405, + "epoch": 0.7136856422770732, + "grad_norm": 3.078125, + "learning_rate": 6.3704230359047284e-06, + "loss": 1.0881397247314453, + "mean_token_accuracy": 0.7647965393960476, + "num_tokens": 7943799.0, + "step": 2230 + }, + { + "entropy": 1.1213313553482294, + "epoch": 0.7168860263231588, + "grad_norm": 3.09375, + "learning_rate": 6.2993245645218635e-06, + "loss": 1.161877155303955, + "mean_token_accuracy": 0.7525055252015591, + "num_tokens": 7979863.0, + "step": 2240 + }, + { + "entropy": 1.2231854621320963, + "epoch": 0.7200864103692443, + "grad_norm": 3.5625, + "learning_rate": 6.2282260931389986e-06, + "loss": 1.305109691619873, + "mean_token_accuracy": 0.7315145306289196, + "num_tokens": 8016742.0, + "step": 2250 + }, + { + "entropy": 1.0974082320928573, + "epoch": 0.7232867944153298, + "grad_norm": 2.9375, + "learning_rate": 6.157127621756133e-06, + "loss": 1.150672149658203, + "mean_token_accuracy": 0.7546444937586785, + "num_tokens": 8053669.0, + "step": 2260 + }, + { + "entropy": 1.1266177907586097, + "epoch": 0.7264871784614154, + "grad_norm": 3.265625, + "learning_rate": 6.086029150373267e-06, + "loss": 1.2110174179077149, + "mean_token_accuracy": 0.7550385102629662, + "num_tokens": 8087480.0, + "step": 2270 + }, + { + "entropy": 1.135270792245865, + "epoch": 0.7296875625075009, + "grad_norm": 3.21875, + "learning_rate": 6.014930678990403e-06, + "loss": 1.1860703468322753, + "mean_token_accuracy": 0.7481087513267994, + "num_tokens": 8123625.0, + "step": 2280 + }, + { + "entropy": 1.14946624673903, + "epoch": 0.7328879465535865, + "grad_norm": 3.453125, + "learning_rate": 5.943832207607537e-06, + "loss": 1.190632438659668, + "mean_token_accuracy": 0.7547764800488949, + "num_tokens": 8156665.0, + "step": 2290 + }, + { + "entropy": 1.1558411501348018, + "epoch": 0.7360883305996719, + "grad_norm": 3.1875, + "learning_rate": 5.872733736224671e-06, + "loss": 1.2208381652832032, + "mean_token_accuracy": 0.7456505544483661, + "num_tokens": 8193545.0, + "step": 2300 + }, + { + "entropy": 1.0852365911006927, + "epoch": 0.7392887146457575, + "grad_norm": 3.15625, + "learning_rate": 5.801635264841806e-06, + "loss": 1.1065022468566894, + "mean_token_accuracy": 0.7572917930781842, + "num_tokens": 8226934.0, + "step": 2310 + }, + { + "entropy": 1.0914320670068265, + "epoch": 0.742489098691843, + "grad_norm": 3.78125, + "learning_rate": 5.730536793458941e-06, + "loss": 1.099323844909668, + "mean_token_accuracy": 0.7600689142942428, + "num_tokens": 8260774.0, + "step": 2320 + }, + { + "entropy": 1.1091071009635924, + "epoch": 0.7456894827379286, + "grad_norm": 3.078125, + "learning_rate": 5.6594383220760765e-06, + "loss": 1.1712286949157715, + "mean_token_accuracy": 0.7567473009228707, + "num_tokens": 8295211.0, + "step": 2330 + }, + { + "entropy": 1.1220351219177247, + "epoch": 0.748889866784014, + "grad_norm": 3.140625, + "learning_rate": 5.588339850693211e-06, + "loss": 1.1657176971435548, + "mean_token_accuracy": 0.7564576506614685, + "num_tokens": 8330743.0, + "step": 2340 + }, + { + "entropy": 1.0823850885033608, + "epoch": 0.7520902508300996, + "grad_norm": 3.796875, + "learning_rate": 5.517241379310345e-06, + "loss": 1.1302215576171875, + "mean_token_accuracy": 0.7629231609404087, + "num_tokens": 8363593.0, + "step": 2350 + }, + { + "entropy": 1.114680102840066, + "epoch": 0.7552906348761852, + "grad_norm": 3.390625, + "learning_rate": 5.44614290792748e-06, + "loss": 1.176011848449707, + "mean_token_accuracy": 0.7503922112286091, + "num_tokens": 8402137.0, + "step": 2360 + }, + { + "entropy": 1.0473700985312462, + "epoch": 0.7584910189222707, + "grad_norm": 3.859375, + "learning_rate": 5.375044436544615e-06, + "loss": 1.085726547241211, + "mean_token_accuracy": 0.7641149386763573, + "num_tokens": 8438324.0, + "step": 2370 + }, + { + "entropy": 1.1461260944604874, + "epoch": 0.7616914029683562, + "grad_norm": 3.21875, + "learning_rate": 5.303945965161749e-06, + "loss": 1.2125227928161622, + "mean_token_accuracy": 0.7528879292309284, + "num_tokens": 8474170.0, + "step": 2380 + }, + { + "entropy": 1.1439074050635099, + "epoch": 0.7648917870144417, + "grad_norm": 3.5, + "learning_rate": 5.232847493778884e-06, + "loss": 1.1473745346069335, + "mean_token_accuracy": 0.7443250894546509, + "num_tokens": 8513458.0, + "step": 2390 + }, + { + "entropy": 1.0630555912852286, + "epoch": 0.7680921710605273, + "grad_norm": 3.5, + "learning_rate": 5.161749022396019e-06, + "loss": 1.1160799980163574, + "mean_token_accuracy": 0.7668336167931556, + "num_tokens": 8547193.0, + "step": 2400 + }, + { + "entropy": 1.0855234183371067, + "epoch": 0.7712925551066128, + "grad_norm": 3.171875, + "learning_rate": 5.090650551013153e-06, + "loss": 1.1209659576416016, + "mean_token_accuracy": 0.7573182880878448, + "num_tokens": 8583593.0, + "step": 2410 + }, + { + "entropy": 1.16197330057621, + "epoch": 0.7744929391526983, + "grad_norm": 3.4375, + "learning_rate": 5.019552079630289e-06, + "loss": 1.234616184234619, + "mean_token_accuracy": 0.7473996456712484, + "num_tokens": 8619710.0, + "step": 2420 + }, + { + "entropy": 1.2096669979393482, + "epoch": 0.7776933231987838, + "grad_norm": 3.421875, + "learning_rate": 4.948453608247423e-06, + "loss": 1.2796695709228516, + "mean_token_accuracy": 0.7381796896457672, + "num_tokens": 8657019.0, + "step": 2430 + }, + { + "entropy": 1.1043912775814533, + "epoch": 0.7808937072448694, + "grad_norm": 3.5, + "learning_rate": 4.877355136864558e-06, + "loss": 1.121436882019043, + "mean_token_accuracy": 0.75357426404953, + "num_tokens": 8693681.0, + "step": 2440 + }, + { + "entropy": 1.0841567002236843, + "epoch": 0.7840940912909549, + "grad_norm": 2.859375, + "learning_rate": 4.806256665481693e-06, + "loss": 1.104970645904541, + "mean_token_accuracy": 0.7571829192340374, + "num_tokens": 8730108.0, + "step": 2450 + }, + { + "entropy": 1.126195802539587, + "epoch": 0.7872944753370404, + "grad_norm": 3.359375, + "learning_rate": 4.735158194098827e-06, + "loss": 1.1975386619567872, + "mean_token_accuracy": 0.7514684118330479, + "num_tokens": 8764461.0, + "step": 2460 + }, + { + "entropy": 1.0726923126727343, + "epoch": 0.790494859383126, + "grad_norm": 3.90625, + "learning_rate": 4.6640597227159615e-06, + "loss": 1.1125378608703613, + "mean_token_accuracy": 0.7626473598182202, + "num_tokens": 8799890.0, + "step": 2470 + }, + { + "entropy": 1.1773266084492207, + "epoch": 0.7936952434292115, + "grad_norm": 3.4375, + "learning_rate": 4.592961251333097e-06, + "loss": 1.2108798980712892, + "mean_token_accuracy": 0.7410904221236706, + "num_tokens": 8838818.0, + "step": 2480 + }, + { + "entropy": 1.090137529373169, + "epoch": 0.7968956274752971, + "grad_norm": 3.59375, + "learning_rate": 4.521862779950232e-06, + "loss": 1.1148558616638184, + "mean_token_accuracy": 0.7648926541209221, + "num_tokens": 8870816.0, + "step": 2490 + }, + { + "entropy": 1.1689164392650127, + "epoch": 0.8000960115213825, + "grad_norm": 2.890625, + "learning_rate": 4.450764308567366e-06, + "loss": 1.2032492637634278, + "mean_token_accuracy": 0.7398371711373329, + "num_tokens": 8909484.0, + "step": 2500 + }, + { + "entropy": 1.0879596583545208, + "epoch": 0.8032963955674681, + "grad_norm": 3.203125, + "learning_rate": 4.379665837184501e-06, + "loss": 1.1078178405761718, + "mean_token_accuracy": 0.7553422212600708, + "num_tokens": 8944951.0, + "step": 2510 + }, + { + "entropy": 1.1411900214850903, + "epoch": 0.8064967796135536, + "grad_norm": 3.875, + "learning_rate": 4.308567365801636e-06, + "loss": 1.1979658126831054, + "mean_token_accuracy": 0.7543410055339337, + "num_tokens": 8979216.0, + "step": 2520 + }, + { + "entropy": 1.1815967209637166, + "epoch": 0.8096971636596392, + "grad_norm": 3.46875, + "learning_rate": 4.23746889441877e-06, + "loss": 1.2603289604187011, + "mean_token_accuracy": 0.7485571041703224, + "num_tokens": 9011929.0, + "step": 2530 + }, + { + "entropy": 1.1050000466406344, + "epoch": 0.8128975477057246, + "grad_norm": 3.328125, + "learning_rate": 4.166370423035905e-06, + "loss": 1.1503895759582519, + "mean_token_accuracy": 0.7591159790754318, + "num_tokens": 9045737.0, + "step": 2540 + }, + { + "entropy": 1.046723449230194, + "epoch": 0.8160979317518102, + "grad_norm": 3.1875, + "learning_rate": 4.0952719516530395e-06, + "loss": 1.0639203071594239, + "mean_token_accuracy": 0.7683053657412529, + "num_tokens": 9079676.0, + "step": 2550 + }, + { + "entropy": 1.1944607935845852, + "epoch": 0.8192983157978957, + "grad_norm": 3.125, + "learning_rate": 4.024173480270175e-06, + "loss": 1.247727394104004, + "mean_token_accuracy": 0.7391177780926228, + "num_tokens": 9114644.0, + "step": 2560 + }, + { + "entropy": 1.1602688152343035, + "epoch": 0.8224986998439813, + "grad_norm": 3.296875, + "learning_rate": 3.95307500888731e-06, + "loss": 1.2294767379760743, + "mean_token_accuracy": 0.7495340570807457, + "num_tokens": 9152236.0, + "step": 2570 + }, + { + "entropy": 1.2158122390508652, + "epoch": 0.8256990838900669, + "grad_norm": 2.90625, + "learning_rate": 3.881976537504444e-06, + "loss": 1.2398143768310548, + "mean_token_accuracy": 0.7380207255482674, + "num_tokens": 9190733.0, + "step": 2580 + }, + { + "entropy": 1.1768857415765523, + "epoch": 0.8288994679361523, + "grad_norm": 4.21875, + "learning_rate": 3.810878066121579e-06, + "loss": 1.2839090347290039, + "mean_token_accuracy": 0.7433359183371067, + "num_tokens": 9225408.0, + "step": 2590 + }, + { + "entropy": 1.0817514464259148, + "epoch": 0.8320998519822379, + "grad_norm": 2.890625, + "learning_rate": 3.739779594738713e-06, + "loss": 1.1311766624450683, + "mean_token_accuracy": 0.7620703734457492, + "num_tokens": 9259655.0, + "step": 2600 + }, + { + "entropy": 1.1603660874068737, + "epoch": 0.8353002360283234, + "grad_norm": 2.921875, + "learning_rate": 3.6686811233558482e-06, + "loss": 1.2468878746032714, + "mean_token_accuracy": 0.7444131776690484, + "num_tokens": 9296792.0, + "step": 2610 + }, + { + "entropy": 1.1298386432230472, + "epoch": 0.838500620074409, + "grad_norm": 3.1875, + "learning_rate": 3.597582651972983e-06, + "loss": 1.1794001579284668, + "mean_token_accuracy": 0.7496243976056576, + "num_tokens": 9333174.0, + "step": 2620 + }, + { + "entropy": 1.1448624573647976, + "epoch": 0.8417010041204944, + "grad_norm": 3.125, + "learning_rate": 3.526484180590118e-06, + "loss": 1.1680842399597169, + "mean_token_accuracy": 0.7462442465126514, + "num_tokens": 9374875.0, + "step": 2630 + }, + { + "entropy": 1.1212282598018646, + "epoch": 0.84490138816658, + "grad_norm": 3.5, + "learning_rate": 3.455385709207252e-06, + "loss": 1.18281888961792, + "mean_token_accuracy": 0.7593762136995792, + "num_tokens": 9408490.0, + "step": 2640 + }, + { + "entropy": 1.1325732119381429, + "epoch": 0.8481017722126655, + "grad_norm": 3.578125, + "learning_rate": 3.384287237824387e-06, + "loss": 1.1580286979675294, + "mean_token_accuracy": 0.7503605335950851, + "num_tokens": 9443447.0, + "step": 2650 + }, + { + "entropy": 1.142949765175581, + "epoch": 0.851302156258751, + "grad_norm": 3.203125, + "learning_rate": 3.313188766441522e-06, + "loss": 1.1941743850708009, + "mean_token_accuracy": 0.752988663315773, + "num_tokens": 9478354.0, + "step": 2660 + }, + { + "entropy": 1.1249619655311107, + "epoch": 0.8545025403048366, + "grad_norm": 3.46875, + "learning_rate": 3.2420902950586565e-06, + "loss": 1.2092914581298828, + "mean_token_accuracy": 0.7525562759488821, + "num_tokens": 9517475.0, + "step": 2670 + }, + { + "entropy": 1.1741073250770568, + "epoch": 0.8577029243509221, + "grad_norm": 3.578125, + "learning_rate": 3.1709918236757916e-06, + "loss": 1.262247371673584, + "mean_token_accuracy": 0.7427960857748985, + "num_tokens": 9553289.0, + "step": 2680 + }, + { + "entropy": 1.105969700962305, + "epoch": 0.8609033083970077, + "grad_norm": 3.3125, + "learning_rate": 3.0998933522929258e-06, + "loss": 1.1461322784423829, + "mean_token_accuracy": 0.75559606179595, + "num_tokens": 9588549.0, + "step": 2690 + }, + { + "entropy": 1.1349318251013756, + "epoch": 0.8641036924430932, + "grad_norm": 3.078125, + "learning_rate": 3.028794880910061e-06, + "loss": 1.1659625053405762, + "mean_token_accuracy": 0.7462083771824837, + "num_tokens": 9625305.0, + "step": 2700 + }, + { + "entropy": 1.0998256973922254, + "epoch": 0.8673040764891787, + "grad_norm": 2.90625, + "learning_rate": 2.9576964095271955e-06, + "loss": 1.173351001739502, + "mean_token_accuracy": 0.7538725100457668, + "num_tokens": 9663574.0, + "step": 2710 + }, + { + "entropy": 1.1989767000079155, + "epoch": 0.8705044605352642, + "grad_norm": 3.515625, + "learning_rate": 2.8865979381443297e-06, + "loss": 1.278292465209961, + "mean_token_accuracy": 0.737822500616312, + "num_tokens": 9699248.0, + "step": 2720 + }, + { + "entropy": 1.131580077856779, + "epoch": 0.8737048445813498, + "grad_norm": 3.1875, + "learning_rate": 2.8154994667614648e-06, + "loss": 1.1636852264404296, + "mean_token_accuracy": 0.7542126163840294, + "num_tokens": 9733347.0, + "step": 2730 + }, + { + "entropy": 1.0964635238051414, + "epoch": 0.8769052286274353, + "grad_norm": 3.6875, + "learning_rate": 2.7444009953785994e-06, + "loss": 1.1526992797851563, + "mean_token_accuracy": 0.7599952027201653, + "num_tokens": 9770524.0, + "step": 2740 + }, + { + "entropy": 1.0702364332973957, + "epoch": 0.8801056126735208, + "grad_norm": 3.59375, + "learning_rate": 2.6733025239957345e-06, + "loss": 1.0925715446472168, + "mean_token_accuracy": 0.7613530211150646, + "num_tokens": 9805904.0, + "step": 2750 + }, + { + "entropy": 1.143309585005045, + "epoch": 0.8833059967196063, + "grad_norm": 3.4375, + "learning_rate": 2.6022040526128687e-06, + "loss": 1.185093879699707, + "mean_token_accuracy": 0.748717375099659, + "num_tokens": 9843097.0, + "step": 2760 + }, + { + "entropy": 1.1525050312280656, + "epoch": 0.8865063807656919, + "grad_norm": 3.484375, + "learning_rate": 2.5311055812300038e-06, + "loss": 1.186710262298584, + "mean_token_accuracy": 0.7498737536370754, + "num_tokens": 9878625.0, + "step": 2770 + }, + { + "entropy": 1.096405889093876, + "epoch": 0.8897067648117775, + "grad_norm": 3.0625, + "learning_rate": 2.4600071098471384e-06, + "loss": 1.1552643775939941, + "mean_token_accuracy": 0.7564327225089074, + "num_tokens": 9913940.0, + "step": 2780 + }, + { + "entropy": 1.075688973069191, + "epoch": 0.8929071488578629, + "grad_norm": 3.59375, + "learning_rate": 2.388908638464273e-06, + "loss": 1.0996969223022461, + "mean_token_accuracy": 0.7608482711017132, + "num_tokens": 9950221.0, + "step": 2790 + }, + { + "entropy": 1.1116474494338036, + "epoch": 0.8961075329039485, + "grad_norm": 3.578125, + "learning_rate": 2.317810167081408e-06, + "loss": 1.183712863922119, + "mean_token_accuracy": 0.7586163900792599, + "num_tokens": 9982826.0, + "step": 2800 + }, + { + "entropy": 1.1222521997988224, + "epoch": 0.899307916950034, + "grad_norm": 3.296875, + "learning_rate": 2.2467116956985428e-06, + "loss": 1.1775464057922362, + "mean_token_accuracy": 0.7527868509292602, + "num_tokens": 10018325.0, + "step": 2810 + }, + { + "entropy": 1.12875221632421, + "epoch": 0.9025083009961196, + "grad_norm": 3.328125, + "learning_rate": 2.1756132243156774e-06, + "loss": 1.206295108795166, + "mean_token_accuracy": 0.7542196542024613, + "num_tokens": 10052547.0, + "step": 2820 + }, + { + "entropy": 1.1360722210258245, + "epoch": 0.905708685042205, + "grad_norm": 3.515625, + "learning_rate": 2.104514752932812e-06, + "loss": 1.1869568824768066, + "mean_token_accuracy": 0.7474234350025654, + "num_tokens": 10091594.0, + "step": 2830 + }, + { + "entropy": 1.0468792729079723, + "epoch": 0.9089090690882906, + "grad_norm": 3.703125, + "learning_rate": 2.033416281549947e-06, + "loss": 1.0732879638671875, + "mean_token_accuracy": 0.7691854566335679, + "num_tokens": 10126242.0, + "step": 2840 + }, + { + "entropy": 1.1514717623591424, + "epoch": 0.9121094531343761, + "grad_norm": 3.265625, + "learning_rate": 1.9623178101670813e-06, + "loss": 1.196326732635498, + "mean_token_accuracy": 0.7465457141399383, + "num_tokens": 10162663.0, + "step": 2850 + }, + { + "entropy": 1.1170444838702678, + "epoch": 0.9153098371804617, + "grad_norm": 2.8125, + "learning_rate": 1.8912193387842162e-06, + "loss": 1.1753318786621094, + "mean_token_accuracy": 0.7546933814883232, + "num_tokens": 10199344.0, + "step": 2860 + }, + { + "entropy": 1.1827090494334698, + "epoch": 0.9185102212265472, + "grad_norm": 3.625, + "learning_rate": 1.820120867401351e-06, + "loss": 1.2415277481079101, + "mean_token_accuracy": 0.739153602719307, + "num_tokens": 10233777.0, + "step": 2870 + }, + { + "entropy": 1.0766679864376782, + "epoch": 0.9217106052726327, + "grad_norm": 3.6875, + "learning_rate": 1.7490223960184857e-06, + "loss": 1.1411251068115233, + "mean_token_accuracy": 0.7570614032447338, + "num_tokens": 10270302.0, + "step": 2880 + }, + { + "entropy": 1.0990701586008071, + "epoch": 0.9249109893187183, + "grad_norm": 3.421875, + "learning_rate": 1.6779239246356205e-06, + "loss": 1.1710631370544433, + "mean_token_accuracy": 0.7585064023733139, + "num_tokens": 10305178.0, + "step": 2890 + }, + { + "entropy": 1.0720692560076714, + "epoch": 0.9281113733648038, + "grad_norm": 3.21875, + "learning_rate": 1.6068254532527552e-06, + "loss": 1.0985607147216796, + "mean_token_accuracy": 0.7623618088662625, + "num_tokens": 10341351.0, + "step": 2900 + }, + { + "entropy": 1.0809118885546922, + "epoch": 0.9313117574108893, + "grad_norm": 3.34375, + "learning_rate": 1.53572698186989e-06, + "loss": 1.1065154075622559, + "mean_token_accuracy": 0.7592159628868103, + "num_tokens": 10378308.0, + "step": 2910 + }, + { + "entropy": 1.0883469644933939, + "epoch": 0.9345121414569748, + "grad_norm": 3.140625, + "learning_rate": 1.4646285104870247e-06, + "loss": 1.1307219505310058, + "mean_token_accuracy": 0.7604426823556423, + "num_tokens": 10411774.0, + "step": 2920 + }, + { + "entropy": 1.1618533357977867, + "epoch": 0.9377125255030604, + "grad_norm": 3.75, + "learning_rate": 1.3935300391041593e-06, + "loss": 1.2212342262268066, + "mean_token_accuracy": 0.746428107470274, + "num_tokens": 10447555.0, + "step": 2930 + }, + { + "entropy": 1.0985857374966144, + "epoch": 0.9409129095491459, + "grad_norm": 3.1875, + "learning_rate": 1.322431567721294e-06, + "loss": 1.1086782455444335, + "mean_token_accuracy": 0.7579902283847332, + "num_tokens": 10485342.0, + "step": 2940 + }, + { + "entropy": 1.113938895612955, + "epoch": 0.9441132935952314, + "grad_norm": 3.4375, + "learning_rate": 1.2513330963384288e-06, + "loss": 1.1549474716186523, + "mean_token_accuracy": 0.7552253149449826, + "num_tokens": 10519459.0, + "step": 2950 + }, + { + "entropy": 1.1388332910835743, + "epoch": 0.9473136776413169, + "grad_norm": 3.234375, + "learning_rate": 1.1802346249555635e-06, + "loss": 1.213517189025879, + "mean_token_accuracy": 0.7471270561218262, + "num_tokens": 10555227.0, + "step": 2960 + }, + { + "entropy": 1.0419878501445055, + "epoch": 0.9505140616874025, + "grad_norm": 3.359375, + "learning_rate": 1.1091361535726983e-06, + "loss": 1.085500717163086, + "mean_token_accuracy": 0.7708059817552566, + "num_tokens": 10590479.0, + "step": 2970 + }, + { + "entropy": 1.1476553842425345, + "epoch": 0.9537144457334881, + "grad_norm": 3.40625, + "learning_rate": 1.038037682189833e-06, + "loss": 1.1976530075073242, + "mean_token_accuracy": 0.7469619035720825, + "num_tokens": 10626605.0, + "step": 2980 + }, + { + "entropy": 1.126973857730627, + "epoch": 0.9569148297795735, + "grad_norm": 3.578125, + "learning_rate": 9.669392108069678e-07, + "loss": 1.1434929847717286, + "mean_token_accuracy": 0.7535457745194435, + "num_tokens": 10660781.0, + "step": 2990 + }, + { + "entropy": 1.0904895570129156, + "epoch": 0.9601152138256591, + "grad_norm": 3.59375, + "learning_rate": 8.958407394241024e-07, + "loss": 1.144233798980713, + "mean_token_accuracy": 0.7603662610054016, + "num_tokens": 10695321.0, + "step": 3000 + }, + { + "entropy": 1.0829311583191157, + "epoch": 0.9633155978717446, + "grad_norm": 3.3125, + "learning_rate": 8.247422680412372e-07, + "loss": 1.1176697731018066, + "mean_token_accuracy": 0.7594211801886559, + "num_tokens": 10730739.0, + "step": 3010 + }, + { + "entropy": 1.1191859498620034, + "epoch": 0.9665159819178302, + "grad_norm": 3.9375, + "learning_rate": 7.536437966583719e-07, + "loss": 1.1339409828186036, + "mean_token_accuracy": 0.7513566389679909, + "num_tokens": 10765642.0, + "step": 3020 + }, + { + "entropy": 1.1836695678532123, + "epoch": 0.9697163659639156, + "grad_norm": 3.296875, + "learning_rate": 6.825453252755067e-07, + "loss": 1.2448589324951171, + "mean_token_accuracy": 0.7450046464800835, + "num_tokens": 10798289.0, + "step": 3030 + }, + { + "entropy": 1.12890649959445, + "epoch": 0.9729167500100012, + "grad_norm": 3.078125, + "learning_rate": 6.114468538926413e-07, + "loss": 1.1511384010314942, + "mean_token_accuracy": 0.7531974203884602, + "num_tokens": 10833449.0, + "step": 3040 + }, + { + "entropy": 1.148883668333292, + "epoch": 0.9761171340560867, + "grad_norm": 3.1875, + "learning_rate": 5.403483825097761e-07, + "loss": 1.2151781082153321, + "mean_token_accuracy": 0.7502546660602093, + "num_tokens": 10868447.0, + "step": 3050 + }, + { + "entropy": 1.0672550223767758, + "epoch": 0.9793175181021723, + "grad_norm": 3.109375, + "learning_rate": 4.6924991112691083e-07, + "loss": 1.082494354248047, + "mean_token_accuracy": 0.7678111597895623, + "num_tokens": 10901598.0, + "step": 3060 + }, + { + "entropy": 1.042473478242755, + "epoch": 0.9825179021482577, + "grad_norm": 3.6875, + "learning_rate": 3.9815143974404553e-07, + "loss": 1.114789867401123, + "mean_token_accuracy": 0.772429596632719, + "num_tokens": 10933312.0, + "step": 3070 + }, + { + "entropy": 1.1300311595201493, + "epoch": 0.9857182861943433, + "grad_norm": 3.640625, + "learning_rate": 3.270529683611803e-07, + "loss": 1.2087718009948731, + "mean_token_accuracy": 0.7560340866446496, + "num_tokens": 10966726.0, + "step": 3080 + }, + { + "entropy": 1.1198654279112816, + "epoch": 0.9889186702404289, + "grad_norm": 3.484375, + "learning_rate": 2.5595449697831497e-07, + "loss": 1.1944404602050782, + "mean_token_accuracy": 0.7567917600274086, + "num_tokens": 11002300.0, + "step": 3090 + }, + { + "entropy": 1.1630706571042537, + "epoch": 0.9921190542865144, + "grad_norm": 3.734375, + "learning_rate": 1.8485602559544972e-07, + "loss": 1.2107239723205567, + "mean_token_accuracy": 0.7448431417346001, + "num_tokens": 11039145.0, + "step": 3100 + }, + { + "entropy": 1.0874366093426944, + "epoch": 0.9953194383326, + "grad_norm": 3.328125, + "learning_rate": 1.1375755421258444e-07, + "loss": 1.1232175827026367, + "mean_token_accuracy": 0.7573442026972771, + "num_tokens": 11074957.0, + "step": 3110 + }, + { + "entropy": 1.163031455129385, + "epoch": 0.9985198223786854, + "grad_norm": 3.140625, + "learning_rate": 4.265908282971917e-08, + "loss": 1.1890130996704102, + "mean_token_accuracy": 0.7419414661824704, + "num_tokens": 11114151.0, + "step": 3120 + } + ], + "logging_steps": 10, + "max_steps": 3125, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.2571499019753062e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-3125/training_args.bin b/checkpoint-3125/training_args.bin new file mode 100644 index 0000000..63f4fe5 --- /dev/null +++ b/checkpoint-3125/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441c0f259e1061b526f1dd66cbe2e44a8dc117cd80c34164eaa0ec7b80ae108d +size 5713 diff --git a/checkpoint-4375/chat_template.jinja b/checkpoint-4375/chat_template.jinja new file mode 100644 index 0000000..a43db74 --- /dev/null +++ b/checkpoint-4375/chat_template.jinja @@ -0,0 +1,94 @@ +{%- set enable_thinking = true %} +{%- set default_system_message = "You are a knowledgeable assistant. For multiple-choice questions, reason step by step, then write only the letter of the correct answer inside \\boxed{}." %} +{%- if messages[0]['role'] != 'system' %} + {%- set messages = [{'role': 'system', 'content': default_system_message}] + messages %} +{%- endif %} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-4375/config.json b/checkpoint-4375/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-4375/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-4375/generation_config.json b/checkpoint-4375/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-4375/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-4375/model.safetensors b/checkpoint-4375/model.safetensors new file mode 100644 index 0000000..d76ac9a --- /dev/null +++ b/checkpoint-4375/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cd6cb79c7a5acadd07e56f559eb72c54302262e65ef715815d566bffea239d6 +size 3441185608 diff --git a/checkpoint-4375/optimizer.pt b/checkpoint-4375/optimizer.pt new file mode 100644 index 0000000..329fa26 --- /dev/null +++ b/checkpoint-4375/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e9040dcc23135b96bf09f8eb9d08474b107de832f349918ceb9444e47941d36 +size 6882572207 diff --git a/checkpoint-4375/rng_state.pth b/checkpoint-4375/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-4375/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-4375/scheduler.pt b/checkpoint-4375/scheduler.pt new file mode 100644 index 0000000..3806e1f --- /dev/null +++ b/checkpoint-4375/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ddd71d29679ee72f9abf0313d554a64d313fe2aee3d0782e3b10fb0236e86e +size 1465 diff --git a/checkpoint-4375/tokenizer.json b/checkpoint-4375/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-4375/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-4375/tokenizer_config.json b/checkpoint-4375/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-4375/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-4375/trainer_state.json b/checkpoint-4375/trainer_state.json new file mode 100644 index 0000000..e9afecf --- /dev/null +++ b/checkpoint-4375/trainer_state.json @@ -0,0 +1,4404 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 4375, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.4608087057247758, + "epoch": 0.002285714285714286, + "grad_norm": 262.0, + "learning_rate": 8.256880733944956e-07, + "loss": 4.128694152832031, + "mean_token_accuracy": 0.5559752065688371, + "num_tokens": 30583.0, + "step": 10 + }, + { + "entropy": 0.4808954084292054, + "epoch": 0.004571428571428572, + "grad_norm": 208.0, + "learning_rate": 1.743119266055046e-06, + "loss": 4.0306652069091795, + "mean_token_accuracy": 0.5491989776492119, + "num_tokens": 64875.0, + "step": 20 + }, + { + "entropy": 0.4882832657545805, + "epoch": 0.006857142857142857, + "grad_norm": 88.0, + "learning_rate": 2.6605504587155968e-06, + "loss": 3.4539260864257812, + "mean_token_accuracy": 0.5994260810315609, + "num_tokens": 96479.0, + "step": 30 + }, + { + "entropy": 0.6207152051851154, + "epoch": 0.009142857142857144, + "grad_norm": 50.25, + "learning_rate": 3.5779816513761473e-06, + "loss": 3.065783882141113, + "mean_token_accuracy": 0.6057824719697237, + "num_tokens": 129267.0, + "step": 40 + }, + { + "entropy": 0.7509429801255465, + "epoch": 0.011428571428571429, + "grad_norm": 47.5, + "learning_rate": 4.4954128440366975e-06, + "loss": 2.414494514465332, + "mean_token_accuracy": 0.6280621752142906, + "num_tokens": 157969.0, + "step": 50 + }, + { + "entropy": 0.9442974735051394, + "epoch": 0.013714285714285714, + "grad_norm": 21.625, + "learning_rate": 5.412844036697248e-06, + "loss": 2.0897136688232423, + "mean_token_accuracy": 0.6582186311483383, + "num_tokens": 191753.0, + "step": 60 + }, + { + "entropy": 1.133295001834631, + "epoch": 0.016, + "grad_norm": 15.6875, + "learning_rate": 6.330275229357799e-06, + "loss": 1.9386701583862305, + "mean_token_accuracy": 0.6817213766276836, + "num_tokens": 224874.0, + "step": 70 + }, + { + "entropy": 1.161771859228611, + "epoch": 0.018285714285714287, + "grad_norm": 18.875, + "learning_rate": 7.247706422018349e-06, + "loss": 1.6953561782836915, + "mean_token_accuracy": 0.7058230180293321, + "num_tokens": 259557.0, + "step": 80 + }, + { + "entropy": 1.0966269705444573, + "epoch": 0.02057142857142857, + "grad_norm": 15.0625, + "learning_rate": 8.1651376146789e-06, + "loss": 1.4140602111816407, + "mean_token_accuracy": 0.7321123115718364, + "num_tokens": 290863.0, + "step": 90 + }, + { + "entropy": 1.0357405820861458, + "epoch": 0.022857142857142857, + "grad_norm": 16.75, + "learning_rate": 9.08256880733945e-06, + "loss": 1.3797801971435546, + "mean_token_accuracy": 0.7626342628151178, + "num_tokens": 324054.0, + "step": 100 + }, + { + "entropy": 0.9538093984127045, + "epoch": 0.025142857142857144, + "grad_norm": 11.75, + "learning_rate": 1e-05, + "loss": 1.3015310287475585, + "mean_token_accuracy": 0.7864577785134316, + "num_tokens": 358048.0, + "step": 110 + }, + { + "entropy": 0.931211198028177, + "epoch": 0.027428571428571427, + "grad_norm": 12.625, + "learning_rate": 1.091743119266055e-05, + "loss": 1.2297636985778808, + "mean_token_accuracy": 0.7865072574466467, + "num_tokens": 389056.0, + "step": 120 + }, + { + "entropy": 0.9721011022105813, + "epoch": 0.029714285714285714, + "grad_norm": 15.125, + "learning_rate": 1.1834862385321102e-05, + "loss": 1.3690268516540527, + "mean_token_accuracy": 0.7615244656801223, + "num_tokens": 422901.0, + "step": 130 + }, + { + "entropy": 1.0186741236597299, + "epoch": 0.032, + "grad_norm": 10.3125, + "learning_rate": 1.2752293577981652e-05, + "loss": 1.3675042152404786, + "mean_token_accuracy": 0.7797191683202982, + "num_tokens": 452506.0, + "step": 140 + }, + { + "entropy": 0.9464143239893019, + "epoch": 0.03428571428571429, + "grad_norm": 13.0625, + "learning_rate": 1.3669724770642203e-05, + "loss": 1.3897374153137207, + "mean_token_accuracy": 0.7744678042829036, + "num_tokens": 483811.0, + "step": 150 + }, + { + "entropy": 0.9546993720345199, + "epoch": 0.036571428571428574, + "grad_norm": 10.0625, + "learning_rate": 1.4587155963302753e-05, + "loss": 1.2690893173217774, + "mean_token_accuracy": 0.7874016337096691, + "num_tokens": 511576.0, + "step": 160 + }, + { + "entropy": 1.002905413042754, + "epoch": 0.038857142857142854, + "grad_norm": 9.75, + "learning_rate": 1.5504587155963304e-05, + "loss": 1.3503832817077637, + "mean_token_accuracy": 0.7721866790205241, + "num_tokens": 543265.0, + "step": 170 + }, + { + "entropy": 0.7613611572422088, + "epoch": 0.04114285714285714, + "grad_norm": 10.4375, + "learning_rate": 1.6422018348623852e-05, + "loss": 0.977170753479004, + "mean_token_accuracy": 0.8188995130360126, + "num_tokens": 576882.0, + "step": 180 + }, + { + "entropy": 0.9535474652424455, + "epoch": 0.04342857142857143, + "grad_norm": 10.5625, + "learning_rate": 1.7339449541284407e-05, + "loss": 1.2448718070983886, + "mean_token_accuracy": 0.7809058628976345, + "num_tokens": 612893.0, + "step": 190 + }, + { + "entropy": 0.8438695728778839, + "epoch": 0.045714285714285714, + "grad_norm": 8.625, + "learning_rate": 1.8256880733944955e-05, + "loss": 1.161054801940918, + "mean_token_accuracy": 0.810321356356144, + "num_tokens": 645362.0, + "step": 200 + }, + { + "entropy": 0.8822290134616196, + "epoch": 0.048, + "grad_norm": 10.5, + "learning_rate": 1.9174311926605506e-05, + "loss": 1.1970745086669923, + "mean_token_accuracy": 0.7906114481389522, + "num_tokens": 678993.0, + "step": 210 + }, + { + "entropy": 0.9281622164882719, + "epoch": 0.05028571428571429, + "grad_norm": 9.25, + "learning_rate": 1.9995188838104405e-05, + "loss": 1.2374305725097656, + "mean_token_accuracy": 0.7800474755465985, + "num_tokens": 714146.0, + "step": 220 + }, + { + "entropy": 1.06684466060251, + "epoch": 0.052571428571428575, + "grad_norm": 8.25, + "learning_rate": 1.9947077219148424e-05, + "loss": 1.3902817726135255, + "mean_token_accuracy": 0.7615161284804344, + "num_tokens": 747373.0, + "step": 230 + }, + { + "entropy": 0.9200185919180512, + "epoch": 0.054857142857142854, + "grad_norm": 11.5, + "learning_rate": 1.9898965600192447e-05, + "loss": 1.201821517944336, + "mean_token_accuracy": 0.7907239098101855, + "num_tokens": 780155.0, + "step": 240 + }, + { + "entropy": 0.7674560694023966, + "epoch": 0.05714285714285714, + "grad_norm": 10.1875, + "learning_rate": 1.985085398123647e-05, + "loss": 1.063106632232666, + "mean_token_accuracy": 0.8229942351579667, + "num_tokens": 814128.0, + "step": 250 + }, + { + "entropy": 1.0007737869396807, + "epoch": 0.05942857142857143, + "grad_norm": 12.125, + "learning_rate": 1.9802742362280492e-05, + "loss": 1.2750150680541992, + "mean_token_accuracy": 0.7728326875716448, + "num_tokens": 845910.0, + "step": 260 + }, + { + "entropy": 0.9125091641210019, + "epoch": 0.061714285714285715, + "grad_norm": 10.125, + "learning_rate": 1.9754630743324514e-05, + "loss": 1.2319644927978515, + "mean_token_accuracy": 0.7889250412583351, + "num_tokens": 875424.0, + "step": 270 + }, + { + "entropy": 0.783863732777536, + "epoch": 0.064, + "grad_norm": 9.625, + "learning_rate": 1.9706519124368537e-05, + "loss": 1.1022482872009278, + "mean_token_accuracy": 0.8143158428370952, + "num_tokens": 912199.0, + "step": 280 + }, + { + "entropy": 0.7469094243831933, + "epoch": 0.06628571428571428, + "grad_norm": 9.0, + "learning_rate": 1.965840750541256e-05, + "loss": 1.012520694732666, + "mean_token_accuracy": 0.8163027696311473, + "num_tokens": 945467.0, + "step": 290 + }, + { + "entropy": 0.8857646442949771, + "epoch": 0.06857142857142857, + "grad_norm": 8.0625, + "learning_rate": 1.9610295886456582e-05, + "loss": 1.158743667602539, + "mean_token_accuracy": 0.7900811523199082, + "num_tokens": 975590.0, + "step": 300 + }, + { + "entropy": 0.909662488102913, + "epoch": 0.07085714285714285, + "grad_norm": 7.75, + "learning_rate": 1.9562184267500604e-05, + "loss": 1.2604731559753417, + "mean_token_accuracy": 0.7842940967530012, + "num_tokens": 1006723.0, + "step": 310 + }, + { + "entropy": 0.7755309957079589, + "epoch": 0.07314285714285715, + "grad_norm": 7.96875, + "learning_rate": 1.9514072648544624e-05, + "loss": 1.0989351272583008, + "mean_token_accuracy": 0.8179159574210644, + "num_tokens": 1039345.0, + "step": 320 + }, + { + "entropy": 0.6633960926905275, + "epoch": 0.07542857142857143, + "grad_norm": 8.75, + "learning_rate": 1.9465961029588646e-05, + "loss": 0.9405019760131836, + "mean_token_accuracy": 0.8347477428615093, + "num_tokens": 1072231.0, + "step": 330 + }, + { + "entropy": 0.7519668426364661, + "epoch": 0.07771428571428571, + "grad_norm": 11.125, + "learning_rate": 1.941784941063267e-05, + "loss": 1.120675754547119, + "mean_token_accuracy": 0.8173437312245369, + "num_tokens": 1107977.0, + "step": 340 + }, + { + "entropy": 0.8688408816233277, + "epoch": 0.08, + "grad_norm": 7.53125, + "learning_rate": 1.936973779167669e-05, + "loss": 1.137878131866455, + "mean_token_accuracy": 0.8090878508985042, + "num_tokens": 1137182.0, + "step": 350 + }, + { + "entropy": 0.7986814582720398, + "epoch": 0.08228571428571428, + "grad_norm": 8.0625, + "learning_rate": 1.9321626172720714e-05, + "loss": 1.0986030578613282, + "mean_token_accuracy": 0.8076537061482668, + "num_tokens": 1171297.0, + "step": 360 + }, + { + "entropy": 0.7295990631915629, + "epoch": 0.08457142857142858, + "grad_norm": 9.1875, + "learning_rate": 1.9273514553764736e-05, + "loss": 1.0371208190917969, + "mean_token_accuracy": 0.8235810197889805, + "num_tokens": 1203878.0, + "step": 370 + }, + { + "entropy": 0.7101914969272911, + "epoch": 0.08685714285714285, + "grad_norm": 8.5625, + "learning_rate": 1.922540293480876e-05, + "loss": 0.9649307250976562, + "mean_token_accuracy": 0.8310844466090203, + "num_tokens": 1238543.0, + "step": 380 + }, + { + "entropy": 0.8718695783987641, + "epoch": 0.08914285714285715, + "grad_norm": 9.0625, + "learning_rate": 1.917729131585278e-05, + "loss": 1.1625357627868653, + "mean_token_accuracy": 0.8035985276103019, + "num_tokens": 1269739.0, + "step": 390 + }, + { + "entropy": 0.7572122471407056, + "epoch": 0.09142857142857143, + "grad_norm": 7.375, + "learning_rate": 1.9129179696896804e-05, + "loss": 1.0944371223449707, + "mean_token_accuracy": 0.8201560072600842, + "num_tokens": 1300389.0, + "step": 400 + }, + { + "entropy": 0.7699693274684251, + "epoch": 0.09371428571428571, + "grad_norm": 7.5625, + "learning_rate": 1.9081068077940826e-05, + "loss": 1.1060125350952148, + "mean_token_accuracy": 0.8187883667647838, + "num_tokens": 1332082.0, + "step": 410 + }, + { + "entropy": 0.7879621520638466, + "epoch": 0.096, + "grad_norm": 7.125, + "learning_rate": 1.9032956458984845e-05, + "loss": 1.0424601554870605, + "mean_token_accuracy": 0.8205794245004654, + "num_tokens": 1365790.0, + "step": 420 + }, + { + "entropy": 0.7572797378525138, + "epoch": 0.09828571428571428, + "grad_norm": 7.34375, + "learning_rate": 1.8984844840028868e-05, + "loss": 1.078255844116211, + "mean_token_accuracy": 0.8151584453880787, + "num_tokens": 1400056.0, + "step": 430 + }, + { + "entropy": 0.7952604771591723, + "epoch": 0.10057142857142858, + "grad_norm": 7.375, + "learning_rate": 1.893673322107289e-05, + "loss": 1.105555820465088, + "mean_token_accuracy": 0.8153948955237865, + "num_tokens": 1430264.0, + "step": 440 + }, + { + "entropy": 0.8230121753178536, + "epoch": 0.10285714285714286, + "grad_norm": 7.125, + "learning_rate": 1.8888621602116913e-05, + "loss": 1.0710087776184083, + "mean_token_accuracy": 0.8046157158911228, + "num_tokens": 1461481.0, + "step": 450 + }, + { + "entropy": 0.9548539834097027, + "epoch": 0.10514285714285715, + "grad_norm": 7.09375, + "learning_rate": 1.8840509983160935e-05, + "loss": 1.240159511566162, + "mean_token_accuracy": 0.7864300020039081, + "num_tokens": 1493711.0, + "step": 460 + }, + { + "entropy": 1.0574401247315108, + "epoch": 0.10742857142857143, + "grad_norm": 7.90625, + "learning_rate": 1.8792398364204958e-05, + "loss": 1.345158290863037, + "mean_token_accuracy": 0.7603430174291134, + "num_tokens": 1526418.0, + "step": 470 + }, + { + "entropy": 0.8240675249136984, + "epoch": 0.10971428571428571, + "grad_norm": 7.65625, + "learning_rate": 1.874428674524898e-05, + "loss": 1.0730896949768067, + "mean_token_accuracy": 0.8100373946130276, + "num_tokens": 1558046.0, + "step": 480 + }, + { + "entropy": 0.8802061447873711, + "epoch": 0.112, + "grad_norm": 7.84375, + "learning_rate": 1.8696175126293003e-05, + "loss": 1.145187759399414, + "mean_token_accuracy": 0.7953318640589714, + "num_tokens": 1586212.0, + "step": 490 + }, + { + "entropy": 0.7748386798426509, + "epoch": 0.11428571428571428, + "grad_norm": 7.625, + "learning_rate": 1.8648063507337025e-05, + "loss": 0.9954061508178711, + "mean_token_accuracy": 0.8151729434728623, + "num_tokens": 1622364.0, + "step": 500 + }, + { + "entropy": 0.7383366953581572, + "epoch": 0.11657142857142858, + "grad_norm": 8.75, + "learning_rate": 1.8599951888381044e-05, + "loss": 0.9818957328796387, + "mean_token_accuracy": 0.8187249258160592, + "num_tokens": 1655806.0, + "step": 510 + }, + { + "entropy": 0.7778955462388695, + "epoch": 0.11885714285714286, + "grad_norm": 7.6875, + "learning_rate": 1.8551840269425067e-05, + "loss": 1.070115566253662, + "mean_token_accuracy": 0.8072714075446129, + "num_tokens": 1687198.0, + "step": 520 + }, + { + "entropy": 0.806643515639007, + "epoch": 0.12114285714285715, + "grad_norm": 8.5, + "learning_rate": 1.850372865046909e-05, + "loss": 1.1104951858520509, + "mean_token_accuracy": 0.8077441163361072, + "num_tokens": 1718591.0, + "step": 530 + }, + { + "entropy": 0.8236157631501555, + "epoch": 0.12342857142857143, + "grad_norm": 7.84375, + "learning_rate": 1.8455617031513112e-05, + "loss": 1.0954437255859375, + "mean_token_accuracy": 0.8092149019241333, + "num_tokens": 1750163.0, + "step": 540 + }, + { + "entropy": 0.9514989891089499, + "epoch": 0.12571428571428572, + "grad_norm": 8.0, + "learning_rate": 1.8407505412557134e-05, + "loss": 1.2232957839965821, + "mean_token_accuracy": 0.7788458302617073, + "num_tokens": 1783654.0, + "step": 550 + }, + { + "entropy": 0.8126472384203225, + "epoch": 0.128, + "grad_norm": 7.25, + "learning_rate": 1.8359393793601157e-05, + "loss": 1.1193376541137696, + "mean_token_accuracy": 0.8038583904504776, + "num_tokens": 1818298.0, + "step": 560 + }, + { + "entropy": 0.7461063047870994, + "epoch": 0.13028571428571428, + "grad_norm": 10.125, + "learning_rate": 1.8311282174645176e-05, + "loss": 1.0971127510070802, + "mean_token_accuracy": 0.8208389431238174, + "num_tokens": 1852353.0, + "step": 570 + }, + { + "entropy": 0.8738343502394855, + "epoch": 0.13257142857142856, + "grad_norm": 7.78125, + "learning_rate": 1.8263170555689202e-05, + "loss": 1.1203450202941894, + "mean_token_accuracy": 0.7919381201267243, + "num_tokens": 1883233.0, + "step": 580 + }, + { + "entropy": 0.9372453663498164, + "epoch": 0.13485714285714287, + "grad_norm": 7.09375, + "learning_rate": 1.8215058936733224e-05, + "loss": 1.2445517539978028, + "mean_token_accuracy": 0.7886677496135235, + "num_tokens": 1914856.0, + "step": 590 + }, + { + "entropy": 0.922083193808794, + "epoch": 0.13714285714285715, + "grad_norm": 7.28125, + "learning_rate": 1.8166947317777243e-05, + "loss": 1.2312170028686524, + "mean_token_accuracy": 0.7963043257594109, + "num_tokens": 1944932.0, + "step": 600 + }, + { + "entropy": 0.885446681920439, + "epoch": 0.13942857142857143, + "grad_norm": 7.75, + "learning_rate": 1.8118835698821266e-05, + "loss": 1.1832991600036622, + "mean_token_accuracy": 0.7979222141206265, + "num_tokens": 1977483.0, + "step": 610 + }, + { + "entropy": 0.7927633517421782, + "epoch": 0.1417142857142857, + "grad_norm": 10.875, + "learning_rate": 1.807072407986529e-05, + "loss": 1.072523021697998, + "mean_token_accuracy": 0.8096244156360626, + "num_tokens": 2012646.0, + "step": 620 + }, + { + "entropy": 0.8450389008037746, + "epoch": 0.144, + "grad_norm": 8.5625, + "learning_rate": 1.802261246090931e-05, + "loss": 1.1467965126037598, + "mean_token_accuracy": 0.8004457049071789, + "num_tokens": 2044969.0, + "step": 630 + }, + { + "entropy": 0.9473820111714304, + "epoch": 0.1462857142857143, + "grad_norm": 7.8125, + "learning_rate": 1.7974500841953333e-05, + "loss": 1.2415997505187988, + "mean_token_accuracy": 0.7821900390088559, + "num_tokens": 2077063.0, + "step": 640 + }, + { + "entropy": 0.8002779926173389, + "epoch": 0.14857142857142858, + "grad_norm": 7.1875, + "learning_rate": 1.7926389222997356e-05, + "loss": 1.08231201171875, + "mean_token_accuracy": 0.8153595373034477, + "num_tokens": 2108559.0, + "step": 650 + }, + { + "entropy": 0.7409520236775279, + "epoch": 0.15085714285714286, + "grad_norm": 6.625, + "learning_rate": 1.7878277604041375e-05, + "loss": 1.0647228240966797, + "mean_token_accuracy": 0.8233258232474328, + "num_tokens": 2143996.0, + "step": 660 + }, + { + "entropy": 0.6605528288520872, + "epoch": 0.15314285714285714, + "grad_norm": 7.0, + "learning_rate": 1.7830165985085398e-05, + "loss": 0.8986818313598632, + "mean_token_accuracy": 0.8348655626177788, + "num_tokens": 2177219.0, + "step": 670 + }, + { + "entropy": 0.8855553255416453, + "epoch": 0.15542857142857142, + "grad_norm": 7.25, + "learning_rate": 1.7782054366129423e-05, + "loss": 1.2236814498901367, + "mean_token_accuracy": 0.7893378108739852, + "num_tokens": 2209056.0, + "step": 680 + }, + { + "entropy": 0.7778299384750426, + "epoch": 0.15771428571428572, + "grad_norm": 8.75, + "learning_rate": 1.7733942747173446e-05, + "loss": 1.1849853515625, + "mean_token_accuracy": 0.8148446299135685, + "num_tokens": 2241537.0, + "step": 690 + }, + { + "entropy": 0.9553269637282937, + "epoch": 0.16, + "grad_norm": 7.9375, + "learning_rate": 1.7685831128217465e-05, + "loss": 1.185690212249756, + "mean_token_accuracy": 0.7805978901684284, + "num_tokens": 2273475.0, + "step": 700 + }, + { + "entropy": 0.921497387625277, + "epoch": 0.16228571428571428, + "grad_norm": 7.65625, + "learning_rate": 1.7637719509261488e-05, + "loss": 1.1969280242919922, + "mean_token_accuracy": 0.7880251817405224, + "num_tokens": 2303700.0, + "step": 710 + }, + { + "entropy": 0.9623362662270665, + "epoch": 0.16457142857142856, + "grad_norm": 7.84375, + "learning_rate": 1.758960789030551e-05, + "loss": 1.2279460906982422, + "mean_token_accuracy": 0.7762100405991077, + "num_tokens": 2335528.0, + "step": 720 + }, + { + "entropy": 0.7501861874945461, + "epoch": 0.16685714285714287, + "grad_norm": 6.53125, + "learning_rate": 1.7541496271349533e-05, + "loss": 1.092702579498291, + "mean_token_accuracy": 0.8136215195059776, + "num_tokens": 2369322.0, + "step": 730 + }, + { + "entropy": 0.741880859900266, + "epoch": 0.16914285714285715, + "grad_norm": 9.1875, + "learning_rate": 1.7493384652393555e-05, + "loss": 1.0542023658752442, + "mean_token_accuracy": 0.8248877495527267, + "num_tokens": 2402789.0, + "step": 740 + }, + { + "entropy": 0.7251557628624141, + "epoch": 0.17142857142857143, + "grad_norm": 7.21875, + "learning_rate": 1.7445273033437578e-05, + "loss": 0.9859682083129883, + "mean_token_accuracy": 0.8190685380250216, + "num_tokens": 2435813.0, + "step": 750 + }, + { + "entropy": 0.8274217823520302, + "epoch": 0.1737142857142857, + "grad_norm": 7.8125, + "learning_rate": 1.7397161414481597e-05, + "loss": 1.1144951820373534, + "mean_token_accuracy": 0.8048250667750836, + "num_tokens": 2469061.0, + "step": 760 + }, + { + "entropy": 0.9110285563394427, + "epoch": 0.176, + "grad_norm": 7.9375, + "learning_rate": 1.7349049795525623e-05, + "loss": 1.2487107276916505, + "mean_token_accuracy": 0.7836838386952877, + "num_tokens": 2499540.0, + "step": 770 + }, + { + "entropy": 0.7952723279595375, + "epoch": 0.1782857142857143, + "grad_norm": 8.125, + "learning_rate": 1.7300938176569645e-05, + "loss": 1.1460785865783691, + "mean_token_accuracy": 0.8108866758644581, + "num_tokens": 2528841.0, + "step": 780 + }, + { + "entropy": 0.8681487245485187, + "epoch": 0.18057142857142858, + "grad_norm": 7.0625, + "learning_rate": 1.7252826557613664e-05, + "loss": 1.088980770111084, + "mean_token_accuracy": 0.7965113393962383, + "num_tokens": 2560120.0, + "step": 790 + }, + { + "entropy": 0.7640921414829791, + "epoch": 0.18285714285714286, + "grad_norm": 9.75, + "learning_rate": 1.7204714938657687e-05, + "loss": 1.1072608947753906, + "mean_token_accuracy": 0.81538320556283, + "num_tokens": 2592460.0, + "step": 800 + }, + { + "entropy": 0.8132671658881009, + "epoch": 0.18514285714285714, + "grad_norm": 7.1875, + "learning_rate": 1.715660331970171e-05, + "loss": 1.1375692367553711, + "mean_token_accuracy": 0.8027771405875683, + "num_tokens": 2628458.0, + "step": 810 + }, + { + "entropy": 0.71722816741094, + "epoch": 0.18742857142857142, + "grad_norm": 6.625, + "learning_rate": 1.7108491700745732e-05, + "loss": 0.9269392013549804, + "mean_token_accuracy": 0.8260728389024734, + "num_tokens": 2663216.0, + "step": 820 + }, + { + "entropy": 0.7395319850184023, + "epoch": 0.18971428571428572, + "grad_norm": 8.0625, + "learning_rate": 1.7060380081789754e-05, + "loss": 0.9782637596130371, + "mean_token_accuracy": 0.8158168852329254, + "num_tokens": 2695026.0, + "step": 830 + }, + { + "entropy": 0.7296101478859782, + "epoch": 0.192, + "grad_norm": 7.21875, + "learning_rate": 1.7012268462833777e-05, + "loss": 1.0164703369140624, + "mean_token_accuracy": 0.8283582173287869, + "num_tokens": 2727324.0, + "step": 840 + }, + { + "entropy": 0.8305218723602593, + "epoch": 0.19428571428571428, + "grad_norm": 8.0, + "learning_rate": 1.6964156843877796e-05, + "loss": 1.1646985054016112, + "mean_token_accuracy": 0.8054761447012424, + "num_tokens": 2759422.0, + "step": 850 + }, + { + "entropy": 0.8524466481991112, + "epoch": 0.19657142857142856, + "grad_norm": 8.25, + "learning_rate": 1.691604522492182e-05, + "loss": 1.1327482223510743, + "mean_token_accuracy": 0.8032685436308384, + "num_tokens": 2792518.0, + "step": 860 + }, + { + "entropy": 0.6850544813089072, + "epoch": 0.19885714285714284, + "grad_norm": 8.1875, + "learning_rate": 1.6867933605965844e-05, + "loss": 0.9866686820983886, + "mean_token_accuracy": 0.8292375847697258, + "num_tokens": 2826870.0, + "step": 870 + }, + { + "entropy": 0.7483350836671889, + "epoch": 0.20114285714285715, + "grad_norm": 9.3125, + "learning_rate": 1.6819821987009863e-05, + "loss": 1.0796706199645996, + "mean_token_accuracy": 0.8100242014974356, + "num_tokens": 2859320.0, + "step": 880 + }, + { + "entropy": 0.761015557963401, + "epoch": 0.20342857142857143, + "grad_norm": 7.65625, + "learning_rate": 1.6771710368053886e-05, + "loss": 1.0168207168579102, + "mean_token_accuracy": 0.8171908967196941, + "num_tokens": 2893696.0, + "step": 890 + }, + { + "entropy": 0.7136277809739113, + "epoch": 0.2057142857142857, + "grad_norm": 6.75, + "learning_rate": 1.672359874909791e-05, + "loss": 0.9646918296813964, + "mean_token_accuracy": 0.8310928396880627, + "num_tokens": 2925196.0, + "step": 900 + }, + { + "entropy": 0.8552220237441361, + "epoch": 0.208, + "grad_norm": 7.28125, + "learning_rate": 1.667548713014193e-05, + "loss": 1.0989733695983888, + "mean_token_accuracy": 0.8012546695768833, + "num_tokens": 2957056.0, + "step": 910 + }, + { + "entropy": 0.8601699252612889, + "epoch": 0.2102857142857143, + "grad_norm": 7.09375, + "learning_rate": 1.6627375511185953e-05, + "loss": 1.1599449157714843, + "mean_token_accuracy": 0.7934970580041408, + "num_tokens": 2988772.0, + "step": 920 + }, + { + "entropy": 0.7465516209602356, + "epoch": 0.21257142857142858, + "grad_norm": 6.0625, + "learning_rate": 1.6579263892229976e-05, + "loss": 1.014716911315918, + "mean_token_accuracy": 0.8273697007447481, + "num_tokens": 3024521.0, + "step": 930 + }, + { + "entropy": 0.7951956107281148, + "epoch": 0.21485714285714286, + "grad_norm": 7.0, + "learning_rate": 1.6531152273273995e-05, + "loss": 1.023974895477295, + "mean_token_accuracy": 0.81702711135149, + "num_tokens": 3055362.0, + "step": 940 + }, + { + "entropy": 0.8373465910553932, + "epoch": 0.21714285714285714, + "grad_norm": 8.5625, + "learning_rate": 1.6483040654318018e-05, + "loss": 1.149094009399414, + "mean_token_accuracy": 0.7994145810604095, + "num_tokens": 3085932.0, + "step": 950 + }, + { + "entropy": 0.7051204042509198, + "epoch": 0.21942857142857142, + "grad_norm": 6.375, + "learning_rate": 1.643492903536204e-05, + "loss": 0.9659648895263672, + "mean_token_accuracy": 0.8331427097320556, + "num_tokens": 3114791.0, + "step": 960 + }, + { + "entropy": 0.7798629926517606, + "epoch": 0.22171428571428572, + "grad_norm": 7.1875, + "learning_rate": 1.6386817416406066e-05, + "loss": 1.008477783203125, + "mean_token_accuracy": 0.8194496557116508, + "num_tokens": 3149473.0, + "step": 970 + }, + { + "entropy": 0.6989101554267109, + "epoch": 0.224, + "grad_norm": 8.125, + "learning_rate": 1.6338705797450085e-05, + "loss": 0.9564552307128906, + "mean_token_accuracy": 0.8339050948619843, + "num_tokens": 3182913.0, + "step": 980 + }, + { + "entropy": 0.8009612016379833, + "epoch": 0.22628571428571428, + "grad_norm": 7.96875, + "learning_rate": 1.6290594178494108e-05, + "loss": 1.082752799987793, + "mean_token_accuracy": 0.8148751087486744, + "num_tokens": 3214329.0, + "step": 990 + }, + { + "entropy": 0.7472866786643863, + "epoch": 0.22857142857142856, + "grad_norm": 7.03125, + "learning_rate": 1.624248255953813e-05, + "loss": 0.9995194435119629, + "mean_token_accuracy": 0.8205188862979412, + "num_tokens": 3246465.0, + "step": 1000 + }, + { + "entropy": 0.8845632201060653, + "epoch": 0.23085714285714284, + "grad_norm": 7.28125, + "learning_rate": 1.6194370940582153e-05, + "loss": 1.1626070976257323, + "mean_token_accuracy": 0.7972030624747276, + "num_tokens": 3280259.0, + "step": 1010 + }, + { + "entropy": 0.7615264546126127, + "epoch": 0.23314285714285715, + "grad_norm": 6.59375, + "learning_rate": 1.6146259321626175e-05, + "loss": 1.0563675880432128, + "mean_token_accuracy": 0.8191345028579236, + "num_tokens": 3308134.0, + "step": 1020 + }, + { + "entropy": 0.7456497238948941, + "epoch": 0.23542857142857143, + "grad_norm": 9.125, + "learning_rate": 1.6098147702670198e-05, + "loss": 0.9785372734069824, + "mean_token_accuracy": 0.8168387658894062, + "num_tokens": 3339804.0, + "step": 1030 + }, + { + "entropy": 0.9259420620277524, + "epoch": 0.2377142857142857, + "grad_norm": 7.1875, + "learning_rate": 1.6050036083714217e-05, + "loss": 1.2626006126403808, + "mean_token_accuracy": 0.7905944272875786, + "num_tokens": 3374600.0, + "step": 1040 + }, + { + "entropy": 0.7013174806721508, + "epoch": 0.24, + "grad_norm": 7.28125, + "learning_rate": 1.600192446475824e-05, + "loss": 0.8472931861877442, + "mean_token_accuracy": 0.834729814529419, + "num_tokens": 3406084.0, + "step": 1050 + }, + { + "entropy": 0.8099798344075679, + "epoch": 0.2422857142857143, + "grad_norm": 8.375, + "learning_rate": 1.5953812845802265e-05, + "loss": 1.1302724838256837, + "mean_token_accuracy": 0.8053847432136536, + "num_tokens": 3436595.0, + "step": 1060 + }, + { + "entropy": 0.6409325916320086, + "epoch": 0.24457142857142858, + "grad_norm": 8.1875, + "learning_rate": 1.5905701226846284e-05, + "loss": 0.9276237487792969, + "mean_token_accuracy": 0.8369442120194435, + "num_tokens": 3468462.0, + "step": 1070 + }, + { + "entropy": 0.6651454066857696, + "epoch": 0.24685714285714286, + "grad_norm": 8.75, + "learning_rate": 1.5857589607890307e-05, + "loss": 0.9005106925964356, + "mean_token_accuracy": 0.8366246894001961, + "num_tokens": 3502363.0, + "step": 1080 + }, + { + "entropy": 0.6705873743630946, + "epoch": 0.24914285714285714, + "grad_norm": 6.96875, + "learning_rate": 1.580947798893433e-05, + "loss": 0.9289634704589844, + "mean_token_accuracy": 0.8348163716495037, + "num_tokens": 3532774.0, + "step": 1090 + }, + { + "entropy": 0.8024829808622599, + "epoch": 0.25142857142857145, + "grad_norm": 8.25, + "learning_rate": 1.5761366369978352e-05, + "loss": 1.1945523262023925, + "mean_token_accuracy": 0.8085858777165413, + "num_tokens": 3565974.0, + "step": 1100 + }, + { + "entropy": 0.7451043974608182, + "epoch": 0.2537142857142857, + "grad_norm": 7.0625, + "learning_rate": 1.5713254751022374e-05, + "loss": 1.0138811111450194, + "mean_token_accuracy": 0.8233392249792815, + "num_tokens": 3599243.0, + "step": 1110 + }, + { + "entropy": 0.8042388122528792, + "epoch": 0.256, + "grad_norm": 6.71875, + "learning_rate": 1.5665143132066397e-05, + "loss": 1.0674107551574707, + "mean_token_accuracy": 0.8107536815106868, + "num_tokens": 3631217.0, + "step": 1120 + }, + { + "entropy": 0.6658579808659851, + "epoch": 0.2582857142857143, + "grad_norm": 7.9375, + "learning_rate": 1.5617031513110416e-05, + "loss": 0.8685308456420898, + "mean_token_accuracy": 0.8395798467099667, + "num_tokens": 3663654.0, + "step": 1130 + }, + { + "entropy": 0.6915052223019302, + "epoch": 0.26057142857142856, + "grad_norm": 6.75, + "learning_rate": 1.556891989415444e-05, + "loss": 0.9267234802246094, + "mean_token_accuracy": 0.8305907912552357, + "num_tokens": 3695008.0, + "step": 1140 + }, + { + "entropy": 0.7746626659296453, + "epoch": 0.26285714285714284, + "grad_norm": 7.46875, + "learning_rate": 1.552080827519846e-05, + "loss": 1.1303828239440918, + "mean_token_accuracy": 0.8152431160211563, + "num_tokens": 3725769.0, + "step": 1150 + }, + { + "entropy": 0.815077618137002, + "epoch": 0.2651428571428571, + "grad_norm": 8.375, + "learning_rate": 1.5472696656242483e-05, + "loss": 1.1592507362365723, + "mean_token_accuracy": 0.8096996866166591, + "num_tokens": 3759222.0, + "step": 1160 + }, + { + "entropy": 0.9105647809803485, + "epoch": 0.2674285714285714, + "grad_norm": 7.84375, + "learning_rate": 1.5424585037286506e-05, + "loss": 1.2050976753234863, + "mean_token_accuracy": 0.7931383229792118, + "num_tokens": 3789235.0, + "step": 1170 + }, + { + "entropy": 0.8901564389467239, + "epoch": 0.26971428571428574, + "grad_norm": 7.40625, + "learning_rate": 1.537647341833053e-05, + "loss": 1.1315418243408204, + "mean_token_accuracy": 0.7916867211461067, + "num_tokens": 3822530.0, + "step": 1180 + }, + { + "entropy": 0.8138524909503758, + "epoch": 0.272, + "grad_norm": 7.46875, + "learning_rate": 1.532836179937455e-05, + "loss": 1.0824806213378906, + "mean_token_accuracy": 0.807539875805378, + "num_tokens": 3857294.0, + "step": 1190 + }, + { + "entropy": 0.8327909021638333, + "epoch": 0.2742857142857143, + "grad_norm": 7.40625, + "learning_rate": 1.5280250180418573e-05, + "loss": 1.0807191848754882, + "mean_token_accuracy": 0.8046631902456284, + "num_tokens": 3887638.0, + "step": 1200 + }, + { + "entropy": 0.6820507633499802, + "epoch": 0.2765714285714286, + "grad_norm": 7.25, + "learning_rate": 1.5232138561462594e-05, + "loss": 0.9238566398620606, + "mean_token_accuracy": 0.8344717890024185, + "num_tokens": 3921320.0, + "step": 1210 + }, + { + "entropy": 0.6690340504981578, + "epoch": 0.27885714285714286, + "grad_norm": 8.625, + "learning_rate": 1.5184026942506615e-05, + "loss": 0.9770146369934082, + "mean_token_accuracy": 0.83714384958148, + "num_tokens": 3952659.0, + "step": 1220 + }, + { + "entropy": 0.7821415845304728, + "epoch": 0.28114285714285714, + "grad_norm": 8.9375, + "learning_rate": 1.513591532355064e-05, + "loss": 1.1273323059082032, + "mean_token_accuracy": 0.8086842469871044, + "num_tokens": 3985673.0, + "step": 1230 + }, + { + "entropy": 0.7746237487532198, + "epoch": 0.2834285714285714, + "grad_norm": 6.96875, + "learning_rate": 1.5087803704594662e-05, + "loss": 0.9969350814819335, + "mean_token_accuracy": 0.8176074028015137, + "num_tokens": 4014425.0, + "step": 1240 + }, + { + "entropy": 0.7822927181608975, + "epoch": 0.2857142857142857, + "grad_norm": 7.90625, + "learning_rate": 1.5039692085638682e-05, + "loss": 1.0581744194030762, + "mean_token_accuracy": 0.8114182919263839, + "num_tokens": 4044735.0, + "step": 1250 + }, + { + "entropy": 0.8262449711561203, + "epoch": 0.288, + "grad_norm": 8.125, + "learning_rate": 1.4991580466682705e-05, + "loss": 1.1058235168457031, + "mean_token_accuracy": 0.8093136258423328, + "num_tokens": 4077366.0, + "step": 1260 + }, + { + "entropy": 0.7183008762076497, + "epoch": 0.29028571428571426, + "grad_norm": 6.75, + "learning_rate": 1.4943468847726727e-05, + "loss": 1.0642513275146483, + "mean_token_accuracy": 0.8284725159406662, + "num_tokens": 4112231.0, + "step": 1270 + }, + { + "entropy": 0.7519657079130411, + "epoch": 0.2925714285714286, + "grad_norm": 7.65625, + "learning_rate": 1.489535722877075e-05, + "loss": 1.0380656242370605, + "mean_token_accuracy": 0.8182306826114655, + "num_tokens": 4141706.0, + "step": 1280 + }, + { + "entropy": 0.8959639485925436, + "epoch": 0.2948571428571429, + "grad_norm": 8.8125, + "learning_rate": 1.484724560981477e-05, + "loss": 1.1817408561706544, + "mean_token_accuracy": 0.7842808924615383, + "num_tokens": 4174772.0, + "step": 1290 + }, + { + "entropy": 0.7940410540439189, + "epoch": 0.29714285714285715, + "grad_norm": 7.9375, + "learning_rate": 1.4799133990858793e-05, + "loss": 1.0543545722961425, + "mean_token_accuracy": 0.8153316840529442, + "num_tokens": 4207107.0, + "step": 1300 + }, + { + "entropy": 0.7827737585641443, + "epoch": 0.29942857142857143, + "grad_norm": 11.0, + "learning_rate": 1.4751022371902818e-05, + "loss": 1.1160078048706055, + "mean_token_accuracy": 0.812502384185791, + "num_tokens": 4242521.0, + "step": 1310 + }, + { + "entropy": 0.735058065969497, + "epoch": 0.3017142857142857, + "grad_norm": 7.5, + "learning_rate": 1.4702910752946837e-05, + "loss": 1.0581280708312988, + "mean_token_accuracy": 0.8288455225527287, + "num_tokens": 4276982.0, + "step": 1320 + }, + { + "entropy": 0.7287354637868703, + "epoch": 0.304, + "grad_norm": 6.9375, + "learning_rate": 1.465479913399086e-05, + "loss": 0.9759371757507325, + "mean_token_accuracy": 0.8249299876391888, + "num_tokens": 4306046.0, + "step": 1330 + }, + { + "entropy": 0.7787643402814866, + "epoch": 0.3062857142857143, + "grad_norm": 7.5, + "learning_rate": 1.4606687515034883e-05, + "loss": 1.0230366706848144, + "mean_token_accuracy": 0.816937967389822, + "num_tokens": 4338652.0, + "step": 1340 + }, + { + "entropy": 0.8934064561501145, + "epoch": 0.30857142857142855, + "grad_norm": 5.875, + "learning_rate": 1.4558575896078904e-05, + "loss": 1.086491012573242, + "mean_token_accuracy": 0.7942407630383969, + "num_tokens": 4370871.0, + "step": 1350 + }, + { + "entropy": 0.833009172603488, + "epoch": 0.31085714285714283, + "grad_norm": 7.625, + "learning_rate": 1.4510464277122927e-05, + "loss": 1.1905370712280274, + "mean_token_accuracy": 0.8013229362666607, + "num_tokens": 4402989.0, + "step": 1360 + }, + { + "entropy": 0.7927345955744386, + "epoch": 0.31314285714285717, + "grad_norm": 6.875, + "learning_rate": 1.4462352658166949e-05, + "loss": 1.1041228294372558, + "mean_token_accuracy": 0.8128574416041374, + "num_tokens": 4434506.0, + "step": 1370 + }, + { + "entropy": 0.7229848534800112, + "epoch": 0.31542857142857145, + "grad_norm": 7.53125, + "learning_rate": 1.441424103921097e-05, + "loss": 1.0535061836242676, + "mean_token_accuracy": 0.8302266910672188, + "num_tokens": 4467742.0, + "step": 1380 + }, + { + "entropy": 0.6736922577954829, + "epoch": 0.3177142857142857, + "grad_norm": 8.3125, + "learning_rate": 1.4366129420254992e-05, + "loss": 0.9416308403015137, + "mean_token_accuracy": 0.8366274513304234, + "num_tokens": 4498266.0, + "step": 1390 + }, + { + "entropy": 0.7866813028231263, + "epoch": 0.32, + "grad_norm": 7.0, + "learning_rate": 1.4318017801299015e-05, + "loss": 1.0644832611083985, + "mean_token_accuracy": 0.8135197319090366, + "num_tokens": 4526424.0, + "step": 1400 + }, + { + "entropy": 0.755329312197864, + "epoch": 0.3222857142857143, + "grad_norm": 7.25, + "learning_rate": 1.4269906182343036e-05, + "loss": 1.0085858345031737, + "mean_token_accuracy": 0.8171801075339318, + "num_tokens": 4556781.0, + "step": 1410 + }, + { + "entropy": 0.7350449176505208, + "epoch": 0.32457142857142857, + "grad_norm": 8.0, + "learning_rate": 1.4221794563387058e-05, + "loss": 1.0485240936279296, + "mean_token_accuracy": 0.8203142821788788, + "num_tokens": 4586938.0, + "step": 1420 + }, + { + "entropy": 0.7315349031239748, + "epoch": 0.32685714285714285, + "grad_norm": 6.4375, + "learning_rate": 1.4173682944431082e-05, + "loss": 0.9750032424926758, + "mean_token_accuracy": 0.8210594050586224, + "num_tokens": 4623204.0, + "step": 1430 + }, + { + "entropy": 0.8201613875105978, + "epoch": 0.3291428571428571, + "grad_norm": 8.375, + "learning_rate": 1.4125571325475103e-05, + "loss": 1.0923168182373046, + "mean_token_accuracy": 0.8091586381196976, + "num_tokens": 4655204.0, + "step": 1440 + }, + { + "entropy": 0.731718104891479, + "epoch": 0.3314285714285714, + "grad_norm": 7.65625, + "learning_rate": 1.4077459706519126e-05, + "loss": 0.939968490600586, + "mean_token_accuracy": 0.8259035252034664, + "num_tokens": 4684009.0, + "step": 1450 + }, + { + "entropy": 0.7582659061998129, + "epoch": 0.33371428571428574, + "grad_norm": 7.78125, + "learning_rate": 1.4029348087563148e-05, + "loss": 1.141510581970215, + "mean_token_accuracy": 0.8165373567491769, + "num_tokens": 4713643.0, + "step": 1460 + }, + { + "entropy": 0.8316841575317084, + "epoch": 0.336, + "grad_norm": 6.8125, + "learning_rate": 1.3981236468607169e-05, + "loss": 1.0487807273864747, + "mean_token_accuracy": 0.8056000970304013, + "num_tokens": 4745897.0, + "step": 1470 + }, + { + "entropy": 0.7131205608136952, + "epoch": 0.3382857142857143, + "grad_norm": 7.59375, + "learning_rate": 1.3933124849651192e-05, + "loss": 0.9551529884338379, + "mean_token_accuracy": 0.8345349170267582, + "num_tokens": 4779342.0, + "step": 1480 + }, + { + "entropy": 0.7310262706130743, + "epoch": 0.3405714285714286, + "grad_norm": 8.0, + "learning_rate": 1.3885013230695214e-05, + "loss": 1.071888256072998, + "mean_token_accuracy": 0.8165734991431236, + "num_tokens": 4810006.0, + "step": 1490 + }, + { + "entropy": 0.8448133600875736, + "epoch": 0.34285714285714286, + "grad_norm": 7.71875, + "learning_rate": 1.3836901611739235e-05, + "loss": 1.1719935417175293, + "mean_token_accuracy": 0.8007844358682632, + "num_tokens": 4841370.0, + "step": 1500 + }, + { + "entropy": 0.6549595400691033, + "epoch": 0.34514285714285714, + "grad_norm": 6.125, + "learning_rate": 1.3788789992783257e-05, + "loss": 0.9889472007751465, + "mean_token_accuracy": 0.8453674554824829, + "num_tokens": 4872052.0, + "step": 1510 + }, + { + "entropy": 0.8164440121501684, + "epoch": 0.3474285714285714, + "grad_norm": 7.5625, + "learning_rate": 1.3740678373827282e-05, + "loss": 1.0972503662109374, + "mean_token_accuracy": 0.810026689618826, + "num_tokens": 4908227.0, + "step": 1520 + }, + { + "entropy": 0.7791354645043611, + "epoch": 0.3497142857142857, + "grad_norm": 7.8125, + "learning_rate": 1.36925667548713e-05, + "loss": 1.0852085113525392, + "mean_token_accuracy": 0.8248721182346344, + "num_tokens": 4940131.0, + "step": 1530 + }, + { + "entropy": 0.8590768910944462, + "epoch": 0.352, + "grad_norm": 7.9375, + "learning_rate": 1.3644455135915325e-05, + "loss": 1.1454149246215821, + "mean_token_accuracy": 0.8017402648925781, + "num_tokens": 4971176.0, + "step": 1540 + }, + { + "entropy": 0.6812219545245171, + "epoch": 0.35428571428571426, + "grad_norm": 7.0, + "learning_rate": 1.3596343516959347e-05, + "loss": 0.919462776184082, + "mean_token_accuracy": 0.8335798256099224, + "num_tokens": 5002886.0, + "step": 1550 + }, + { + "entropy": 0.6795941894873977, + "epoch": 0.3565714285714286, + "grad_norm": 6.8125, + "learning_rate": 1.354823189800337e-05, + "loss": 0.9362202644348144, + "mean_token_accuracy": 0.8309865556657314, + "num_tokens": 5034604.0, + "step": 1560 + }, + { + "entropy": 0.8111153118312359, + "epoch": 0.3588571428571429, + "grad_norm": 7.5625, + "learning_rate": 1.350012027904739e-05, + "loss": 1.1012299537658692, + "mean_token_accuracy": 0.8071676261723042, + "num_tokens": 5067068.0, + "step": 1570 + }, + { + "entropy": 0.8018769213929773, + "epoch": 0.36114285714285715, + "grad_norm": 7.625, + "learning_rate": 1.3452008660091413e-05, + "loss": 1.0980539321899414, + "mean_token_accuracy": 0.8049051895737648, + "num_tokens": 5101840.0, + "step": 1580 + }, + { + "entropy": 0.8108965801075101, + "epoch": 0.36342857142857143, + "grad_norm": 9.4375, + "learning_rate": 1.3403897041135436e-05, + "loss": 1.097167682647705, + "mean_token_accuracy": 0.8064502902328968, + "num_tokens": 5133106.0, + "step": 1590 + }, + { + "entropy": 0.7126147777773439, + "epoch": 0.3657142857142857, + "grad_norm": 7.21875, + "learning_rate": 1.3355785422179457e-05, + "loss": 0.9641182899475098, + "mean_token_accuracy": 0.8308103702962398, + "num_tokens": 5163625.0, + "step": 1600 + }, + { + "entropy": 0.6579177615232765, + "epoch": 0.368, + "grad_norm": 7.6875, + "learning_rate": 1.3307673803223479e-05, + "loss": 0.8962146759033203, + "mean_token_accuracy": 0.8413269713521003, + "num_tokens": 5197413.0, + "step": 1610 + }, + { + "entropy": 0.8362961991690099, + "epoch": 0.3702857142857143, + "grad_norm": 7.53125, + "learning_rate": 1.3259562184267503e-05, + "loss": 1.1765104293823243, + "mean_token_accuracy": 0.8035941451787949, + "num_tokens": 5230359.0, + "step": 1620 + }, + { + "entropy": 0.8460681514814496, + "epoch": 0.37257142857142855, + "grad_norm": 7.125, + "learning_rate": 1.3211450565311524e-05, + "loss": 1.1312347412109376, + "mean_token_accuracy": 0.806734037399292, + "num_tokens": 5261631.0, + "step": 1630 + }, + { + "entropy": 0.8401033844798803, + "epoch": 0.37485714285714283, + "grad_norm": 7.6875, + "learning_rate": 1.3163338946355547e-05, + "loss": 1.1048646926879884, + "mean_token_accuracy": 0.8016112253069878, + "num_tokens": 5293666.0, + "step": 1640 + }, + { + "entropy": 0.8607065804302693, + "epoch": 0.37714285714285717, + "grad_norm": 6.25, + "learning_rate": 1.3115227327399569e-05, + "loss": 1.224336814880371, + "mean_token_accuracy": 0.7972042057663202, + "num_tokens": 5328829.0, + "step": 1650 + }, + { + "entropy": 0.6591957551427186, + "epoch": 0.37942857142857145, + "grad_norm": 8.3125, + "learning_rate": 1.306711570844359e-05, + "loss": 0.9134272575378418, + "mean_token_accuracy": 0.8408956363797188, + "num_tokens": 5359723.0, + "step": 1660 + }, + { + "entropy": 0.7585498026572168, + "epoch": 0.38171428571428573, + "grad_norm": 6.84375, + "learning_rate": 1.3019004089487612e-05, + "loss": 1.0762030601501464, + "mean_token_accuracy": 0.815737747400999, + "num_tokens": 5391030.0, + "step": 1670 + }, + { + "entropy": 0.7825943292118609, + "epoch": 0.384, + "grad_norm": 6.6875, + "learning_rate": 1.2970892470531635e-05, + "loss": 1.0779932975769042, + "mean_token_accuracy": 0.8169842332601547, + "num_tokens": 5422205.0, + "step": 1680 + }, + { + "entropy": 0.8554381128400564, + "epoch": 0.3862857142857143, + "grad_norm": 7.375, + "learning_rate": 1.2922780851575656e-05, + "loss": 1.12919340133667, + "mean_token_accuracy": 0.8013027586042881, + "num_tokens": 5452941.0, + "step": 1690 + }, + { + "entropy": 0.7908354031853377, + "epoch": 0.38857142857142857, + "grad_norm": 7.28125, + "learning_rate": 1.2874669232619678e-05, + "loss": 1.0936867713928222, + "mean_token_accuracy": 0.8094826623797416, + "num_tokens": 5486102.0, + "step": 1700 + }, + { + "entropy": 0.8072810024954379, + "epoch": 0.39085714285714285, + "grad_norm": 6.09375, + "learning_rate": 1.28265576136637e-05, + "loss": 1.0153223991394043, + "mean_token_accuracy": 0.8126743011176586, + "num_tokens": 5518871.0, + "step": 1710 + }, + { + "entropy": 0.7387428401969374, + "epoch": 0.3931428571428571, + "grad_norm": 7.34375, + "learning_rate": 1.2778445994707722e-05, + "loss": 1.021730613708496, + "mean_token_accuracy": 0.8178475465625524, + "num_tokens": 5548678.0, + "step": 1720 + }, + { + "entropy": 0.6844640583731234, + "epoch": 0.3954285714285714, + "grad_norm": 8.0625, + "learning_rate": 1.2730334375751746e-05, + "loss": 0.9658615112304687, + "mean_token_accuracy": 0.833624093234539, + "num_tokens": 5580416.0, + "step": 1730 + }, + { + "entropy": 0.8338166723027826, + "epoch": 0.3977142857142857, + "grad_norm": 7.375, + "learning_rate": 1.2682222756795768e-05, + "loss": 1.0406496047973632, + "mean_token_accuracy": 0.8031456172466278, + "num_tokens": 5613199.0, + "step": 1740 + }, + { + "entropy": 0.6859139285050333, + "epoch": 0.4, + "grad_norm": 8.1875, + "learning_rate": 1.2634111137839789e-05, + "loss": 1.0559725761413574, + "mean_token_accuracy": 0.8320677742362023, + "num_tokens": 5644289.0, + "step": 1750 + }, + { + "entropy": 0.8010108925402164, + "epoch": 0.4022857142857143, + "grad_norm": 7.875, + "learning_rate": 1.2585999518883812e-05, + "loss": 1.113631820678711, + "mean_token_accuracy": 0.8092500284314156, + "num_tokens": 5676021.0, + "step": 1760 + }, + { + "entropy": 0.8805123227648437, + "epoch": 0.4045714285714286, + "grad_norm": 8.5625, + "learning_rate": 1.2537887899927834e-05, + "loss": 1.1203717231750487, + "mean_token_accuracy": 0.799303562939167, + "num_tokens": 5708259.0, + "step": 1770 + }, + { + "entropy": 0.7750554161146284, + "epoch": 0.40685714285714286, + "grad_norm": 7.15625, + "learning_rate": 1.2489776280971855e-05, + "loss": 1.1084386825561523, + "mean_token_accuracy": 0.8079812493175268, + "num_tokens": 5741700.0, + "step": 1780 + }, + { + "entropy": 0.8413273308426141, + "epoch": 0.40914285714285714, + "grad_norm": 7.21875, + "learning_rate": 1.2441664662015877e-05, + "loss": 1.0813825607299805, + "mean_token_accuracy": 0.8091316163539887, + "num_tokens": 5774895.0, + "step": 1790 + }, + { + "entropy": 0.754073722101748, + "epoch": 0.4114285714285714, + "grad_norm": 6.90625, + "learning_rate": 1.23935530430599e-05, + "loss": 1.0666906356811523, + "mean_token_accuracy": 0.819086953997612, + "num_tokens": 5808852.0, + "step": 1800 + }, + { + "entropy": 0.7851035035215318, + "epoch": 0.4137142857142857, + "grad_norm": 7.25, + "learning_rate": 1.234544142410392e-05, + "loss": 1.1307811737060547, + "mean_token_accuracy": 0.8178651243448257, + "num_tokens": 5841915.0, + "step": 1810 + }, + { + "entropy": 0.6699810542166234, + "epoch": 0.416, + "grad_norm": 6.375, + "learning_rate": 1.2297329805147943e-05, + "loss": 0.948750114440918, + "mean_token_accuracy": 0.8368007637560367, + "num_tokens": 5876884.0, + "step": 1820 + }, + { + "entropy": 0.7877312513999641, + "epoch": 0.41828571428571426, + "grad_norm": 8.375, + "learning_rate": 1.2249218186191967e-05, + "loss": 1.0736566543579102, + "mean_token_accuracy": 0.8172481268644333, + "num_tokens": 5909524.0, + "step": 1830 + }, + { + "entropy": 0.7636531024239958, + "epoch": 0.4205714285714286, + "grad_norm": 7.375, + "learning_rate": 1.2201106567235988e-05, + "loss": 1.0859886169433595, + "mean_token_accuracy": 0.8179905354976654, + "num_tokens": 5944343.0, + "step": 1840 + }, + { + "entropy": 0.7100124446675181, + "epoch": 0.4228571428571429, + "grad_norm": 7.15625, + "learning_rate": 1.215299494828001e-05, + "loss": 1.042219352722168, + "mean_token_accuracy": 0.8213589303195477, + "num_tokens": 5979532.0, + "step": 1850 + }, + { + "entropy": 0.7539400187321007, + "epoch": 0.42514285714285716, + "grad_norm": 7.03125, + "learning_rate": 1.2104883329324033e-05, + "loss": 1.0224475860595703, + "mean_token_accuracy": 0.8227170191705226, + "num_tokens": 6013588.0, + "step": 1860 + }, + { + "entropy": 0.6926140191964805, + "epoch": 0.42742857142857144, + "grad_norm": 6.1875, + "learning_rate": 1.2056771710368056e-05, + "loss": 0.9315377235412597, + "mean_token_accuracy": 0.8341933377087116, + "num_tokens": 6049253.0, + "step": 1870 + }, + { + "entropy": 0.8801758374087513, + "epoch": 0.4297142857142857, + "grad_norm": 7.53125, + "learning_rate": 1.2008660091412077e-05, + "loss": 1.1752006530761718, + "mean_token_accuracy": 0.7919330909848213, + "num_tokens": 6079866.0, + "step": 1880 + }, + { + "entropy": 0.899750160984695, + "epoch": 0.432, + "grad_norm": 7.21875, + "learning_rate": 1.1960548472456099e-05, + "loss": 1.1849931716918944, + "mean_token_accuracy": 0.7877020232379437, + "num_tokens": 6109239.0, + "step": 1890 + }, + { + "entropy": 0.7825359049253165, + "epoch": 0.4342857142857143, + "grad_norm": 7.46875, + "learning_rate": 1.1912436853500122e-05, + "loss": 1.0007784843444825, + "mean_token_accuracy": 0.818300225585699, + "num_tokens": 6138853.0, + "step": 1900 + }, + { + "entropy": 0.7607327317818999, + "epoch": 0.43657142857142855, + "grad_norm": 8.0, + "learning_rate": 1.1864325234544142e-05, + "loss": 1.0711479187011719, + "mean_token_accuracy": 0.8240827091038228, + "num_tokens": 6172839.0, + "step": 1910 + }, + { + "entropy": 0.8882574066519737, + "epoch": 0.43885714285714283, + "grad_norm": 7.71875, + "learning_rate": 1.1816213615588167e-05, + "loss": 1.2051098823547364, + "mean_token_accuracy": 0.7916370362043381, + "num_tokens": 6204250.0, + "step": 1920 + }, + { + "entropy": 0.6951690092682838, + "epoch": 0.44114285714285717, + "grad_norm": 8.5, + "learning_rate": 1.1768101996632189e-05, + "loss": 0.9240032196044922, + "mean_token_accuracy": 0.8321454405784607, + "num_tokens": 6236171.0, + "step": 1930 + }, + { + "entropy": 0.7487597663886845, + "epoch": 0.44342857142857145, + "grad_norm": 7.71875, + "learning_rate": 1.171999037767621e-05, + "loss": 0.9808469772338867, + "mean_token_accuracy": 0.8196887351572514, + "num_tokens": 6268135.0, + "step": 1940 + }, + { + "entropy": 0.8591405102983117, + "epoch": 0.44571428571428573, + "grad_norm": 7.78125, + "learning_rate": 1.1671878758720232e-05, + "loss": 1.1255832672119142, + "mean_token_accuracy": 0.7943931568413973, + "num_tokens": 6299543.0, + "step": 1950 + }, + { + "entropy": 0.9221633022651077, + "epoch": 0.448, + "grad_norm": 7.34375, + "learning_rate": 1.1623767139764255e-05, + "loss": 1.218088436126709, + "mean_token_accuracy": 0.7879602633416652, + "num_tokens": 6328776.0, + "step": 1960 + }, + { + "entropy": 0.7521216680295766, + "epoch": 0.4502857142857143, + "grad_norm": 4.125, + "learning_rate": 1.1575655520808276e-05, + "loss": 1.0790873527526856, + "mean_token_accuracy": 0.8138045072555542, + "num_tokens": 6359815.0, + "step": 1970 + }, + { + "entropy": 0.7044279471971094, + "epoch": 0.45257142857142857, + "grad_norm": 8.125, + "learning_rate": 1.1527543901852298e-05, + "loss": 0.925055980682373, + "mean_token_accuracy": 0.8344494268298149, + "num_tokens": 6393539.0, + "step": 1980 + }, + { + "entropy": 0.7118433593772352, + "epoch": 0.45485714285714285, + "grad_norm": 6.03125, + "learning_rate": 1.147943228289632e-05, + "loss": 0.9561774253845214, + "mean_token_accuracy": 0.8290005512535572, + "num_tokens": 6426378.0, + "step": 1990 + }, + { + "entropy": 0.7948009348474443, + "epoch": 0.45714285714285713, + "grad_norm": 6.78125, + "learning_rate": 1.1431320663940341e-05, + "loss": 1.0137686729431152, + "mean_token_accuracy": 0.8080471381545067, + "num_tokens": 6460145.0, + "step": 2000 + }, + { + "entropy": 0.7212188862264156, + "epoch": 0.4594285714285714, + "grad_norm": 6.3125, + "learning_rate": 1.1383209044984364e-05, + "loss": 0.9326017379760743, + "mean_token_accuracy": 0.8261952839791775, + "num_tokens": 6492010.0, + "step": 2010 + }, + { + "entropy": 0.7147583740763366, + "epoch": 0.4617142857142857, + "grad_norm": 7.96875, + "learning_rate": 1.1335097426028388e-05, + "loss": 1.0504631996154785, + "mean_token_accuracy": 0.831389506906271, + "num_tokens": 6526521.0, + "step": 2020 + }, + { + "entropy": 0.8111877050250769, + "epoch": 0.464, + "grad_norm": 7.71875, + "learning_rate": 1.1286985807072407e-05, + "loss": 1.0568114280700684, + "mean_token_accuracy": 0.8077960222959518, + "num_tokens": 6559743.0, + "step": 2030 + }, + { + "entropy": 0.8017427391372621, + "epoch": 0.4662857142857143, + "grad_norm": 8.125, + "learning_rate": 1.1238874188116432e-05, + "loss": 1.0879751205444337, + "mean_token_accuracy": 0.8064081892371178, + "num_tokens": 6591260.0, + "step": 2040 + }, + { + "entropy": 0.7713497207500041, + "epoch": 0.4685714285714286, + "grad_norm": 8.0, + "learning_rate": 1.1190762569160454e-05, + "loss": 1.1264875411987305, + "mean_token_accuracy": 0.8138973511755466, + "num_tokens": 6622826.0, + "step": 2050 + }, + { + "entropy": 0.7645074154250324, + "epoch": 0.47085714285714286, + "grad_norm": 7.5625, + "learning_rate": 1.1142650950204475e-05, + "loss": 1.0584564208984375, + "mean_token_accuracy": 0.8198134288191795, + "num_tokens": 6655484.0, + "step": 2060 + }, + { + "entropy": 0.8299776021391153, + "epoch": 0.47314285714285714, + "grad_norm": 6.40625, + "learning_rate": 1.1094539331248497e-05, + "loss": 1.1237126350402833, + "mean_token_accuracy": 0.8051372021436691, + "num_tokens": 6685590.0, + "step": 2070 + }, + { + "entropy": 0.7288541275076568, + "epoch": 0.4754285714285714, + "grad_norm": 7.125, + "learning_rate": 1.104642771229252e-05, + "loss": 0.9620414733886719, + "mean_token_accuracy": 0.8219352796673774, + "num_tokens": 6721547.0, + "step": 2080 + }, + { + "entropy": 0.773728856164962, + "epoch": 0.4777142857142857, + "grad_norm": 6.1875, + "learning_rate": 1.099831609333654e-05, + "loss": 1.0007498741149903, + "mean_token_accuracy": 0.8184696063399315, + "num_tokens": 6754220.0, + "step": 2090 + }, + { + "entropy": 0.7731793771497906, + "epoch": 0.48, + "grad_norm": 7.34375, + "learning_rate": 1.0950204474380563e-05, + "loss": 1.026624584197998, + "mean_token_accuracy": 0.8176170207560063, + "num_tokens": 6785160.0, + "step": 2100 + }, + { + "entropy": 0.6699161239899695, + "epoch": 0.48228571428571426, + "grad_norm": 7.09375, + "learning_rate": 1.0902092855424586e-05, + "loss": 0.9981835365295411, + "mean_token_accuracy": 0.8343960210680962, + "num_tokens": 6819546.0, + "step": 2110 + }, + { + "entropy": 0.773171486146748, + "epoch": 0.4845714285714286, + "grad_norm": 6.375, + "learning_rate": 1.0853981236468606e-05, + "loss": 1.0869997024536133, + "mean_token_accuracy": 0.8096975103020668, + "num_tokens": 6849214.0, + "step": 2120 + }, + { + "entropy": 0.7241250389255584, + "epoch": 0.4868571428571429, + "grad_norm": 7.71875, + "learning_rate": 1.080586961751263e-05, + "loss": 1.0170108795166015, + "mean_token_accuracy": 0.826240535825491, + "num_tokens": 6880349.0, + "step": 2130 + }, + { + "entropy": 0.7947101121768355, + "epoch": 0.48914285714285716, + "grad_norm": 6.15625, + "learning_rate": 1.0757757998556653e-05, + "loss": 1.0710148811340332, + "mean_token_accuracy": 0.8159072741866111, + "num_tokens": 6914887.0, + "step": 2140 + }, + { + "entropy": 0.8211502507328987, + "epoch": 0.49142857142857144, + "grad_norm": 7.625, + "learning_rate": 1.0709646379600676e-05, + "loss": 1.1436882972717286, + "mean_token_accuracy": 0.8062957197427749, + "num_tokens": 6947890.0, + "step": 2150 + }, + { + "entropy": 0.7476996140554547, + "epoch": 0.4937142857142857, + "grad_norm": 7.375, + "learning_rate": 1.0661534760644696e-05, + "loss": 1.0358779907226563, + "mean_token_accuracy": 0.82193743288517, + "num_tokens": 6976819.0, + "step": 2160 + }, + { + "entropy": 0.8045617615804076, + "epoch": 0.496, + "grad_norm": 7.375, + "learning_rate": 1.0613423141688719e-05, + "loss": 1.023204517364502, + "mean_token_accuracy": 0.8044866040349007, + "num_tokens": 7006918.0, + "step": 2170 + }, + { + "entropy": 0.7655783969908952, + "epoch": 0.4982857142857143, + "grad_norm": 7.125, + "learning_rate": 1.0565311522732741e-05, + "loss": 1.0738348007202148, + "mean_token_accuracy": 0.8179361633956432, + "num_tokens": 7040325.0, + "step": 2180 + }, + { + "entropy": 0.7762483460828662, + "epoch": 0.5005714285714286, + "grad_norm": 7.75, + "learning_rate": 1.0517199903776762e-05, + "loss": 1.0218994140625, + "mean_token_accuracy": 0.8140566550195217, + "num_tokens": 7073925.0, + "step": 2190 + }, + { + "entropy": 0.6888056830503047, + "epoch": 0.5028571428571429, + "grad_norm": 7.84375, + "learning_rate": 1.0469088284820785e-05, + "loss": 1.0055737495422363, + "mean_token_accuracy": 0.8302330307662487, + "num_tokens": 7106764.0, + "step": 2200 + }, + { + "entropy": 0.8743803231045604, + "epoch": 0.5051428571428571, + "grad_norm": 7.46875, + "learning_rate": 1.0420976665864809e-05, + "loss": 1.14207181930542, + "mean_token_accuracy": 0.7981645628809929, + "num_tokens": 7139473.0, + "step": 2210 + }, + { + "entropy": 0.7461634024046362, + "epoch": 0.5074285714285715, + "grad_norm": 7.84375, + "learning_rate": 1.0372865046908828e-05, + "loss": 0.9760645866394043, + "mean_token_accuracy": 0.8265540286898613, + "num_tokens": 7170534.0, + "step": 2220 + }, + { + "entropy": 0.6452060268260539, + "epoch": 0.5097142857142857, + "grad_norm": 6.59375, + "learning_rate": 1.0324753427952852e-05, + "loss": 0.8791199684143066, + "mean_token_accuracy": 0.8427741400897503, + "num_tokens": 7203692.0, + "step": 2230 + }, + { + "entropy": 0.7903048783540726, + "epoch": 0.512, + "grad_norm": 7.09375, + "learning_rate": 1.0276641808996875e-05, + "loss": 1.0589832305908202, + "mean_token_accuracy": 0.8105465799570084, + "num_tokens": 7235512.0, + "step": 2240 + }, + { + "entropy": 0.757332621794194, + "epoch": 0.5142857142857142, + "grad_norm": 6.34375, + "learning_rate": 1.0228530190040896e-05, + "loss": 1.0270901679992677, + "mean_token_accuracy": 0.8171210527420044, + "num_tokens": 7268071.0, + "step": 2250 + }, + { + "entropy": 0.7633547084406018, + "epoch": 0.5165714285714286, + "grad_norm": 6.84375, + "learning_rate": 1.0180418571084918e-05, + "loss": 1.0296350479125977, + "mean_token_accuracy": 0.8162448532879353, + "num_tokens": 7298520.0, + "step": 2260 + }, + { + "entropy": 0.7190552426502108, + "epoch": 0.5188571428571429, + "grad_norm": 6.90625, + "learning_rate": 1.013230695212894e-05, + "loss": 1.059223461151123, + "mean_token_accuracy": 0.8252359744161367, + "num_tokens": 7329180.0, + "step": 2270 + }, + { + "entropy": 0.7730093291960657, + "epoch": 0.5211428571428571, + "grad_norm": 7.65625, + "learning_rate": 1.0084195333172961e-05, + "loss": 1.1134568214416505, + "mean_token_accuracy": 0.810255641490221, + "num_tokens": 7363397.0, + "step": 2280 + }, + { + "entropy": 0.7890252031385898, + "epoch": 0.5234285714285715, + "grad_norm": 8.5625, + "learning_rate": 1.0036083714216984e-05, + "loss": 1.062121868133545, + "mean_token_accuracy": 0.8097737640142441, + "num_tokens": 7391404.0, + "step": 2290 + }, + { + "entropy": 0.7083958725444972, + "epoch": 0.5257142857142857, + "grad_norm": 7.625, + "learning_rate": 9.987972095261006e-06, + "loss": 0.9515584945678711, + "mean_token_accuracy": 0.8254938945174217, + "num_tokens": 7423099.0, + "step": 2300 + }, + { + "entropy": 0.6942234938032925, + "epoch": 0.528, + "grad_norm": 6.84375, + "learning_rate": 9.939860476305029e-06, + "loss": 0.9809045791625977, + "mean_token_accuracy": 0.8210146620869636, + "num_tokens": 7456548.0, + "step": 2310 + }, + { + "entropy": 0.7144082696177065, + "epoch": 0.5302857142857142, + "grad_norm": 7.40625, + "learning_rate": 9.89174885734905e-06, + "loss": 1.0528292655944824, + "mean_token_accuracy": 0.8277196332812309, + "num_tokens": 7489319.0, + "step": 2320 + }, + { + "entropy": 0.7327054421417415, + "epoch": 0.5325714285714286, + "grad_norm": 8.1875, + "learning_rate": 9.843637238393072e-06, + "loss": 1.0515410423278808, + "mean_token_accuracy": 0.822493951767683, + "num_tokens": 7524338.0, + "step": 2330 + }, + { + "entropy": 0.7691349984146655, + "epoch": 0.5348571428571428, + "grad_norm": 7.875, + "learning_rate": 9.795525619437095e-06, + "loss": 1.078810977935791, + "mean_token_accuracy": 0.8136450231075287, + "num_tokens": 7557739.0, + "step": 2340 + }, + { + "entropy": 0.8708611608482897, + "epoch": 0.5371428571428571, + "grad_norm": 6.75, + "learning_rate": 9.747414000481117e-06, + "loss": 1.1824729919433594, + "mean_token_accuracy": 0.7978243462741375, + "num_tokens": 7590943.0, + "step": 2350 + }, + { + "entropy": 0.7193792495876551, + "epoch": 0.5394285714285715, + "grad_norm": 6.09375, + "learning_rate": 9.699302381525138e-06, + "loss": 1.009437370300293, + "mean_token_accuracy": 0.8306246355175972, + "num_tokens": 7624726.0, + "step": 2360 + }, + { + "entropy": 0.7145123222842813, + "epoch": 0.5417142857142857, + "grad_norm": 8.25, + "learning_rate": 9.651190762569162e-06, + "loss": 1.0352288246154786, + "mean_token_accuracy": 0.8277433931827545, + "num_tokens": 7655654.0, + "step": 2370 + }, + { + "entropy": 0.7357332297600806, + "epoch": 0.544, + "grad_norm": 8.1875, + "learning_rate": 9.603079143613183e-06, + "loss": 1.113128662109375, + "mean_token_accuracy": 0.821478446573019, + "num_tokens": 7687948.0, + "step": 2380 + }, + { + "entropy": 0.7043834974989295, + "epoch": 0.5462857142857143, + "grad_norm": 7.375, + "learning_rate": 9.554967524657206e-06, + "loss": 0.9789829254150391, + "mean_token_accuracy": 0.8271778743714094, + "num_tokens": 7720028.0, + "step": 2390 + }, + { + "entropy": 0.8476279047317803, + "epoch": 0.5485714285714286, + "grad_norm": 7.6875, + "learning_rate": 9.506855905701228e-06, + "loss": 1.1467313766479492, + "mean_token_accuracy": 0.8044891074299813, + "num_tokens": 7754391.0, + "step": 2400 + }, + { + "entropy": 0.7447409811429679, + "epoch": 0.5508571428571428, + "grad_norm": 6.84375, + "learning_rate": 9.458744286745249e-06, + "loss": 1.0126939773559571, + "mean_token_accuracy": 0.8301733404397964, + "num_tokens": 7788615.0, + "step": 2410 + }, + { + "entropy": 0.6487678562290966, + "epoch": 0.5531428571428572, + "grad_norm": 6.90625, + "learning_rate": 9.410632667789273e-06, + "loss": 0.9349337577819824, + "mean_token_accuracy": 0.8430794902145863, + "num_tokens": 7823652.0, + "step": 2420 + }, + { + "entropy": 0.7297495853155851, + "epoch": 0.5554285714285714, + "grad_norm": 8.0, + "learning_rate": 9.362521048833294e-06, + "loss": 1.0516100883483888, + "mean_token_accuracy": 0.8286796424537897, + "num_tokens": 7855368.0, + "step": 2430 + }, + { + "entropy": 0.7656750591471791, + "epoch": 0.5577142857142857, + "grad_norm": 8.5, + "learning_rate": 9.314409429877316e-06, + "loss": 1.0632817268371582, + "mean_token_accuracy": 0.8258335165679455, + "num_tokens": 7885759.0, + "step": 2440 + }, + { + "entropy": 0.7843017770908773, + "epoch": 0.56, + "grad_norm": 8.875, + "learning_rate": 9.266297810921339e-06, + "loss": 1.074361228942871, + "mean_token_accuracy": 0.8084652818739414, + "num_tokens": 7916094.0, + "step": 2450 + }, + { + "entropy": 0.7119234027341008, + "epoch": 0.5622857142857143, + "grad_norm": 7.375, + "learning_rate": 9.21818619196536e-06, + "loss": 1.0436591148376464, + "mean_token_accuracy": 0.8270700290799141, + "num_tokens": 7948214.0, + "step": 2460 + }, + { + "entropy": 0.6967569976113737, + "epoch": 0.5645714285714286, + "grad_norm": 7.03125, + "learning_rate": 9.170074573009382e-06, + "loss": 0.9530592918395996, + "mean_token_accuracy": 0.8300783924758435, + "num_tokens": 7981204.0, + "step": 2470 + }, + { + "entropy": 0.7442635943181812, + "epoch": 0.5668571428571428, + "grad_norm": 6.96875, + "learning_rate": 9.121962954053405e-06, + "loss": 1.0191211700439453, + "mean_token_accuracy": 0.8192649222910404, + "num_tokens": 8013076.0, + "step": 2480 + }, + { + "entropy": 0.7923426426947117, + "epoch": 0.5691428571428572, + "grad_norm": 7.25, + "learning_rate": 9.073851335097427e-06, + "loss": 1.1116843223571777, + "mean_token_accuracy": 0.813247837871313, + "num_tokens": 8047471.0, + "step": 2490 + }, + { + "entropy": 0.7783824296668171, + "epoch": 0.5714285714285714, + "grad_norm": 7.1875, + "learning_rate": 9.025739716141448e-06, + "loss": 1.0703957557678223, + "mean_token_accuracy": 0.813652578741312, + "num_tokens": 8075967.0, + "step": 2500 + }, + { + "entropy": 0.7609690563753247, + "epoch": 0.5737142857142857, + "grad_norm": 7.84375, + "learning_rate": 8.97762809718547e-06, + "loss": 0.9853609085083008, + "mean_token_accuracy": 0.8176385164260864, + "num_tokens": 8109554.0, + "step": 2510 + }, + { + "entropy": 0.796685915440321, + "epoch": 0.576, + "grad_norm": 7.8125, + "learning_rate": 8.929516478229493e-06, + "loss": 1.047004508972168, + "mean_token_accuracy": 0.811551482975483, + "num_tokens": 8142729.0, + "step": 2520 + }, + { + "entropy": 0.717453905660659, + "epoch": 0.5782857142857143, + "grad_norm": 7.0, + "learning_rate": 8.881404859273516e-06, + "loss": 1.0155123710632323, + "mean_token_accuracy": 0.8243556626141071, + "num_tokens": 8171878.0, + "step": 2530 + }, + { + "entropy": 0.7719296976923943, + "epoch": 0.5805714285714285, + "grad_norm": 6.625, + "learning_rate": 8.833293240317538e-06, + "loss": 1.060032844543457, + "mean_token_accuracy": 0.8186002224683762, + "num_tokens": 8203389.0, + "step": 2540 + }, + { + "entropy": 0.6772220591083169, + "epoch": 0.5828571428571429, + "grad_norm": 7.5625, + "learning_rate": 8.785181621361559e-06, + "loss": 0.9524177551269531, + "mean_token_accuracy": 0.8351402230560779, + "num_tokens": 8233812.0, + "step": 2550 + }, + { + "entropy": 0.7519471907988191, + "epoch": 0.5851428571428572, + "grad_norm": 7.15625, + "learning_rate": 8.737070002405581e-06, + "loss": 1.0217690467834473, + "mean_token_accuracy": 0.8147253841161728, + "num_tokens": 8264075.0, + "step": 2560 + }, + { + "entropy": 0.8792341394349933, + "epoch": 0.5874285714285714, + "grad_norm": 7.625, + "learning_rate": 8.688958383449604e-06, + "loss": 1.171504306793213, + "mean_token_accuracy": 0.7875504352152347, + "num_tokens": 8294266.0, + "step": 2570 + }, + { + "entropy": 0.5147198906168342, + "epoch": 0.5897142857142857, + "grad_norm": 7.1875, + "learning_rate": 8.640846764493626e-06, + "loss": 0.7446643352508545, + "mean_token_accuracy": 0.8638051569461822, + "num_tokens": 8328371.0, + "step": 2580 + }, + { + "entropy": 0.5512435308657586, + "epoch": 0.592, + "grad_norm": 6.71875, + "learning_rate": 8.592735145537649e-06, + "loss": 0.7508518695831299, + "mean_token_accuracy": 0.8605974681675435, + "num_tokens": 8358614.0, + "step": 2590 + }, + { + "entropy": 0.753514638543129, + "epoch": 0.5942857142857143, + "grad_norm": 7.96875, + "learning_rate": 8.54462352658167e-06, + "loss": 1.0647204399108887, + "mean_token_accuracy": 0.8166808724403382, + "num_tokens": 8390572.0, + "step": 2600 + }, + { + "entropy": 0.6976428182795644, + "epoch": 0.5965714285714285, + "grad_norm": 6.96875, + "learning_rate": 8.496511907625692e-06, + "loss": 0.9744165420532227, + "mean_token_accuracy": 0.8334845989942551, + "num_tokens": 8422186.0, + "step": 2610 + }, + { + "entropy": 0.6873942642472685, + "epoch": 0.5988571428571429, + "grad_norm": 8.1875, + "learning_rate": 8.448400288669715e-06, + "loss": 1.0646495819091797, + "mean_token_accuracy": 0.8329671390354634, + "num_tokens": 8456520.0, + "step": 2620 + }, + { + "entropy": 0.7481124849990011, + "epoch": 0.6011428571428571, + "grad_norm": 7.15625, + "learning_rate": 8.400288669713737e-06, + "loss": 0.9621360778808594, + "mean_token_accuracy": 0.8160779684782028, + "num_tokens": 8489868.0, + "step": 2630 + }, + { + "entropy": 0.7540229024365545, + "epoch": 0.6034285714285714, + "grad_norm": 8.1875, + "learning_rate": 8.352177050757758e-06, + "loss": 1.068429946899414, + "mean_token_accuracy": 0.8157521851360798, + "num_tokens": 8522110.0, + "step": 2640 + }, + { + "entropy": 0.8098990395665169, + "epoch": 0.6057142857142858, + "grad_norm": 7.46875, + "learning_rate": 8.30406543180178e-06, + "loss": 1.0495105743408204, + "mean_token_accuracy": 0.8076829589903355, + "num_tokens": 8554094.0, + "step": 2650 + }, + { + "entropy": 0.8042045352049172, + "epoch": 0.608, + "grad_norm": 7.25, + "learning_rate": 8.255953812845803e-06, + "loss": 1.1015840530395509, + "mean_token_accuracy": 0.8066806077957154, + "num_tokens": 8589902.0, + "step": 2660 + }, + { + "entropy": 0.9022464168258011, + "epoch": 0.6102857142857143, + "grad_norm": 6.59375, + "learning_rate": 8.207842193889826e-06, + "loss": 1.2131352424621582, + "mean_token_accuracy": 0.7915476217865944, + "num_tokens": 8623794.0, + "step": 2670 + }, + { + "entropy": 0.8746935517527163, + "epoch": 0.6125714285714285, + "grad_norm": 6.96875, + "learning_rate": 8.159730574933848e-06, + "loss": 1.1575665473937988, + "mean_token_accuracy": 0.7986876778304577, + "num_tokens": 8651194.0, + "step": 2680 + }, + { + "entropy": 0.8122401271015406, + "epoch": 0.6148571428571429, + "grad_norm": 6.65625, + "learning_rate": 8.111618955977869e-06, + "loss": 1.0654847145080566, + "mean_token_accuracy": 0.8131648566573858, + "num_tokens": 8681096.0, + "step": 2690 + }, + { + "entropy": 0.8027309827506542, + "epoch": 0.6171428571428571, + "grad_norm": 9.0, + "learning_rate": 8.063507337021891e-06, + "loss": 1.1084778785705567, + "mean_token_accuracy": 0.8057775877416133, + "num_tokens": 8713083.0, + "step": 2700 + }, + { + "entropy": 0.7622825523838401, + "epoch": 0.6194285714285714, + "grad_norm": 7.59375, + "learning_rate": 8.015395718065914e-06, + "loss": 1.033258819580078, + "mean_token_accuracy": 0.818463982641697, + "num_tokens": 8745917.0, + "step": 2710 + }, + { + "entropy": 0.7353868483565748, + "epoch": 0.6217142857142857, + "grad_norm": 7.5, + "learning_rate": 7.967284099109935e-06, + "loss": 1.0230457305908203, + "mean_token_accuracy": 0.8215658769011498, + "num_tokens": 8781507.0, + "step": 2720 + }, + { + "entropy": 0.7007935558445751, + "epoch": 0.624, + "grad_norm": 7.375, + "learning_rate": 7.919172480153959e-06, + "loss": 0.9617188453674317, + "mean_token_accuracy": 0.8268887743353843, + "num_tokens": 8815964.0, + "step": 2730 + }, + { + "entropy": 0.7421483082696796, + "epoch": 0.6262857142857143, + "grad_norm": 7.25, + "learning_rate": 7.87106086119798e-06, + "loss": 1.0290477752685547, + "mean_token_accuracy": 0.8229492858052254, + "num_tokens": 8848344.0, + "step": 2740 + }, + { + "entropy": 0.73214913867414, + "epoch": 0.6285714285714286, + "grad_norm": 8.625, + "learning_rate": 7.822949242242002e-06, + "loss": 1.060285472869873, + "mean_token_accuracy": 0.8267331637442112, + "num_tokens": 8879118.0, + "step": 2750 + }, + { + "entropy": 0.8049950825981795, + "epoch": 0.6308571428571429, + "grad_norm": 7.625, + "learning_rate": 7.774837623286025e-06, + "loss": 1.1363885879516602, + "mean_token_accuracy": 0.8102126717567444, + "num_tokens": 8913125.0, + "step": 2760 + }, + { + "entropy": 0.6610826853662729, + "epoch": 0.6331428571428571, + "grad_norm": 6.125, + "learning_rate": 7.726726004330045e-06, + "loss": 0.9042505264282227, + "mean_token_accuracy": 0.8309091664850712, + "num_tokens": 8946194.0, + "step": 2770 + }, + { + "entropy": 0.6915491444058717, + "epoch": 0.6354285714285715, + "grad_norm": 8.0, + "learning_rate": 7.678614385374068e-06, + "loss": 0.96341552734375, + "mean_token_accuracy": 0.8264155894517898, + "num_tokens": 8982077.0, + "step": 2780 + }, + { + "entropy": 0.6272562616504729, + "epoch": 0.6377142857142857, + "grad_norm": 7.875, + "learning_rate": 7.63050276641809e-06, + "loss": 0.928917121887207, + "mean_token_accuracy": 0.8460888244211674, + "num_tokens": 9019879.0, + "step": 2790 + }, + { + "entropy": 0.701786683825776, + "epoch": 0.64, + "grad_norm": 7.1875, + "learning_rate": 7.582391147462112e-06, + "loss": 1.046267604827881, + "mean_token_accuracy": 0.8271472752094269, + "num_tokens": 9053435.0, + "step": 2800 + }, + { + "entropy": 0.8251914168708027, + "epoch": 0.6422857142857142, + "grad_norm": 7.8125, + "learning_rate": 7.534279528506135e-06, + "loss": 1.1346126556396485, + "mean_token_accuracy": 0.8051226794719696, + "num_tokens": 9087735.0, + "step": 2810 + }, + { + "entropy": 0.8158627865836025, + "epoch": 0.6445714285714286, + "grad_norm": 7.5, + "learning_rate": 7.486167909550157e-06, + "loss": 1.0751116752624512, + "mean_token_accuracy": 0.8058493196964264, + "num_tokens": 9120051.0, + "step": 2820 + }, + { + "entropy": 0.8153355809859931, + "epoch": 0.6468571428571429, + "grad_norm": 8.1875, + "learning_rate": 7.438056290594179e-06, + "loss": 1.1167887687683105, + "mean_token_accuracy": 0.8044289343059063, + "num_tokens": 9150131.0, + "step": 2830 + }, + { + "entropy": 0.7215716702863574, + "epoch": 0.6491428571428571, + "grad_norm": 6.8125, + "learning_rate": 7.389944671638201e-06, + "loss": 0.9800872802734375, + "mean_token_accuracy": 0.8230889447033405, + "num_tokens": 9183592.0, + "step": 2840 + }, + { + "entropy": 0.6650403100065887, + "epoch": 0.6514285714285715, + "grad_norm": 6.6875, + "learning_rate": 7.341833052682224e-06, + "loss": 0.930327320098877, + "mean_token_accuracy": 0.8409841381013393, + "num_tokens": 9215447.0, + "step": 2850 + }, + { + "entropy": 0.8310886896215379, + "epoch": 0.6537142857142857, + "grad_norm": 6.59375, + "learning_rate": 7.2937214337262455e-06, + "loss": 1.108950424194336, + "mean_token_accuracy": 0.8052147164940834, + "num_tokens": 9247756.0, + "step": 2860 + }, + { + "entropy": 0.7922366439364851, + "epoch": 0.656, + "grad_norm": 9.3125, + "learning_rate": 7.245609814770268e-06, + "loss": 1.089939498901367, + "mean_token_accuracy": 0.8171895481646061, + "num_tokens": 9281235.0, + "step": 2870 + }, + { + "entropy": 0.7316854421980679, + "epoch": 0.6582857142857143, + "grad_norm": 6.96875, + "learning_rate": 7.19749819581429e-06, + "loss": 0.975350284576416, + "mean_token_accuracy": 0.8272683911025525, + "num_tokens": 9315295.0, + "step": 2880 + }, + { + "entropy": 0.7693224214017391, + "epoch": 0.6605714285714286, + "grad_norm": 8.3125, + "learning_rate": 7.149386576858311e-06, + "loss": 1.1402573585510254, + "mean_token_accuracy": 0.8134943917393684, + "num_tokens": 9346923.0, + "step": 2890 + }, + { + "entropy": 0.7617660995572806, + "epoch": 0.6628571428571428, + "grad_norm": 7.59375, + "learning_rate": 7.101274957902335e-06, + "loss": 1.038425350189209, + "mean_token_accuracy": 0.8179185189306736, + "num_tokens": 9379433.0, + "step": 2900 + }, + { + "entropy": 0.7871399355120957, + "epoch": 0.6651428571428571, + "grad_norm": 7.46875, + "learning_rate": 7.053163338946356e-06, + "loss": 1.0193154335021972, + "mean_token_accuracy": 0.8183310203254223, + "num_tokens": 9413192.0, + "step": 2910 + }, + { + "entropy": 0.811190924886614, + "epoch": 0.6674285714285715, + "grad_norm": 6.625, + "learning_rate": 7.005051719990378e-06, + "loss": 1.1274406433105468, + "mean_token_accuracy": 0.8116770260035991, + "num_tokens": 9445678.0, + "step": 2920 + }, + { + "entropy": 0.8758432329632342, + "epoch": 0.6697142857142857, + "grad_norm": 8.6875, + "learning_rate": 6.9569401010344005e-06, + "loss": 1.195134162902832, + "mean_token_accuracy": 0.7941108390688896, + "num_tokens": 9476534.0, + "step": 2930 + }, + { + "entropy": 0.8360557379201055, + "epoch": 0.672, + "grad_norm": 6.46875, + "learning_rate": 6.908828482078422e-06, + "loss": 1.1135424613952636, + "mean_token_accuracy": 0.80475138053298, + "num_tokens": 9509834.0, + "step": 2940 + }, + { + "entropy": 0.8121384960599244, + "epoch": 0.6742857142857143, + "grad_norm": 7.09375, + "learning_rate": 6.860716863122444e-06, + "loss": 1.0921581268310547, + "mean_token_accuracy": 0.8104615144431591, + "num_tokens": 9544560.0, + "step": 2950 + }, + { + "entropy": 0.6809657582081854, + "epoch": 0.6765714285714286, + "grad_norm": 6.78125, + "learning_rate": 6.812605244166467e-06, + "loss": 0.9061110496520997, + "mean_token_accuracy": 0.8335768587887287, + "num_tokens": 9576699.0, + "step": 2960 + }, + { + "entropy": 0.7368661808781326, + "epoch": 0.6788571428571428, + "grad_norm": 7.0, + "learning_rate": 6.764493625210489e-06, + "loss": 0.9945035934448242, + "mean_token_accuracy": 0.8268292259424925, + "num_tokens": 9609831.0, + "step": 2970 + }, + { + "entropy": 0.7591518526896834, + "epoch": 0.6811428571428572, + "grad_norm": 7.875, + "learning_rate": 6.716382006254511e-06, + "loss": 1.0467313766479491, + "mean_token_accuracy": 0.8203754242509603, + "num_tokens": 9642043.0, + "step": 2980 + }, + { + "entropy": 0.6456474749371409, + "epoch": 0.6834285714285714, + "grad_norm": 8.5625, + "learning_rate": 6.668270387298533e-06, + "loss": 0.9298653602600098, + "mean_token_accuracy": 0.8404752813279629, + "num_tokens": 9672316.0, + "step": 2990 + }, + { + "entropy": 0.7651321108452975, + "epoch": 0.6857142857142857, + "grad_norm": 7.78125, + "learning_rate": 6.620158768342555e-06, + "loss": 0.9670675277709961, + "mean_token_accuracy": 0.8202950492501259, + "num_tokens": 9704667.0, + "step": 3000 + }, + { + "entropy": 0.7684257586486638, + "epoch": 0.688, + "grad_norm": 6.96875, + "learning_rate": 6.572047149386578e-06, + "loss": 1.0198678016662597, + "mean_token_accuracy": 0.8141922578215599, + "num_tokens": 9736453.0, + "step": 3010 + }, + { + "entropy": 0.9071576375514269, + "epoch": 0.6902857142857143, + "grad_norm": 7.46875, + "learning_rate": 6.5239355304306e-06, + "loss": 1.2006649017333983, + "mean_token_accuracy": 0.7815720088779926, + "num_tokens": 9766239.0, + "step": 3020 + }, + { + "entropy": 0.7002435548231005, + "epoch": 0.6925714285714286, + "grad_norm": 8.1875, + "learning_rate": 6.475823911474621e-06, + "loss": 1.0064776420593262, + "mean_token_accuracy": 0.8239927910268307, + "num_tokens": 9797633.0, + "step": 3030 + }, + { + "entropy": 0.7579085680190474, + "epoch": 0.6948571428571428, + "grad_norm": 7.71875, + "learning_rate": 6.427712292518644e-06, + "loss": 1.0041309356689454, + "mean_token_accuracy": 0.8205542616546154, + "num_tokens": 9829628.0, + "step": 3040 + }, + { + "entropy": 0.8784082194790244, + "epoch": 0.6971428571428572, + "grad_norm": 7.40625, + "learning_rate": 6.379600673562666e-06, + "loss": 1.1010237693786622, + "mean_token_accuracy": 0.7967388294637203, + "num_tokens": 9864302.0, + "step": 3050 + }, + { + "entropy": 0.6545436557382345, + "epoch": 0.6994285714285714, + "grad_norm": 7.71875, + "learning_rate": 6.331489054606688e-06, + "loss": 1.0129745483398438, + "mean_token_accuracy": 0.8387912206351757, + "num_tokens": 9895779.0, + "step": 3060 + }, + { + "entropy": 0.8425466694869101, + "epoch": 0.7017142857142857, + "grad_norm": 6.78125, + "learning_rate": 6.2833774356507104e-06, + "loss": 1.0562131881713868, + "mean_token_accuracy": 0.8017945110797882, + "num_tokens": 9924158.0, + "step": 3070 + }, + { + "entropy": 0.873399674333632, + "epoch": 0.704, + "grad_norm": 8.75, + "learning_rate": 6.235265816694732e-06, + "loss": 1.1048128128051757, + "mean_token_accuracy": 0.7917656324803829, + "num_tokens": 9955099.0, + "step": 3080 + }, + { + "entropy": 0.6073906374163925, + "epoch": 0.7062857142857143, + "grad_norm": 9.0625, + "learning_rate": 6.187154197738754e-06, + "loss": 0.8535347938537597, + "mean_token_accuracy": 0.8521837316453457, + "num_tokens": 9988123.0, + "step": 3090 + }, + { + "entropy": 0.8428573332726955, + "epoch": 0.7085714285714285, + "grad_norm": 7.15625, + "learning_rate": 6.139042578782777e-06, + "loss": 1.1206453323364258, + "mean_token_accuracy": 0.7989229038357735, + "num_tokens": 10016364.0, + "step": 3100 + }, + { + "entropy": 0.740996487159282, + "epoch": 0.7108571428571429, + "grad_norm": 7.96875, + "learning_rate": 6.090930959826799e-06, + "loss": 1.0498098373413085, + "mean_token_accuracy": 0.8253378972411156, + "num_tokens": 10047098.0, + "step": 3110 + }, + { + "entropy": 0.7634292658418417, + "epoch": 0.7131428571428572, + "grad_norm": 7.40625, + "learning_rate": 6.042819340870821e-06, + "loss": 1.027073860168457, + "mean_token_accuracy": 0.8156747639179229, + "num_tokens": 10078377.0, + "step": 3120 + }, + { + "entropy": 0.7800739608705044, + "epoch": 0.7154285714285714, + "grad_norm": 8.0625, + "learning_rate": 5.994707721914843e-06, + "loss": 1.0465456962585449, + "mean_token_accuracy": 0.8120176091790199, + "num_tokens": 10108276.0, + "step": 3130 + }, + { + "entropy": 0.6941058835014701, + "epoch": 0.7177142857142857, + "grad_norm": 8.25, + "learning_rate": 5.946596102958865e-06, + "loss": 0.9293928146362305, + "mean_token_accuracy": 0.8261455290019513, + "num_tokens": 10140397.0, + "step": 3140 + }, + { + "entropy": 0.7566414731554687, + "epoch": 0.72, + "grad_norm": 7.78125, + "learning_rate": 5.898484484002888e-06, + "loss": 1.0103553771972655, + "mean_token_accuracy": 0.8191342234611512, + "num_tokens": 10170396.0, + "step": 3150 + }, + { + "entropy": 0.6843882665038109, + "epoch": 0.7222857142857143, + "grad_norm": 6.90625, + "learning_rate": 5.85037286504691e-06, + "loss": 0.9306538581848145, + "mean_token_accuracy": 0.8384991563856602, + "num_tokens": 10203787.0, + "step": 3160 + }, + { + "entropy": 0.9223319502547384, + "epoch": 0.7245714285714285, + "grad_norm": 7.09375, + "learning_rate": 5.802261246090931e-06, + "loss": 1.2310810089111328, + "mean_token_accuracy": 0.7850386075675487, + "num_tokens": 10237964.0, + "step": 3170 + }, + { + "entropy": 0.9315114512108267, + "epoch": 0.7268571428571429, + "grad_norm": 7.875, + "learning_rate": 5.754149627134954e-06, + "loss": 1.2219575881958007, + "mean_token_accuracy": 0.788118976727128, + "num_tokens": 10272012.0, + "step": 3180 + }, + { + "entropy": 0.9613786770962178, + "epoch": 0.7291428571428571, + "grad_norm": 7.71875, + "learning_rate": 5.706038008178975e-06, + "loss": 1.2217192649841309, + "mean_token_accuracy": 0.7753416560590267, + "num_tokens": 10301067.0, + "step": 3190 + }, + { + "entropy": 0.8978491752408445, + "epoch": 0.7314285714285714, + "grad_norm": 8.125, + "learning_rate": 5.657926389222997e-06, + "loss": 1.220133399963379, + "mean_token_accuracy": 0.7887196414172649, + "num_tokens": 10332467.0, + "step": 3200 + }, + { + "entropy": 0.7975409649312496, + "epoch": 0.7337142857142858, + "grad_norm": 6.875, + "learning_rate": 5.60981477026702e-06, + "loss": 1.0842624664306642, + "mean_token_accuracy": 0.8157493643462658, + "num_tokens": 10363576.0, + "step": 3210 + }, + { + "entropy": 0.7229632311500609, + "epoch": 0.736, + "grad_norm": 5.84375, + "learning_rate": 5.561703151311042e-06, + "loss": 0.9485625267028809, + "mean_token_accuracy": 0.8276536233723164, + "num_tokens": 10395203.0, + "step": 3220 + }, + { + "entropy": 0.7023040059953928, + "epoch": 0.7382857142857143, + "grad_norm": 6.90625, + "learning_rate": 5.513591532355064e-06, + "loss": 1.0307831764221191, + "mean_token_accuracy": 0.8284949451684952, + "num_tokens": 10426392.0, + "step": 3230 + }, + { + "entropy": 0.7922056226991117, + "epoch": 0.7405714285714285, + "grad_norm": 7.78125, + "learning_rate": 5.465479913399086e-06, + "loss": 1.0614632606506347, + "mean_token_accuracy": 0.8134492255747319, + "num_tokens": 10461629.0, + "step": 3240 + }, + { + "entropy": 0.5897016246803105, + "epoch": 0.7428571428571429, + "grad_norm": 7.1875, + "learning_rate": 5.417368294443108e-06, + "loss": 0.9088504791259766, + "mean_token_accuracy": 0.8521728955209256, + "num_tokens": 10497681.0, + "step": 3250 + }, + { + "entropy": 0.7967038029804826, + "epoch": 0.7451428571428571, + "grad_norm": 7.8125, + "learning_rate": 5.36925667548713e-06, + "loss": 1.0916454315185546, + "mean_token_accuracy": 0.8093243841081857, + "num_tokens": 10531276.0, + "step": 3260 + }, + { + "entropy": 0.7693165981210768, + "epoch": 0.7474285714285714, + "grad_norm": 6.84375, + "learning_rate": 5.321145056531153e-06, + "loss": 0.9624992370605469, + "mean_token_accuracy": 0.818351661413908, + "num_tokens": 10560781.0, + "step": 3270 + }, + { + "entropy": 0.7283342545852065, + "epoch": 0.7497142857142857, + "grad_norm": 6.78125, + "learning_rate": 5.2730334375751746e-06, + "loss": 1.0297183990478516, + "mean_token_accuracy": 0.8240480080246926, + "num_tokens": 10594592.0, + "step": 3280 + }, + { + "entropy": 0.7524079182185233, + "epoch": 0.752, + "grad_norm": 6.96875, + "learning_rate": 5.224921818619197e-06, + "loss": 0.980162239074707, + "mean_token_accuracy": 0.8201256044209003, + "num_tokens": 10627039.0, + "step": 3290 + }, + { + "entropy": 0.7365978182293474, + "epoch": 0.7542857142857143, + "grad_norm": 6.9375, + "learning_rate": 5.1768101996632196e-06, + "loss": 0.9751132011413575, + "mean_token_accuracy": 0.8289193719625473, + "num_tokens": 10659890.0, + "step": 3300 + }, + { + "entropy": 0.766766385640949, + "epoch": 0.7565714285714286, + "grad_norm": 7.0, + "learning_rate": 5.128698580707241e-06, + "loss": 1.066256618499756, + "mean_token_accuracy": 0.813833087682724, + "num_tokens": 10689935.0, + "step": 3310 + }, + { + "entropy": 0.7413464878685773, + "epoch": 0.7588571428571429, + "grad_norm": 7.4375, + "learning_rate": 5.080586961751264e-06, + "loss": 0.9947978019714355, + "mean_token_accuracy": 0.8286945290863514, + "num_tokens": 10719202.0, + "step": 3320 + }, + { + "entropy": 0.8110045059584081, + "epoch": 0.7611428571428571, + "grad_norm": 6.875, + "learning_rate": 5.032475342795285e-06, + "loss": 1.1004474639892579, + "mean_token_accuracy": 0.8095826417207718, + "num_tokens": 10752706.0, + "step": 3330 + }, + { + "entropy": 0.8748351650312542, + "epoch": 0.7634285714285715, + "grad_norm": 5.71875, + "learning_rate": 4.984363723839308e-06, + "loss": 1.1687129020690918, + "mean_token_accuracy": 0.792143489792943, + "num_tokens": 10785853.0, + "step": 3340 + }, + { + "entropy": 0.7911639436148107, + "epoch": 0.7657142857142857, + "grad_norm": 7.25, + "learning_rate": 4.9362521048833295e-06, + "loss": 1.075576114654541, + "mean_token_accuracy": 0.8155123472213746, + "num_tokens": 10818229.0, + "step": 3350 + }, + { + "entropy": 0.7263018532656134, + "epoch": 0.768, + "grad_norm": 7.625, + "learning_rate": 4.888140485927352e-06, + "loss": 1.0433449745178223, + "mean_token_accuracy": 0.8173217661678791, + "num_tokens": 10850567.0, + "step": 3360 + }, + { + "entropy": 0.8579608911648393, + "epoch": 0.7702857142857142, + "grad_norm": 8.0, + "learning_rate": 4.840028866971374e-06, + "loss": 1.106894302368164, + "mean_token_accuracy": 0.7974334202706814, + "num_tokens": 10883673.0, + "step": 3370 + }, + { + "entropy": 0.844954667147249, + "epoch": 0.7725714285714286, + "grad_norm": 7.09375, + "learning_rate": 4.791917248015396e-06, + "loss": 1.1396818161010742, + "mean_token_accuracy": 0.8037762485444546, + "num_tokens": 10916999.0, + "step": 3380 + }, + { + "entropy": 0.8294042806141079, + "epoch": 0.7748571428571429, + "grad_norm": 7.15625, + "learning_rate": 4.743805629059418e-06, + "loss": 1.07614164352417, + "mean_token_accuracy": 0.8013096928596497, + "num_tokens": 10947976.0, + "step": 3390 + }, + { + "entropy": 0.699165354296565, + "epoch": 0.7771428571428571, + "grad_norm": 7.3125, + "learning_rate": 4.69569401010344e-06, + "loss": 0.9428619384765625, + "mean_token_accuracy": 0.8234991244971752, + "num_tokens": 10981402.0, + "step": 3400 + }, + { + "entropy": 0.6559101923368871, + "epoch": 0.7794285714285715, + "grad_norm": 8.6875, + "learning_rate": 4.647582391147463e-06, + "loss": 0.851567554473877, + "mean_token_accuracy": 0.8403317756950855, + "num_tokens": 11011139.0, + "step": 3410 + }, + { + "entropy": 0.8098050164990127, + "epoch": 0.7817142857142857, + "grad_norm": 7.53125, + "learning_rate": 4.5994707721914845e-06, + "loss": 1.11738862991333, + "mean_token_accuracy": 0.8105709552764893, + "num_tokens": 11044319.0, + "step": 3420 + }, + { + "entropy": 0.6640732565894722, + "epoch": 0.784, + "grad_norm": 8.125, + "learning_rate": 4.551359153235507e-06, + "loss": 0.9586344718933105, + "mean_token_accuracy": 0.8361151710152626, + "num_tokens": 11079905.0, + "step": 3430 + }, + { + "entropy": 0.7955016130581498, + "epoch": 0.7862857142857143, + "grad_norm": 8.0, + "learning_rate": 4.503247534279529e-06, + "loss": 1.0718993186950683, + "mean_token_accuracy": 0.8115546323359013, + "num_tokens": 11111800.0, + "step": 3440 + }, + { + "entropy": 0.7911881297826767, + "epoch": 0.7885714285714286, + "grad_norm": 7.78125, + "learning_rate": 4.455135915323551e-06, + "loss": 1.099710750579834, + "mean_token_accuracy": 0.8119720377027988, + "num_tokens": 11145332.0, + "step": 3450 + }, + { + "entropy": 0.774864933360368, + "epoch": 0.7908571428571428, + "grad_norm": 6.21875, + "learning_rate": 4.407024296367573e-06, + "loss": 1.0780227661132813, + "mean_token_accuracy": 0.8145800329744816, + "num_tokens": 11174470.0, + "step": 3460 + }, + { + "entropy": 0.6018477959558368, + "epoch": 0.7931428571428571, + "grad_norm": 7.4375, + "learning_rate": 4.358912677411595e-06, + "loss": 0.8521579742431641, + "mean_token_accuracy": 0.8514960691332817, + "num_tokens": 11210615.0, + "step": 3470 + }, + { + "entropy": 0.8444686807692051, + "epoch": 0.7954285714285714, + "grad_norm": 6.6875, + "learning_rate": 4.310801058455618e-06, + "loss": 1.1099799156188965, + "mean_token_accuracy": 0.8008836135268211, + "num_tokens": 11241513.0, + "step": 3480 + }, + { + "entropy": 0.829711533524096, + "epoch": 0.7977142857142857, + "grad_norm": 6.625, + "learning_rate": 4.2626894394996395e-06, + "loss": 1.0666452407836915, + "mean_token_accuracy": 0.8077532455325127, + "num_tokens": 11274813.0, + "step": 3490 + }, + { + "entropy": 0.7473221772350371, + "epoch": 0.8, + "grad_norm": 7.3125, + "learning_rate": 4.214577820543661e-06, + "loss": 1.0066685676574707, + "mean_token_accuracy": 0.8148701801896095, + "num_tokens": 11303959.0, + "step": 3500 + }, + { + "entropy": 0.8557392791844904, + "epoch": 0.8022857142857143, + "grad_norm": 7.6875, + "learning_rate": 4.166466201587684e-06, + "loss": 1.0541882514953613, + "mean_token_accuracy": 0.7998419582843781, + "num_tokens": 11334910.0, + "step": 3510 + }, + { + "entropy": 0.7890441759489477, + "epoch": 0.8045714285714286, + "grad_norm": 7.71875, + "learning_rate": 4.118354582631705e-06, + "loss": 1.0888317108154297, + "mean_token_accuracy": 0.8178003937005996, + "num_tokens": 11365426.0, + "step": 3520 + }, + { + "entropy": 0.7986143685877323, + "epoch": 0.8068571428571428, + "grad_norm": 6.78125, + "learning_rate": 4.070242963675728e-06, + "loss": 1.0438194274902344, + "mean_token_accuracy": 0.8172411516308784, + "num_tokens": 11396568.0, + "step": 3530 + }, + { + "entropy": 0.7023409645073115, + "epoch": 0.8091428571428572, + "grad_norm": 7.125, + "learning_rate": 4.02213134471975e-06, + "loss": 0.982151985168457, + "mean_token_accuracy": 0.8304105646908283, + "num_tokens": 11429858.0, + "step": 3540 + }, + { + "entropy": 0.8511648692190648, + "epoch": 0.8114285714285714, + "grad_norm": 7.625, + "learning_rate": 3.974019725763773e-06, + "loss": 1.118496799468994, + "mean_token_accuracy": 0.8004165187478065, + "num_tokens": 11458825.0, + "step": 3550 + }, + { + "entropy": 0.8242282169871032, + "epoch": 0.8137142857142857, + "grad_norm": 7.15625, + "learning_rate": 3.9259081068077945e-06, + "loss": 1.0889615058898925, + "mean_token_accuracy": 0.8154613308608532, + "num_tokens": 11485780.0, + "step": 3560 + }, + { + "entropy": 0.8761372335255146, + "epoch": 0.816, + "grad_norm": 8.1875, + "learning_rate": 3.877796487851816e-06, + "loss": 1.1666927337646484, + "mean_token_accuracy": 0.7958627745509148, + "num_tokens": 11515089.0, + "step": 3570 + }, + { + "entropy": 0.8123947971500456, + "epoch": 0.8182857142857143, + "grad_norm": 7.1875, + "learning_rate": 3.829684868895839e-06, + "loss": 1.0921098709106445, + "mean_token_accuracy": 0.8081369504332543, + "num_tokens": 11545683.0, + "step": 3580 + }, + { + "entropy": 0.7641672321595252, + "epoch": 0.8205714285714286, + "grad_norm": 12.875, + "learning_rate": 3.7815732499398603e-06, + "loss": 1.0089197158813477, + "mean_token_accuracy": 0.8187419034540653, + "num_tokens": 11579338.0, + "step": 3590 + }, + { + "entropy": 0.9396181921474636, + "epoch": 0.8228571428571428, + "grad_norm": 6.875, + "learning_rate": 3.733461630983883e-06, + "loss": 1.1994843482971191, + "mean_token_accuracy": 0.7850606590509415, + "num_tokens": 11611832.0, + "step": 3600 + }, + { + "entropy": 0.692871849052608, + "epoch": 0.8251428571428572, + "grad_norm": 7.4375, + "learning_rate": 3.685350012027905e-06, + "loss": 0.9562499046325683, + "mean_token_accuracy": 0.8297605454921723, + "num_tokens": 11647063.0, + "step": 3610 + }, + { + "entropy": 0.8553698582574725, + "epoch": 0.8274285714285714, + "grad_norm": 6.40625, + "learning_rate": 3.6372383930719274e-06, + "loss": 1.1813543319702149, + "mean_token_accuracy": 0.7899277493357658, + "num_tokens": 11678198.0, + "step": 3620 + }, + { + "entropy": 0.7769951789639891, + "epoch": 0.8297142857142857, + "grad_norm": 7.84375, + "learning_rate": 3.589126774115949e-06, + "loss": 1.1048757553100585, + "mean_token_accuracy": 0.8161457117646933, + "num_tokens": 11711274.0, + "step": 3630 + }, + { + "entropy": 0.7809062311425805, + "epoch": 0.832, + "grad_norm": 7.03125, + "learning_rate": 3.5410151551599716e-06, + "loss": 1.116710090637207, + "mean_token_accuracy": 0.8137276962399482, + "num_tokens": 11742228.0, + "step": 3640 + }, + { + "entropy": 0.7451124029234052, + "epoch": 0.8342857142857143, + "grad_norm": 7.53125, + "learning_rate": 3.4929035362039937e-06, + "loss": 0.9379715919494629, + "mean_token_accuracy": 0.8171642825007439, + "num_tokens": 11773088.0, + "step": 3650 + }, + { + "entropy": 0.7952216092497111, + "epoch": 0.8365714285714285, + "grad_norm": 6.40625, + "learning_rate": 3.4447919172480153e-06, + "loss": 1.1145612716674804, + "mean_token_accuracy": 0.8073516443371773, + "num_tokens": 11805230.0, + "step": 3660 + }, + { + "entropy": 0.6641817208379507, + "epoch": 0.8388571428571429, + "grad_norm": 8.5625, + "learning_rate": 3.396680298292038e-06, + "loss": 0.8219084739685059, + "mean_token_accuracy": 0.8350592002272605, + "num_tokens": 11836944.0, + "step": 3670 + }, + { + "entropy": 0.8327237972058356, + "epoch": 0.8411428571428572, + "grad_norm": 7.90625, + "learning_rate": 3.34856867933606e-06, + "loss": 1.152987289428711, + "mean_token_accuracy": 0.8037018492817879, + "num_tokens": 11870633.0, + "step": 3680 + }, + { + "entropy": 0.7739829862490296, + "epoch": 0.8434285714285714, + "grad_norm": 7.6875, + "learning_rate": 3.3004570603800824e-06, + "loss": 1.0689297676086427, + "mean_token_accuracy": 0.8115721188485623, + "num_tokens": 11901509.0, + "step": 3690 + }, + { + "entropy": 0.7023711124435067, + "epoch": 0.8457142857142858, + "grad_norm": 7.09375, + "learning_rate": 3.252345441424104e-06, + "loss": 0.9941452980041504, + "mean_token_accuracy": 0.8265939429402351, + "num_tokens": 11931513.0, + "step": 3700 + }, + { + "entropy": 0.8765215444844217, + "epoch": 0.848, + "grad_norm": 7.75, + "learning_rate": 3.204233822468126e-06, + "loss": 1.1260786056518555, + "mean_token_accuracy": 0.7915762890130281, + "num_tokens": 11964449.0, + "step": 3710 + }, + { + "entropy": 0.8084148010239005, + "epoch": 0.8502857142857143, + "grad_norm": 6.84375, + "learning_rate": 3.1561222035121486e-06, + "loss": 1.1048534393310547, + "mean_token_accuracy": 0.8068965047597885, + "num_tokens": 11994053.0, + "step": 3720 + }, + { + "entropy": 0.8611850501969457, + "epoch": 0.8525714285714285, + "grad_norm": 6.6875, + "learning_rate": 3.1080105845561703e-06, + "loss": 1.1287246704101563, + "mean_token_accuracy": 0.7991515025496483, + "num_tokens": 12027638.0, + "step": 3730 + }, + { + "entropy": 0.6358927502296865, + "epoch": 0.8548571428571429, + "grad_norm": 7.0, + "learning_rate": 3.059898965600193e-06, + "loss": 0.9071885108947754, + "mean_token_accuracy": 0.8444961465895175, + "num_tokens": 12062607.0, + "step": 3740 + }, + { + "entropy": 0.8599010735284537, + "epoch": 0.8571428571428571, + "grad_norm": 6.5, + "learning_rate": 3.011787346644215e-06, + "loss": 1.148247241973877, + "mean_token_accuracy": 0.7988054849207401, + "num_tokens": 12093334.0, + "step": 3750 + }, + { + "entropy": 0.749000935535878, + "epoch": 0.8594285714285714, + "grad_norm": 6.90625, + "learning_rate": 2.9636757276882374e-06, + "loss": 1.0731889724731445, + "mean_token_accuracy": 0.8171209901571274, + "num_tokens": 12124493.0, + "step": 3760 + }, + { + "entropy": 0.6133915192447603, + "epoch": 0.8617142857142858, + "grad_norm": 6.0, + "learning_rate": 2.915564108732259e-06, + "loss": 0.9234248161315918, + "mean_token_accuracy": 0.8530650399625301, + "num_tokens": 12160800.0, + "step": 3770 + }, + { + "entropy": 0.8550255595706403, + "epoch": 0.864, + "grad_norm": 7.40625, + "learning_rate": 2.867452489776281e-06, + "loss": 1.1787774085998535, + "mean_token_accuracy": 0.7914231061935425, + "num_tokens": 12189992.0, + "step": 3780 + }, + { + "entropy": 0.854400450270623, + "epoch": 0.8662857142857143, + "grad_norm": 6.5625, + "learning_rate": 2.8193408708203036e-06, + "loss": 1.1640611648559571, + "mean_token_accuracy": 0.8066024430096149, + "num_tokens": 12224329.0, + "step": 3790 + }, + { + "entropy": 0.8460091168992221, + "epoch": 0.8685714285714285, + "grad_norm": 6.96875, + "learning_rate": 2.7712292518643253e-06, + "loss": 1.1152153015136719, + "mean_token_accuracy": 0.806992469727993, + "num_tokens": 12257176.0, + "step": 3800 + }, + { + "entropy": 0.7765115794725717, + "epoch": 0.8708571428571429, + "grad_norm": 7.1875, + "learning_rate": 2.7231176329083474e-06, + "loss": 1.0872262954711913, + "mean_token_accuracy": 0.813801209628582, + "num_tokens": 12290364.0, + "step": 3810 + }, + { + "entropy": 0.7125355136580765, + "epoch": 0.8731428571428571, + "grad_norm": 6.3125, + "learning_rate": 2.67500601395237e-06, + "loss": 0.9770938873291015, + "mean_token_accuracy": 0.8250508345663548, + "num_tokens": 12325455.0, + "step": 3820 + }, + { + "entropy": 0.7758281454443932, + "epoch": 0.8754285714285714, + "grad_norm": 7.625, + "learning_rate": 2.626894394996392e-06, + "loss": 1.130514430999756, + "mean_token_accuracy": 0.8151266768574714, + "num_tokens": 12358823.0, + "step": 3830 + }, + { + "entropy": 0.7530996017158031, + "epoch": 0.8777142857142857, + "grad_norm": 7.6875, + "learning_rate": 2.578782776040414e-06, + "loss": 1.056538200378418, + "mean_token_accuracy": 0.8141079384833574, + "num_tokens": 12392420.0, + "step": 3840 + }, + { + "entropy": 0.582604228099808, + "epoch": 0.88, + "grad_norm": 8.3125, + "learning_rate": 2.530671157084436e-06, + "loss": 0.8646469116210938, + "mean_token_accuracy": 0.850695987790823, + "num_tokens": 12424048.0, + "step": 3850 + }, + { + "entropy": 0.8167831319384277, + "epoch": 0.8822857142857143, + "grad_norm": 7.09375, + "learning_rate": 2.482559538128458e-06, + "loss": 1.1348252296447754, + "mean_token_accuracy": 0.8092688016593457, + "num_tokens": 12455436.0, + "step": 3860 + }, + { + "entropy": 0.8215475841425359, + "epoch": 0.8845714285714286, + "grad_norm": 7.4375, + "learning_rate": 2.4344479191724803e-06, + "loss": 1.1215962409973144, + "mean_token_accuracy": 0.8079402819275856, + "num_tokens": 12489334.0, + "step": 3870 + }, + { + "entropy": 0.7927017042413353, + "epoch": 0.8868571428571429, + "grad_norm": 7.28125, + "learning_rate": 2.3863363002165024e-06, + "loss": 1.0800617218017579, + "mean_token_accuracy": 0.8097000844776631, + "num_tokens": 12522105.0, + "step": 3880 + }, + { + "entropy": 0.8486208325251937, + "epoch": 0.8891428571428571, + "grad_norm": 7.0, + "learning_rate": 2.338224681260525e-06, + "loss": 1.125669288635254, + "mean_token_accuracy": 0.7976283885538578, + "num_tokens": 12553195.0, + "step": 3890 + }, + { + "entropy": 0.8716784932184964, + "epoch": 0.8914285714285715, + "grad_norm": 6.96875, + "learning_rate": 2.2901130623045465e-06, + "loss": 1.1960984230041505, + "mean_token_accuracy": 0.7935221865773201, + "num_tokens": 12585132.0, + "step": 3900 + }, + { + "entropy": 0.8773073226213455, + "epoch": 0.8937142857142857, + "grad_norm": 8.6875, + "learning_rate": 2.2420014433485686e-06, + "loss": 1.166949462890625, + "mean_token_accuracy": 0.7921496272087097, + "num_tokens": 12620190.0, + "step": 3910 + }, + { + "entropy": 0.8945854822173714, + "epoch": 0.896, + "grad_norm": 7.90625, + "learning_rate": 2.193889824392591e-06, + "loss": 1.20922908782959, + "mean_token_accuracy": 0.7929078787565231, + "num_tokens": 12652043.0, + "step": 3920 + }, + { + "entropy": 0.7110437804833054, + "epoch": 0.8982857142857142, + "grad_norm": 7.4375, + "learning_rate": 2.145778205436613e-06, + "loss": 0.9799498558044434, + "mean_token_accuracy": 0.8256872028112412, + "num_tokens": 12686506.0, + "step": 3930 + }, + { + "entropy": 0.8520329046063125, + "epoch": 0.9005714285714286, + "grad_norm": 7.84375, + "learning_rate": 2.0976665864806353e-06, + "loss": 1.1127940177917481, + "mean_token_accuracy": 0.7984279960393905, + "num_tokens": 12717624.0, + "step": 3940 + }, + { + "entropy": 0.7722635400481522, + "epoch": 0.9028571428571428, + "grad_norm": 6.96875, + "learning_rate": 2.0495549675246573e-06, + "loss": 1.031348705291748, + "mean_token_accuracy": 0.8082475580275059, + "num_tokens": 12751004.0, + "step": 3950 + }, + { + "entropy": 0.8996719061397016, + "epoch": 0.9051428571428571, + "grad_norm": 9.0, + "learning_rate": 2.0014433485686794e-06, + "loss": 1.2313031196594237, + "mean_token_accuracy": 0.7899208262562751, + "num_tokens": 12784386.0, + "step": 3960 + }, + { + "entropy": 0.8467463178560137, + "epoch": 0.9074285714285715, + "grad_norm": 6.3125, + "learning_rate": 1.9533317296127015e-06, + "loss": 1.1081655502319336, + "mean_token_accuracy": 0.7957226369529963, + "num_tokens": 12813355.0, + "step": 3970 + }, + { + "entropy": 0.8022918193601072, + "epoch": 0.9097142857142857, + "grad_norm": 7.90625, + "learning_rate": 1.9052201106567236e-06, + "loss": 1.0847484588623046, + "mean_token_accuracy": 0.8091882210224867, + "num_tokens": 12846173.0, + "step": 3980 + }, + { + "entropy": 0.7512643322348594, + "epoch": 0.912, + "grad_norm": 7.34375, + "learning_rate": 1.8571084917007459e-06, + "loss": 1.0247625350952148, + "mean_token_accuracy": 0.8154100321233273, + "num_tokens": 12879534.0, + "step": 3990 + }, + { + "entropy": 0.8367441557347775, + "epoch": 0.9142857142857143, + "grad_norm": 7.15625, + "learning_rate": 1.808996872744768e-06, + "loss": 1.2222256660461426, + "mean_token_accuracy": 0.8002955429255962, + "num_tokens": 12907935.0, + "step": 4000 + }, + { + "entropy": 0.849480548966676, + "epoch": 0.9165714285714286, + "grad_norm": 7.25, + "learning_rate": 1.7608852537887902e-06, + "loss": 1.1237051963806153, + "mean_token_accuracy": 0.7980944596230983, + "num_tokens": 12939436.0, + "step": 4010 + }, + { + "entropy": 0.7073171893134713, + "epoch": 0.9188571428571428, + "grad_norm": 7.6875, + "learning_rate": 1.7127736348328123e-06, + "loss": 1.0514840126037597, + "mean_token_accuracy": 0.8278530709445476, + "num_tokens": 12968488.0, + "step": 4020 + }, + { + "entropy": 0.7504108159802854, + "epoch": 0.9211428571428572, + "grad_norm": 7.96875, + "learning_rate": 1.6646620158768346e-06, + "loss": 1.0060768127441406, + "mean_token_accuracy": 0.8199899084866047, + "num_tokens": 13002635.0, + "step": 4030 + }, + { + "entropy": 0.906342108361423, + "epoch": 0.9234285714285714, + "grad_norm": 6.84375, + "learning_rate": 1.6165503969208565e-06, + "loss": 1.2012577056884766, + "mean_token_accuracy": 0.7861776262521744, + "num_tokens": 13034904.0, + "step": 4040 + }, + { + "entropy": 0.8890602920204401, + "epoch": 0.9257142857142857, + "grad_norm": 7.09375, + "learning_rate": 1.5684387779648786e-06, + "loss": 1.1589385986328125, + "mean_token_accuracy": 0.7936351835727692, + "num_tokens": 13064519.0, + "step": 4050 + }, + { + "entropy": 0.8940521791577339, + "epoch": 0.928, + "grad_norm": 7.53125, + "learning_rate": 1.5203271590089009e-06, + "loss": 1.1503165245056153, + "mean_token_accuracy": 0.7895523980259895, + "num_tokens": 13098510.0, + "step": 4060 + }, + { + "entropy": 0.6112047893926501, + "epoch": 0.9302857142857143, + "grad_norm": 9.1875, + "learning_rate": 1.4722155400529227e-06, + "loss": 0.8700815200805664, + "mean_token_accuracy": 0.852671717107296, + "num_tokens": 13132618.0, + "step": 4070 + }, + { + "entropy": 0.7455820512957871, + "epoch": 0.9325714285714286, + "grad_norm": 6.96875, + "learning_rate": 1.424103921096945e-06, + "loss": 0.9891318321228028, + "mean_token_accuracy": 0.8168230719864369, + "num_tokens": 13161914.0, + "step": 4080 + }, + { + "entropy": 0.8070791523903609, + "epoch": 0.9348571428571428, + "grad_norm": 7.34375, + "learning_rate": 1.375992302140967e-06, + "loss": 1.1282638549804687, + "mean_token_accuracy": 0.8067759692668914, + "num_tokens": 13189053.0, + "step": 4090 + }, + { + "entropy": 0.8544201260432601, + "epoch": 0.9371428571428572, + "grad_norm": 7.34375, + "learning_rate": 1.3278806831849894e-06, + "loss": 1.2156527519226075, + "mean_token_accuracy": 0.8019159339368344, + "num_tokens": 13219478.0, + "step": 4100 + }, + { + "entropy": 0.8153353109024465, + "epoch": 0.9394285714285714, + "grad_norm": 7.84375, + "learning_rate": 1.2797690642290115e-06, + "loss": 1.0747637748718262, + "mean_token_accuracy": 0.8093126803636551, + "num_tokens": 13254616.0, + "step": 4110 + }, + { + "entropy": 0.7539664410986007, + "epoch": 0.9417142857142857, + "grad_norm": 7.78125, + "learning_rate": 1.2316574452730336e-06, + "loss": 1.0411237716674804, + "mean_token_accuracy": 0.8247020401060581, + "num_tokens": 13286977.0, + "step": 4120 + }, + { + "entropy": 0.7033959114924073, + "epoch": 0.944, + "grad_norm": 7.28125, + "learning_rate": 1.1835458263170556e-06, + "loss": 0.9389317512512207, + "mean_token_accuracy": 0.8293626546859741, + "num_tokens": 13320518.0, + "step": 4130 + }, + { + "entropy": 0.8039538188837468, + "epoch": 0.9462857142857143, + "grad_norm": 7.34375, + "learning_rate": 1.1354342073610777e-06, + "loss": 1.0717804908752442, + "mean_token_accuracy": 0.8100807063281537, + "num_tokens": 13349708.0, + "step": 4140 + }, + { + "entropy": 0.6700900404714047, + "epoch": 0.9485714285714286, + "grad_norm": 5.40625, + "learning_rate": 1.0873225884050998e-06, + "loss": 0.8449858665466309, + "mean_token_accuracy": 0.8380967788398266, + "num_tokens": 13382386.0, + "step": 4150 + }, + { + "entropy": 0.7454655093140901, + "epoch": 0.9508571428571428, + "grad_norm": 8.1875, + "learning_rate": 1.039210969449122e-06, + "loss": 1.0423049926757812, + "mean_token_accuracy": 0.8296139296144247, + "num_tokens": 13415633.0, + "step": 4160 + }, + { + "entropy": 0.651054497435689, + "epoch": 0.9531428571428572, + "grad_norm": 7.46875, + "learning_rate": 9.910993504931442e-07, + "loss": 0.9426624298095703, + "mean_token_accuracy": 0.8385513253509999, + "num_tokens": 13447710.0, + "step": 4170 + }, + { + "entropy": 0.8339191220700741, + "epoch": 0.9554285714285714, + "grad_norm": 9.0625, + "learning_rate": 9.429877315371664e-07, + "loss": 1.211918830871582, + "mean_token_accuracy": 0.801718657463789, + "num_tokens": 13477514.0, + "step": 4180 + }, + { + "entropy": 0.8227419966831804, + "epoch": 0.9577142857142857, + "grad_norm": 7.09375, + "learning_rate": 8.948761125811884e-07, + "loss": 1.0787554740905763, + "mean_token_accuracy": 0.8115898109972477, + "num_tokens": 13512089.0, + "step": 4190 + }, + { + "entropy": 0.8163104236125946, + "epoch": 0.96, + "grad_norm": 5.71875, + "learning_rate": 8.467644936252106e-07, + "loss": 1.0611876487731933, + "mean_token_accuracy": 0.8015380769968032, + "num_tokens": 13545584.0, + "step": 4200 + }, + { + "entropy": 0.7111412956379354, + "epoch": 0.9622857142857143, + "grad_norm": 7.40625, + "learning_rate": 7.986528746692326e-07, + "loss": 0.9729216575622559, + "mean_token_accuracy": 0.8238789007067681, + "num_tokens": 13578776.0, + "step": 4210 + }, + { + "entropy": 0.8085635328665376, + "epoch": 0.9645714285714285, + "grad_norm": 7.5, + "learning_rate": 7.505412557132548e-07, + "loss": 1.1126985549926758, + "mean_token_accuracy": 0.8126947581768036, + "num_tokens": 13612547.0, + "step": 4220 + }, + { + "entropy": 0.7815977847203612, + "epoch": 0.9668571428571429, + "grad_norm": 8.125, + "learning_rate": 7.02429636757277e-07, + "loss": 1.1313226699829102, + "mean_token_accuracy": 0.809683870524168, + "num_tokens": 13643015.0, + "step": 4230 + }, + { + "entropy": 0.7985246267169714, + "epoch": 0.9691428571428572, + "grad_norm": 9.1875, + "learning_rate": 6.54318017801299e-07, + "loss": 1.0159822463989259, + "mean_token_accuracy": 0.8185107290744782, + "num_tokens": 13675162.0, + "step": 4240 + }, + { + "entropy": 0.671059988765046, + "epoch": 0.9714285714285714, + "grad_norm": 8.0625, + "learning_rate": 6.062063988453211e-07, + "loss": 0.9259526252746582, + "mean_token_accuracy": 0.8424249842762948, + "num_tokens": 13710886.0, + "step": 4250 + }, + { + "entropy": 0.7151353804394602, + "epoch": 0.9737142857142858, + "grad_norm": 7.03125, + "learning_rate": 5.580947798893433e-07, + "loss": 0.9961103439331055, + "mean_token_accuracy": 0.8268160626292229, + "num_tokens": 13747143.0, + "step": 4260 + }, + { + "entropy": 0.825507890433073, + "epoch": 0.976, + "grad_norm": 8.4375, + "learning_rate": 5.099831609333655e-07, + "loss": 1.116166591644287, + "mean_token_accuracy": 0.8088098622858524, + "num_tokens": 13779619.0, + "step": 4270 + }, + { + "entropy": 0.8729933000169694, + "epoch": 0.9782857142857143, + "grad_norm": 7.5625, + "learning_rate": 4.618715419773876e-07, + "loss": 1.1322175025939942, + "mean_token_accuracy": 0.7934217296540738, + "num_tokens": 13812123.0, + "step": 4280 + }, + { + "entropy": 0.739137639477849, + "epoch": 0.9805714285714285, + "grad_norm": 8.4375, + "learning_rate": 4.1375992302140966e-07, + "loss": 1.0004518508911133, + "mean_token_accuracy": 0.8218909092247486, + "num_tokens": 13844075.0, + "step": 4290 + }, + { + "entropy": 0.8571365299634636, + "epoch": 0.9828571428571429, + "grad_norm": 8.375, + "learning_rate": 3.6564830406543185e-07, + "loss": 1.1517833709716796, + "mean_token_accuracy": 0.7975596848875284, + "num_tokens": 13874945.0, + "step": 4300 + }, + { + "entropy": 0.8440510330721736, + "epoch": 0.9851428571428571, + "grad_norm": 8.5, + "learning_rate": 3.17536685109454e-07, + "loss": 1.165059471130371, + "mean_token_accuracy": 0.8028948895633221, + "num_tokens": 13906888.0, + "step": 4310 + }, + { + "entropy": 0.839153022505343, + "epoch": 0.9874285714285714, + "grad_norm": 7.5, + "learning_rate": 2.694250661534761e-07, + "loss": 1.0769323348999023, + "mean_token_accuracy": 0.801658408343792, + "num_tokens": 13942830.0, + "step": 4320 + }, + { + "entropy": 0.7648129913024604, + "epoch": 0.9897142857142858, + "grad_norm": 11.9375, + "learning_rate": 2.2131344719749822e-07, + "loss": 1.0284333229064941, + "mean_token_accuracy": 0.8126491412520409, + "num_tokens": 13975429.0, + "step": 4330 + }, + { + "entropy": 0.7054416437633335, + "epoch": 0.992, + "grad_norm": 6.625, + "learning_rate": 1.7320182824152033e-07, + "loss": 0.9879807472229004, + "mean_token_accuracy": 0.834271340072155, + "num_tokens": 14007911.0, + "step": 4340 + }, + { + "entropy": 0.5742446383461356, + "epoch": 0.9942857142857143, + "grad_norm": 6.8125, + "learning_rate": 1.2509020928554246e-07, + "loss": 0.8198482513427734, + "mean_token_accuracy": 0.8546571791172027, + "num_tokens": 14041386.0, + "step": 4350 + }, + { + "entropy": 0.8128698419779539, + "epoch": 0.9965714285714286, + "grad_norm": 6.78125, + "learning_rate": 7.69785903295646e-08, + "loss": 1.0970548629760741, + "mean_token_accuracy": 0.8060387209057808, + "num_tokens": 14074136.0, + "step": 4360 + }, + { + "entropy": 0.7844365012831986, + "epoch": 0.9988571428571429, + "grad_norm": 8.0625, + "learning_rate": 2.8866971373586724e-08, + "loss": 1.0246350288391113, + "mean_token_accuracy": 0.8180203422904014, + "num_tokens": 14106425.0, + "step": 4370 + } + ], + "logging_steps": 10, + "max_steps": 4375, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.602238427340718e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-4375/training_args.bin b/checkpoint-4375/training_args.bin new file mode 100644 index 0000000..84e2775 --- /dev/null +++ b/checkpoint-4375/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5e2b2e8c6e07744b1d87692f3c2af332994a4383033db65bccda3b4b680c932 +size 5713 diff --git a/checkpoint-6241/chat_template.jinja b/checkpoint-6241/chat_template.jinja new file mode 100644 index 0000000..a716f8d --- /dev/null +++ b/checkpoint-6241/chat_template.jinja @@ -0,0 +1,90 @@ +{%- set enable_thinking = true %} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-6241/config.json b/checkpoint-6241/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-6241/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-6241/generation_config.json b/checkpoint-6241/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-6241/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-6241/model.safetensors b/checkpoint-6241/model.safetensors new file mode 100644 index 0000000..8218ae0 --- /dev/null +++ b/checkpoint-6241/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a885f6fb30aa9108bad9504efa18c42083eda34e8e6a67ab7ce3050064d5991e +size 3441185608 diff --git a/checkpoint-6241/optimizer.pt b/checkpoint-6241/optimizer.pt new file mode 100644 index 0000000..47df435 --- /dev/null +++ b/checkpoint-6241/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a05ff0dd33e49c5e7edef5a46b7151c98b2f9651c4cff837a5249beb521ef7a +size 6882572207 diff --git a/checkpoint-6241/rng_state.pth b/checkpoint-6241/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-6241/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-6241/scheduler.pt b/checkpoint-6241/scheduler.pt new file mode 100644 index 0000000..b6725ac --- /dev/null +++ b/checkpoint-6241/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69400fb00e257cbb2bcb2579c47242b849709f5d7f87ce6206c9572949f19027 +size 1465 diff --git a/checkpoint-6241/tokenizer.json b/checkpoint-6241/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-6241/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-6241/tokenizer_config.json b/checkpoint-6241/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/checkpoint-6241/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-6241/trainer_state.json b/checkpoint-6241/trainer_state.json new file mode 100644 index 0000000..3e4754e --- /dev/null +++ b/checkpoint-6241/trainer_state.json @@ -0,0 +1,6274 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 6241, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8924221977591515, + "epoch": 0.0016025320005608862, + "grad_norm": 47.75, + "learning_rate": 2.884615384615385e-07, + "loss": 2.6188112258911134, + "mean_token_accuracy": 0.5233313746750354, + "num_tokens": 66651.0, + "step": 10 + }, + { + "entropy": 1.922156248986721, + "epoch": 0.0032050640011217725, + "grad_norm": 46.75, + "learning_rate": 6.08974358974359e-07, + "loss": 2.563404655456543, + "mean_token_accuracy": 0.5188336454331874, + "num_tokens": 137940.0, + "step": 20 + }, + { + "entropy": 1.9010927319526671, + "epoch": 0.004807596001682658, + "grad_norm": 38.0, + "learning_rate": 9.294871794871796e-07, + "loss": 2.5269372940063475, + "mean_token_accuracy": 0.5221607111394405, + "num_tokens": 208682.0, + "step": 30 + }, + { + "entropy": 1.8908767253160477, + "epoch": 0.006410128002243545, + "grad_norm": 34.5, + "learning_rate": 1.25e-06, + "loss": 2.4589189529418944, + "mean_token_accuracy": 0.5292409997433424, + "num_tokens": 277911.0, + "step": 40 + }, + { + "entropy": 1.9242432832717895, + "epoch": 0.008012660002804432, + "grad_norm": 21.25, + "learning_rate": 1.5705128205128206e-06, + "loss": 2.40079402923584, + "mean_token_accuracy": 0.5290395379066467, + "num_tokens": 348454.0, + "step": 50 + }, + { + "entropy": 1.9205047577619552, + "epoch": 0.009615192003365317, + "grad_norm": 15.1875, + "learning_rate": 1.891025641025641e-06, + "loss": 2.2803781509399412, + "mean_token_accuracy": 0.5345041077584028, + "num_tokens": 417679.0, + "step": 60 + }, + { + "entropy": 1.8985449492931366, + "epoch": 0.011217724003926203, + "grad_norm": 11.4375, + "learning_rate": 2.211538461538462e-06, + "loss": 2.1747177124023436, + "mean_token_accuracy": 0.5503359518945217, + "num_tokens": 484356.0, + "step": 70 + }, + { + "entropy": 1.902299603074789, + "epoch": 0.01282025600448709, + "grad_norm": 5.65625, + "learning_rate": 2.5320512820512823e-06, + "loss": 2.114341163635254, + "mean_token_accuracy": 0.5579096399247646, + "num_tokens": 553094.0, + "step": 80 + }, + { + "entropy": 1.939731851965189, + "epoch": 0.014422788005047977, + "grad_norm": 4.875, + "learning_rate": 2.852564102564103e-06, + "loss": 2.1059520721435545, + "mean_token_accuracy": 0.5593277081847191, + "num_tokens": 622016.0, + "step": 90 + }, + { + "entropy": 1.876069898158312, + "epoch": 0.016025320005608863, + "grad_norm": 3.875, + "learning_rate": 3.1730769230769233e-06, + "loss": 2.016912269592285, + "mean_token_accuracy": 0.5697586502879858, + "num_tokens": 691175.0, + "step": 100 + }, + { + "entropy": 1.8731758743524551, + "epoch": 0.017627852006169748, + "grad_norm": 3.484375, + "learning_rate": 3.4935897435897438e-06, + "loss": 1.9954225540161132, + "mean_token_accuracy": 0.5712465867400169, + "num_tokens": 757429.0, + "step": 110 + }, + { + "entropy": 1.8756513863801956, + "epoch": 0.019230384006730633, + "grad_norm": 3.046875, + "learning_rate": 3.8141025641025643e-06, + "loss": 2.0083831787109374, + "mean_token_accuracy": 0.5699114728718996, + "num_tokens": 826035.0, + "step": 120 + }, + { + "entropy": 1.8890732847154141, + "epoch": 0.02083291600729152, + "grad_norm": 3.15625, + "learning_rate": 4.134615384615385e-06, + "loss": 2.042840766906738, + "mean_token_accuracy": 0.5750202693045139, + "num_tokens": 894823.0, + "step": 130 + }, + { + "entropy": 1.9485681891441344, + "epoch": 0.022435448007852406, + "grad_norm": 2.9375, + "learning_rate": 4.455128205128206e-06, + "loss": 2.033138656616211, + "mean_token_accuracy": 0.5648796543478966, + "num_tokens": 967782.0, + "step": 140 + }, + { + "entropy": 1.88450263813138, + "epoch": 0.02403798000841329, + "grad_norm": 3.28125, + "learning_rate": 4.775641025641027e-06, + "loss": 1.9986297607421875, + "mean_token_accuracy": 0.5769022148102522, + "num_tokens": 1035926.0, + "step": 150 + }, + { + "entropy": 1.9446559742093086, + "epoch": 0.02564051200897418, + "grad_norm": 3.203125, + "learning_rate": 5.096153846153846e-06, + "loss": 2.0234989166259765, + "mean_token_accuracy": 0.5664361469447613, + "num_tokens": 1107073.0, + "step": 160 + }, + { + "entropy": 1.883225505053997, + "epoch": 0.027243044009535065, + "grad_norm": 3.046875, + "learning_rate": 5.416666666666667e-06, + "loss": 1.9557977676391602, + "mean_token_accuracy": 0.5763147968798876, + "num_tokens": 1176735.0, + "step": 170 + }, + { + "entropy": 1.9516266271471978, + "epoch": 0.028845576010095953, + "grad_norm": 2.921875, + "learning_rate": 5.737179487179487e-06, + "loss": 1.9794040679931642, + "mean_token_accuracy": 0.5643045194447041, + "num_tokens": 1249349.0, + "step": 180 + }, + { + "entropy": 1.9053485810756683, + "epoch": 0.030448108010656838, + "grad_norm": 3.046875, + "learning_rate": 6.057692307692308e-06, + "loss": 1.9960391998291016, + "mean_token_accuracy": 0.5730757340788841, + "num_tokens": 1317928.0, + "step": 190 + }, + { + "entropy": 1.8806858271360398, + "epoch": 0.032050640011217726, + "grad_norm": 2.703125, + "learning_rate": 6.378205128205129e-06, + "loss": 1.957114028930664, + "mean_token_accuracy": 0.5820898830890655, + "num_tokens": 1385138.0, + "step": 200 + }, + { + "entropy": 1.887659776210785, + "epoch": 0.03365317201177861, + "grad_norm": 2.828125, + "learning_rate": 6.698717948717949e-06, + "loss": 1.9600671768188476, + "mean_token_accuracy": 0.5803815096616745, + "num_tokens": 1454703.0, + "step": 210 + }, + { + "entropy": 1.867220865190029, + "epoch": 0.035255704012339496, + "grad_norm": 2.796875, + "learning_rate": 7.01923076923077e-06, + "loss": 1.937087631225586, + "mean_token_accuracy": 0.579859958216548, + "num_tokens": 1520990.0, + "step": 220 + }, + { + "entropy": 1.9226368859410286, + "epoch": 0.036858236012900385, + "grad_norm": 2.8125, + "learning_rate": 7.33974358974359e-06, + "loss": 1.9694780349731444, + "mean_token_accuracy": 0.5732789397239685, + "num_tokens": 1591028.0, + "step": 230 + }, + { + "entropy": 1.840953428298235, + "epoch": 0.038460768013461266, + "grad_norm": 2.859375, + "learning_rate": 7.660256410256411e-06, + "loss": 1.9297588348388672, + "mean_token_accuracy": 0.5849139843136072, + "num_tokens": 1656272.0, + "step": 240 + }, + { + "entropy": 1.852292013913393, + "epoch": 0.040063300014022155, + "grad_norm": 3.140625, + "learning_rate": 7.980769230769232e-06, + "loss": 1.932406234741211, + "mean_token_accuracy": 0.5815942466259003, + "num_tokens": 1724103.0, + "step": 250 + }, + { + "entropy": 1.9237169787287711, + "epoch": 0.04166583201458304, + "grad_norm": 2.84375, + "learning_rate": 8.301282051282052e-06, + "loss": 1.9780784606933595, + "mean_token_accuracy": 0.5724721662700176, + "num_tokens": 1793809.0, + "step": 260 + }, + { + "entropy": 1.9092090293765067, + "epoch": 0.043268364015143924, + "grad_norm": 2.75, + "learning_rate": 8.621794871794873e-06, + "loss": 1.9296005249023438, + "mean_token_accuracy": 0.5794690005481243, + "num_tokens": 1862546.0, + "step": 270 + }, + { + "entropy": 1.8140447169542313, + "epoch": 0.04487089601570481, + "grad_norm": 2.6875, + "learning_rate": 8.942307692307693e-06, + "loss": 1.9155027389526367, + "mean_token_accuracy": 0.58444296233356, + "num_tokens": 1928555.0, + "step": 280 + }, + { + "entropy": 1.8786520659923553, + "epoch": 0.0464734280162657, + "grad_norm": 3.046875, + "learning_rate": 9.262820512820514e-06, + "loss": 1.9598024368286133, + "mean_token_accuracy": 0.5811394732445478, + "num_tokens": 1997148.0, + "step": 290 + }, + { + "entropy": 1.8962267510592938, + "epoch": 0.04807596001682658, + "grad_norm": 2.796875, + "learning_rate": 9.583333333333335e-06, + "loss": 1.9487903594970704, + "mean_token_accuracy": 0.5800880625844002, + "num_tokens": 2065063.0, + "step": 300 + }, + { + "entropy": 1.8655135102570057, + "epoch": 0.04967849201738747, + "grad_norm": 2.90625, + "learning_rate": 9.903846153846155e-06, + "loss": 1.9372148513793945, + "mean_token_accuracy": 0.5815446004271507, + "num_tokens": 2132637.0, + "step": 310 + }, + { + "entropy": 1.89379173964262, + "epoch": 0.05128102401794836, + "grad_norm": 3.0625, + "learning_rate": 9.988193624557261e-06, + "loss": 1.9456815719604492, + "mean_token_accuracy": 0.5783611986786127, + "num_tokens": 2200340.0, + "step": 320 + }, + { + "entropy": 1.851027710735798, + "epoch": 0.05288355601850925, + "grad_norm": 2.953125, + "learning_rate": 9.971327373924778e-06, + "loss": 1.9404556274414062, + "mean_token_accuracy": 0.5844585198909045, + "num_tokens": 2268721.0, + "step": 330 + }, + { + "entropy": 1.9101853892207146, + "epoch": 0.05448608801907013, + "grad_norm": 2.984375, + "learning_rate": 9.954461123292292e-06, + "loss": 1.9825191497802734, + "mean_token_accuracy": 0.5763120006769895, + "num_tokens": 2337381.0, + "step": 340 + }, + { + "entropy": 1.8936392977833747, + "epoch": 0.05608862001963102, + "grad_norm": 3.21875, + "learning_rate": 9.937594872659809e-06, + "loss": 1.9490802764892579, + "mean_token_accuracy": 0.5789993833750486, + "num_tokens": 2405770.0, + "step": 350 + }, + { + "entropy": 1.8573398821055889, + "epoch": 0.057691152020191906, + "grad_norm": 2.875, + "learning_rate": 9.920728622027325e-06, + "loss": 1.9417463302612306, + "mean_token_accuracy": 0.5814387623220683, + "num_tokens": 2473102.0, + "step": 360 + }, + { + "entropy": 1.8687433026731015, + "epoch": 0.05929368402075279, + "grad_norm": 2.9375, + "learning_rate": 9.90386237139484e-06, + "loss": 1.9335454940795898, + "mean_token_accuracy": 0.5852543283253908, + "num_tokens": 2540880.0, + "step": 370 + }, + { + "entropy": 1.8238108970224858, + "epoch": 0.060896216021313676, + "grad_norm": 3.609375, + "learning_rate": 9.886996120762356e-06, + "loss": 1.9002412796020507, + "mean_token_accuracy": 0.5933637998998165, + "num_tokens": 2605971.0, + "step": 380 + }, + { + "entropy": 1.8098725199699401, + "epoch": 0.062498748021874564, + "grad_norm": 2.984375, + "learning_rate": 9.87012987012987e-06, + "loss": 1.8953941345214844, + "mean_token_accuracy": 0.5905886631458998, + "num_tokens": 2669950.0, + "step": 390 + }, + { + "entropy": 1.8483323730528354, + "epoch": 0.06410128002243545, + "grad_norm": 2.859375, + "learning_rate": 9.853263619497387e-06, + "loss": 1.9442218780517577, + "mean_token_accuracy": 0.5866008169949055, + "num_tokens": 2735420.0, + "step": 400 + }, + { + "entropy": 1.9439873054623604, + "epoch": 0.06570381202299633, + "grad_norm": 2.828125, + "learning_rate": 9.836397368864901e-06, + "loss": 2.0032651901245115, + "mean_token_accuracy": 0.5694192741066217, + "num_tokens": 2804603.0, + "step": 410 + }, + { + "entropy": 1.872369658946991, + "epoch": 0.06730634402355722, + "grad_norm": 2.625, + "learning_rate": 9.819531118232418e-06, + "loss": 1.9437833786010743, + "mean_token_accuracy": 0.5829788766801357, + "num_tokens": 2872064.0, + "step": 420 + }, + { + "entropy": 1.886989878118038, + "epoch": 0.0689088760241181, + "grad_norm": 2.90625, + "learning_rate": 9.802664867599934e-06, + "loss": 1.9446407318115235, + "mean_token_accuracy": 0.5787219580262899, + "num_tokens": 2940272.0, + "step": 430 + }, + { + "entropy": 1.9421475365757943, + "epoch": 0.07051140802467899, + "grad_norm": 2.640625, + "learning_rate": 9.785798616967448e-06, + "loss": 2.000010871887207, + "mean_token_accuracy": 0.567750496044755, + "num_tokens": 3010674.0, + "step": 440 + }, + { + "entropy": 1.881090347468853, + "epoch": 0.07211394002523988, + "grad_norm": 2.953125, + "learning_rate": 9.768932366334965e-06, + "loss": 1.9491590499877929, + "mean_token_accuracy": 0.5847097650170326, + "num_tokens": 3078595.0, + "step": 450 + }, + { + "entropy": 1.870892095565796, + "epoch": 0.07371647202580077, + "grad_norm": 2.609375, + "learning_rate": 9.75206611570248e-06, + "loss": 1.9273197174072265, + "mean_token_accuracy": 0.5815099891275167, + "num_tokens": 3144455.0, + "step": 460 + }, + { + "entropy": 1.8761539243161678, + "epoch": 0.07531900402636166, + "grad_norm": 2.875, + "learning_rate": 9.735199865069996e-06, + "loss": 1.9557538986206056, + "mean_token_accuracy": 0.5829537663608789, + "num_tokens": 3214310.0, + "step": 470 + }, + { + "entropy": 1.855475628376007, + "epoch": 0.07692153602692253, + "grad_norm": 2.953125, + "learning_rate": 9.718333614437512e-06, + "loss": 1.9161636352539062, + "mean_token_accuracy": 0.5863808520138264, + "num_tokens": 3285062.0, + "step": 480 + }, + { + "entropy": 1.8617834381759166, + "epoch": 0.07852406802748342, + "grad_norm": 2.84375, + "learning_rate": 9.701467363805027e-06, + "loss": 1.9468860626220703, + "mean_token_accuracy": 0.580999382585287, + "num_tokens": 3353498.0, + "step": 490 + }, + { + "entropy": 1.8763719350099564, + "epoch": 0.08012660002804431, + "grad_norm": 3.359375, + "learning_rate": 9.684601113172543e-06, + "loss": 1.9362581253051758, + "mean_token_accuracy": 0.5832551945000887, + "num_tokens": 3422255.0, + "step": 500 + }, + { + "entropy": 1.8736688300967217, + "epoch": 0.0817291320286052, + "grad_norm": 3.140625, + "learning_rate": 9.667734862540057e-06, + "loss": 1.9376916885375977, + "mean_token_accuracy": 0.5802027761936188, + "num_tokens": 3491741.0, + "step": 510 + }, + { + "entropy": 1.8863457903265952, + "epoch": 0.08333166402916609, + "grad_norm": 2.859375, + "learning_rate": 9.650868611907574e-06, + "loss": 1.942121696472168, + "mean_token_accuracy": 0.5779230989515781, + "num_tokens": 3560075.0, + "step": 520 + }, + { + "entropy": 1.8802876263856887, + "epoch": 0.08493419602972697, + "grad_norm": 2.859375, + "learning_rate": 9.634002361275088e-06, + "loss": 1.9324022293090821, + "mean_token_accuracy": 0.5816225569695235, + "num_tokens": 3627966.0, + "step": 530 + }, + { + "entropy": 1.9352840840816499, + "epoch": 0.08653672803028785, + "grad_norm": 2.828125, + "learning_rate": 9.617136110642605e-06, + "loss": 1.990936279296875, + "mean_token_accuracy": 0.5724680010229349, + "num_tokens": 3700177.0, + "step": 540 + }, + { + "entropy": 1.8702165111899376, + "epoch": 0.08813926003084874, + "grad_norm": 3.5, + "learning_rate": 9.600269860010121e-06, + "loss": 1.9451469421386718, + "mean_token_accuracy": 0.5830163966864348, + "num_tokens": 3768640.0, + "step": 550 + }, + { + "entropy": 1.938204861432314, + "epoch": 0.08974179203140963, + "grad_norm": 2.828125, + "learning_rate": 9.583403609377636e-06, + "loss": 1.9839460372924804, + "mean_token_accuracy": 0.5680067148059607, + "num_tokens": 3841684.0, + "step": 560 + }, + { + "entropy": 1.891503432393074, + "epoch": 0.09134432403197051, + "grad_norm": 3.0625, + "learning_rate": 9.566537358745152e-06, + "loss": 1.9441808700561523, + "mean_token_accuracy": 0.576492153853178, + "num_tokens": 3911490.0, + "step": 570 + }, + { + "entropy": 1.8403802424669267, + "epoch": 0.0929468560325314, + "grad_norm": 3.109375, + "learning_rate": 9.549671108112666e-06, + "loss": 1.8948766708374023, + "mean_token_accuracy": 0.5904972556978464, + "num_tokens": 3975499.0, + "step": 580 + }, + { + "entropy": 1.8935207590460776, + "epoch": 0.09454938803309229, + "grad_norm": 2.671875, + "learning_rate": 9.532804857480183e-06, + "loss": 1.9514276504516601, + "mean_token_accuracy": 0.5731811784207821, + "num_tokens": 4046256.0, + "step": 590 + }, + { + "entropy": 1.9148340128362178, + "epoch": 0.09615192003365317, + "grad_norm": 2.625, + "learning_rate": 9.515938606847699e-06, + "loss": 1.9665313720703126, + "mean_token_accuracy": 0.5750534620136023, + "num_tokens": 4113801.0, + "step": 600 + }, + { + "entropy": 1.9208699256181716, + "epoch": 0.09775445203421405, + "grad_norm": 2.734375, + "learning_rate": 9.499072356215214e-06, + "loss": 1.9486780166625977, + "mean_token_accuracy": 0.5765438359230757, + "num_tokens": 4184745.0, + "step": 610 + }, + { + "entropy": 1.8517391234636307, + "epoch": 0.09935698403477494, + "grad_norm": 2.875, + "learning_rate": 9.48220610558273e-06, + "loss": 1.9088247299194336, + "mean_token_accuracy": 0.5816758945584297, + "num_tokens": 4253087.0, + "step": 620 + }, + { + "entropy": 1.8575052984058857, + "epoch": 0.10095951603533583, + "grad_norm": 3.0, + "learning_rate": 9.465339854950245e-06, + "loss": 1.937776756286621, + "mean_token_accuracy": 0.5789704568684101, + "num_tokens": 4323264.0, + "step": 630 + }, + { + "entropy": 1.8845683336257935, + "epoch": 0.10256204803589672, + "grad_norm": 2.875, + "learning_rate": 9.44847360431776e-06, + "loss": 1.919915008544922, + "mean_token_accuracy": 0.5774315193295478, + "num_tokens": 4394313.0, + "step": 640 + }, + { + "entropy": 1.935458205640316, + "epoch": 0.10416458003645761, + "grad_norm": 2.9375, + "learning_rate": 9.431607353685277e-06, + "loss": 1.9815744400024413, + "mean_token_accuracy": 0.5703229490667582, + "num_tokens": 4465024.0, + "step": 650 + }, + { + "entropy": 1.8699700206518173, + "epoch": 0.1057671120370185, + "grad_norm": 2.84375, + "learning_rate": 9.414741103052792e-06, + "loss": 1.9298105239868164, + "mean_token_accuracy": 0.5819987580180168, + "num_tokens": 4532056.0, + "step": 660 + }, + { + "entropy": 1.9036673031747342, + "epoch": 0.10736964403757937, + "grad_norm": 2.71875, + "learning_rate": 9.397874852420308e-06, + "loss": 1.9606338500976563, + "mean_token_accuracy": 0.5763357635587454, + "num_tokens": 4603061.0, + "step": 670 + }, + { + "entropy": 1.9236982017755508, + "epoch": 0.10897217603814026, + "grad_norm": 2.875, + "learning_rate": 9.381008601787823e-06, + "loss": 1.9731718063354493, + "mean_token_accuracy": 0.572434800863266, + "num_tokens": 4672934.0, + "step": 680 + }, + { + "entropy": 1.8915662527084351, + "epoch": 0.11057470803870115, + "grad_norm": 2.96875, + "learning_rate": 9.364142351155339e-06, + "loss": 1.944301223754883, + "mean_token_accuracy": 0.5777950916439295, + "num_tokens": 4742347.0, + "step": 690 + }, + { + "entropy": 1.8608257569372655, + "epoch": 0.11217724003926204, + "grad_norm": 2.9375, + "learning_rate": 9.347276100522854e-06, + "loss": 1.9187082290649413, + "mean_token_accuracy": 0.5854316674172878, + "num_tokens": 4810740.0, + "step": 700 + }, + { + "entropy": 1.91349808126688, + "epoch": 0.11377977203982292, + "grad_norm": 2.8125, + "learning_rate": 9.33040984989037e-06, + "loss": 1.9557458877563476, + "mean_token_accuracy": 0.5716337092220783, + "num_tokens": 4880950.0, + "step": 710 + }, + { + "entropy": 1.880832690000534, + "epoch": 0.11538230404038381, + "grad_norm": 2.765625, + "learning_rate": 9.313543599257886e-06, + "loss": 1.9095714569091797, + "mean_token_accuracy": 0.5788472425192595, + "num_tokens": 4951319.0, + "step": 720 + }, + { + "entropy": 1.8958022996783257, + "epoch": 0.11698483604094469, + "grad_norm": 2.875, + "learning_rate": 9.2966773486254e-06, + "loss": 1.9588214874267578, + "mean_token_accuracy": 0.5734254062175751, + "num_tokens": 5021246.0, + "step": 730 + }, + { + "entropy": 1.8561491340398788, + "epoch": 0.11858736804150558, + "grad_norm": 3.015625, + "learning_rate": 9.279811097992917e-06, + "loss": 1.9234004974365235, + "mean_token_accuracy": 0.5828992377966642, + "num_tokens": 5090404.0, + "step": 740 + }, + { + "entropy": 1.8721059411764145, + "epoch": 0.12018990004206646, + "grad_norm": 2.890625, + "learning_rate": 9.262944847360432e-06, + "loss": 1.9204557418823243, + "mean_token_accuracy": 0.5793162997812032, + "num_tokens": 5161693.0, + "step": 750 + }, + { + "entropy": 1.8398379385471344, + "epoch": 0.12179243204262735, + "grad_norm": 2.984375, + "learning_rate": 9.246078596727948e-06, + "loss": 1.9257741928100587, + "mean_token_accuracy": 0.5886994216591119, + "num_tokens": 5229347.0, + "step": 760 + }, + { + "entropy": 1.8623432606458663, + "epoch": 0.12339496404318824, + "grad_norm": 2.671875, + "learning_rate": 9.229212346095464e-06, + "loss": 1.9316108703613282, + "mean_token_accuracy": 0.5846644170582295, + "num_tokens": 5295808.0, + "step": 770 + }, + { + "entropy": 1.9330397233366967, + "epoch": 0.12499749604374913, + "grad_norm": 2.734375, + "learning_rate": 9.212346095462979e-06, + "loss": 2.002008628845215, + "mean_token_accuracy": 0.572395284473896, + "num_tokens": 5366468.0, + "step": 780 + }, + { + "entropy": 1.9373800560832024, + "epoch": 0.12660002804431, + "grad_norm": 2.90625, + "learning_rate": 9.195479844830495e-06, + "loss": 1.9589813232421875, + "mean_token_accuracy": 0.5738775081932544, + "num_tokens": 5439525.0, + "step": 790 + }, + { + "entropy": 1.7744886934757234, + "epoch": 0.1282025600448709, + "grad_norm": 2.96875, + "learning_rate": 9.17861359419801e-06, + "loss": 1.887784194946289, + "mean_token_accuracy": 0.5975198157131671, + "num_tokens": 5503717.0, + "step": 800 + }, + { + "entropy": 1.8438941523432733, + "epoch": 0.12980509204543178, + "grad_norm": 3.0625, + "learning_rate": 9.161747343565526e-06, + "loss": 1.8990644454956054, + "mean_token_accuracy": 0.5871534280478954, + "num_tokens": 5571371.0, + "step": 810 + }, + { + "entropy": 1.8703908935189246, + "epoch": 0.13140762404599265, + "grad_norm": 2.796875, + "learning_rate": 9.144881092933042e-06, + "loss": 1.8979808807373046, + "mean_token_accuracy": 0.5822482641786337, + "num_tokens": 5639191.0, + "step": 820 + }, + { + "entropy": 1.8991154111921786, + "epoch": 0.13301015604655356, + "grad_norm": 2.75, + "learning_rate": 9.128014842300557e-06, + "loss": 1.9707365036010742, + "mean_token_accuracy": 0.5750925965607167, + "num_tokens": 5710612.0, + "step": 830 + }, + { + "entropy": 1.8629533924162387, + "epoch": 0.13461268804711443, + "grad_norm": 3.078125, + "learning_rate": 9.111148591668073e-06, + "loss": 1.9318613052368163, + "mean_token_accuracy": 0.5863455723971128, + "num_tokens": 5778215.0, + "step": 840 + }, + { + "entropy": 1.8538881994783878, + "epoch": 0.13621522004767533, + "grad_norm": 2.71875, + "learning_rate": 9.094282341035588e-06, + "loss": 1.9297039031982421, + "mean_token_accuracy": 0.5876705326139927, + "num_tokens": 5844726.0, + "step": 850 + }, + { + "entropy": 1.8883967280387879, + "epoch": 0.1378177520482362, + "grad_norm": 3.015625, + "learning_rate": 9.077416090403104e-06, + "loss": 1.9285591125488282, + "mean_token_accuracy": 0.581361586973071, + "num_tokens": 5911163.0, + "step": 860 + }, + { + "entropy": 1.9187811121344567, + "epoch": 0.1394202840487971, + "grad_norm": 2.8125, + "learning_rate": 9.060549839770619e-06, + "loss": 1.9877191543579102, + "mean_token_accuracy": 0.5746748749166727, + "num_tokens": 5983080.0, + "step": 870 + }, + { + "entropy": 1.8208863891661167, + "epoch": 0.14102281604935799, + "grad_norm": 2.921875, + "learning_rate": 9.043683589138137e-06, + "loss": 1.9087957382202148, + "mean_token_accuracy": 0.5921023800969124, + "num_tokens": 6048505.0, + "step": 880 + }, + { + "entropy": 1.881580389291048, + "epoch": 0.14262534804991886, + "grad_norm": 2.703125, + "learning_rate": 9.026817338505651e-06, + "loss": 1.9579143524169922, + "mean_token_accuracy": 0.5844397276639939, + "num_tokens": 6117893.0, + "step": 890 + }, + { + "entropy": 1.8939740881323814, + "epoch": 0.14422788005047976, + "grad_norm": 2.640625, + "learning_rate": 9.009951087873166e-06, + "loss": 1.9614486694335938, + "mean_token_accuracy": 0.5774324163794518, + "num_tokens": 6186567.0, + "step": 900 + }, + { + "entropy": 1.9027862668037414, + "epoch": 0.14583041205104064, + "grad_norm": 2.96875, + "learning_rate": 8.993084837240682e-06, + "loss": 1.9565067291259766, + "mean_token_accuracy": 0.5787767164409161, + "num_tokens": 6255217.0, + "step": 910 + }, + { + "entropy": 1.8023764699697495, + "epoch": 0.14743294405160154, + "grad_norm": 3.546875, + "learning_rate": 8.976218586608197e-06, + "loss": 1.8847948074340821, + "mean_token_accuracy": 0.591859758272767, + "num_tokens": 6322616.0, + "step": 920 + }, + { + "entropy": 1.8418046914041042, + "epoch": 0.1490354760521624, + "grad_norm": 2.796875, + "learning_rate": 8.959352335975713e-06, + "loss": 1.9159833908081054, + "mean_token_accuracy": 0.5902243491262198, + "num_tokens": 6388938.0, + "step": 930 + }, + { + "entropy": 1.8638654142618178, + "epoch": 0.15063800805272332, + "grad_norm": 2.90625, + "learning_rate": 8.94248608534323e-06, + "loss": 1.9090038299560548, + "mean_token_accuracy": 0.5827893916517496, + "num_tokens": 6457677.0, + "step": 940 + }, + { + "entropy": 1.8738030433654784, + "epoch": 0.1522405400532842, + "grad_norm": 2.78125, + "learning_rate": 8.925619834710744e-06, + "loss": 1.9343568801879882, + "mean_token_accuracy": 0.5813350416719913, + "num_tokens": 6525311.0, + "step": 950 + }, + { + "entropy": 1.9377205684781074, + "epoch": 0.15384307205384506, + "grad_norm": 2.765625, + "learning_rate": 8.90875358407826e-06, + "loss": 1.9584550857543945, + "mean_token_accuracy": 0.5714080754667521, + "num_tokens": 6597450.0, + "step": 960 + }, + { + "entropy": 1.8600458979606629, + "epoch": 0.15544560405440597, + "grad_norm": 2.75, + "learning_rate": 8.891887333445775e-06, + "loss": 1.9290195465087892, + "mean_token_accuracy": 0.5868209339678288, + "num_tokens": 6663402.0, + "step": 970 + }, + { + "entropy": 1.9053997635841369, + "epoch": 0.15704813605496684, + "grad_norm": 3.0, + "learning_rate": 8.875021082813291e-06, + "loss": 1.9616933822631837, + "mean_token_accuracy": 0.5766319941729308, + "num_tokens": 6731993.0, + "step": 980 + }, + { + "entropy": 1.8353831797838212, + "epoch": 0.15865066805552774, + "grad_norm": 2.859375, + "learning_rate": 8.858154832180806e-06, + "loss": 1.8858226776123046, + "mean_token_accuracy": 0.5917053803801536, + "num_tokens": 6797163.0, + "step": 990 + }, + { + "entropy": 1.8678475245833397, + "epoch": 0.16025320005608862, + "grad_norm": 2.8125, + "learning_rate": 8.841288581548324e-06, + "loss": 1.9242422103881835, + "mean_token_accuracy": 0.5828753683716059, + "num_tokens": 6865343.0, + "step": 1000 + }, + { + "entropy": 1.8927664116024971, + "epoch": 0.1618557320566495, + "grad_norm": 2.890625, + "learning_rate": 8.824422330915838e-06, + "loss": 1.9455862045288086, + "mean_token_accuracy": 0.5737755268812179, + "num_tokens": 6936619.0, + "step": 1010 + }, + { + "entropy": 1.898373506963253, + "epoch": 0.1634582640572104, + "grad_norm": 2.734375, + "learning_rate": 8.807556080283353e-06, + "loss": 1.945164108276367, + "mean_token_accuracy": 0.5796486139297485, + "num_tokens": 7007010.0, + "step": 1020 + }, + { + "entropy": 1.9343734487891198, + "epoch": 0.16506079605777127, + "grad_norm": 3.03125, + "learning_rate": 8.79068982965087e-06, + "loss": 1.9623563766479493, + "mean_token_accuracy": 0.5699514087289572, + "num_tokens": 7078618.0, + "step": 1030 + }, + { + "entropy": 1.8613235905766488, + "epoch": 0.16666332805833217, + "grad_norm": 2.96875, + "learning_rate": 8.773823579018384e-06, + "loss": 1.9177135467529296, + "mean_token_accuracy": 0.5824566155672073, + "num_tokens": 7150683.0, + "step": 1040 + }, + { + "entropy": 1.8925682470202445, + "epoch": 0.16826586005889305, + "grad_norm": 2.765625, + "learning_rate": 8.756957328385902e-06, + "loss": 1.937409019470215, + "mean_token_accuracy": 0.5796560771763325, + "num_tokens": 7219740.0, + "step": 1050 + }, + { + "entropy": 1.907762125879526, + "epoch": 0.16986839205945395, + "grad_norm": 2.65625, + "learning_rate": 8.740091077753416e-06, + "loss": 1.9718206405639649, + "mean_token_accuracy": 0.5761904675513506, + "num_tokens": 7291651.0, + "step": 1060 + }, + { + "entropy": 1.9134493544697762, + "epoch": 0.17147092406001482, + "grad_norm": 2.75, + "learning_rate": 8.723224827120931e-06, + "loss": 1.9625492095947266, + "mean_token_accuracy": 0.5768929973244667, + "num_tokens": 7361274.0, + "step": 1070 + }, + { + "entropy": 1.8504062108695507, + "epoch": 0.1730734560605757, + "grad_norm": 3.046875, + "learning_rate": 8.706358576488447e-06, + "loss": 1.9088239669799805, + "mean_token_accuracy": 0.5906189180910587, + "num_tokens": 7428468.0, + "step": 1080 + }, + { + "entropy": 1.7697254791855812, + "epoch": 0.1746759880611366, + "grad_norm": 3.15625, + "learning_rate": 8.689492325855962e-06, + "loss": 1.8501579284667968, + "mean_token_accuracy": 0.6005081288516522, + "num_tokens": 7493525.0, + "step": 1090 + }, + { + "entropy": 1.8452809199690818, + "epoch": 0.17627852006169747, + "grad_norm": 2.90625, + "learning_rate": 8.672626075223478e-06, + "loss": 1.8861875534057617, + "mean_token_accuracy": 0.5867275305092334, + "num_tokens": 7560512.0, + "step": 1100 + }, + { + "entropy": 1.8703631803393364, + "epoch": 0.17788105206225838, + "grad_norm": 2.515625, + "learning_rate": 8.655759824590995e-06, + "loss": 1.9371749877929687, + "mean_token_accuracy": 0.5801950611174107, + "num_tokens": 7628328.0, + "step": 1110 + }, + { + "entropy": 1.8785244509577752, + "epoch": 0.17948358406281925, + "grad_norm": 2.765625, + "learning_rate": 8.63889357395851e-06, + "loss": 1.9185890197753905, + "mean_token_accuracy": 0.5809588603675365, + "num_tokens": 7697279.0, + "step": 1120 + }, + { + "entropy": 1.8575244888663291, + "epoch": 0.18108611606338015, + "grad_norm": 3.296875, + "learning_rate": 8.622027323326025e-06, + "loss": 1.9352190017700195, + "mean_token_accuracy": 0.5921396233141423, + "num_tokens": 7762443.0, + "step": 1130 + }, + { + "entropy": 1.8575579948723315, + "epoch": 0.18268864806394103, + "grad_norm": 2.703125, + "learning_rate": 8.60516107269354e-06, + "loss": 1.907987403869629, + "mean_token_accuracy": 0.5868944473564625, + "num_tokens": 7828726.0, + "step": 1140 + }, + { + "entropy": 1.861622007191181, + "epoch": 0.1842911800645019, + "grad_norm": 2.984375, + "learning_rate": 8.588294822061056e-06, + "loss": 1.9466941833496094, + "mean_token_accuracy": 0.5817751791328192, + "num_tokens": 7900221.0, + "step": 1150 + }, + { + "entropy": 1.8293386019766331, + "epoch": 0.1858937120650628, + "grad_norm": 2.84375, + "learning_rate": 8.571428571428571e-06, + "loss": 1.910441017150879, + "mean_token_accuracy": 0.5914882928133011, + "num_tokens": 7966663.0, + "step": 1160 + }, + { + "entropy": 1.8395947359502316, + "epoch": 0.18749624406562368, + "grad_norm": 2.8125, + "learning_rate": 8.554562320796089e-06, + "loss": 1.8976837158203126, + "mean_token_accuracy": 0.5871709201484918, + "num_tokens": 8033258.0, + "step": 1170 + }, + { + "entropy": 1.8713413566350936, + "epoch": 0.18909877606618458, + "grad_norm": 2.734375, + "learning_rate": 8.537696070163604e-06, + "loss": 1.9238271713256836, + "mean_token_accuracy": 0.5826620697975159, + "num_tokens": 8103587.0, + "step": 1180 + }, + { + "entropy": 1.8392857059836387, + "epoch": 0.19070130806674546, + "grad_norm": 2.703125, + "learning_rate": 8.52082981953112e-06, + "loss": 1.8789087295532227, + "mean_token_accuracy": 0.5865398772060871, + "num_tokens": 8170084.0, + "step": 1190 + }, + { + "entropy": 1.7883897237479687, + "epoch": 0.19230384006730633, + "grad_norm": 3.109375, + "learning_rate": 8.503963568898634e-06, + "loss": 1.8636220932006835, + "mean_token_accuracy": 0.594975196197629, + "num_tokens": 8237006.0, + "step": 1200 + }, + { + "entropy": 1.8488180570304393, + "epoch": 0.19390637206786723, + "grad_norm": 2.546875, + "learning_rate": 8.487097318266149e-06, + "loss": 1.8888103485107421, + "mean_token_accuracy": 0.5853704828768969, + "num_tokens": 8307626.0, + "step": 1210 + }, + { + "entropy": 1.9011844545602798, + "epoch": 0.1955089040684281, + "grad_norm": 2.8125, + "learning_rate": 8.470231067633665e-06, + "loss": 1.9513782501220702, + "mean_token_accuracy": 0.5754769399762154, + "num_tokens": 8377805.0, + "step": 1220 + }, + { + "entropy": 1.8066915586590766, + "epoch": 0.197111436068989, + "grad_norm": 2.828125, + "learning_rate": 8.453364817001182e-06, + "loss": 1.8805229187011718, + "mean_token_accuracy": 0.5928327728062868, + "num_tokens": 8443983.0, + "step": 1230 + }, + { + "entropy": 1.871665959060192, + "epoch": 0.19871396806954988, + "grad_norm": 3.25, + "learning_rate": 8.436498566368698e-06, + "loss": 1.9741409301757813, + "mean_token_accuracy": 0.5794212404638529, + "num_tokens": 8513223.0, + "step": 1240 + }, + { + "entropy": 1.865108135342598, + "epoch": 0.2003165000701108, + "grad_norm": 2.71875, + "learning_rate": 8.419632315736212e-06, + "loss": 1.9343843460083008, + "mean_token_accuracy": 0.5839380633085967, + "num_tokens": 8582195.0, + "step": 1250 + }, + { + "entropy": 1.8534103907644748, + "epoch": 0.20191903207067166, + "grad_norm": 2.890625, + "learning_rate": 8.402766065103727e-06, + "loss": 1.9067815780639648, + "mean_token_accuracy": 0.5907664895057678, + "num_tokens": 8650209.0, + "step": 1260 + }, + { + "entropy": 1.838461622595787, + "epoch": 0.20352156407123254, + "grad_norm": 2.984375, + "learning_rate": 8.385899814471243e-06, + "loss": 1.9203620910644532, + "mean_token_accuracy": 0.5879400692880153, + "num_tokens": 8717155.0, + "step": 1270 + }, + { + "entropy": 1.8598333410918713, + "epoch": 0.20512409607179344, + "grad_norm": 3.5, + "learning_rate": 8.36903356383876e-06, + "loss": 1.9286680221557617, + "mean_token_accuracy": 0.5832295440137386, + "num_tokens": 8787194.0, + "step": 1280 + }, + { + "entropy": 1.8951888650655746, + "epoch": 0.2067266280723543, + "grad_norm": 2.796875, + "learning_rate": 8.352167313206276e-06, + "loss": 1.9325904846191406, + "mean_token_accuracy": 0.5815673552453517, + "num_tokens": 8855366.0, + "step": 1290 + }, + { + "entropy": 1.8783101707696914, + "epoch": 0.20832916007291521, + "grad_norm": 3.109375, + "learning_rate": 8.33530106257379e-06, + "loss": 1.9420047760009767, + "mean_token_accuracy": 0.5779575191438198, + "num_tokens": 8923477.0, + "step": 1300 + }, + { + "entropy": 1.8716580137610435, + "epoch": 0.2099316920734761, + "grad_norm": 2.78125, + "learning_rate": 8.318434811941307e-06, + "loss": 1.9265146255493164, + "mean_token_accuracy": 0.5784675717353821, + "num_tokens": 8994430.0, + "step": 1310 + }, + { + "entropy": 1.845998640358448, + "epoch": 0.211534224074037, + "grad_norm": 2.578125, + "learning_rate": 8.301568561308821e-06, + "loss": 1.8925792694091796, + "mean_token_accuracy": 0.5861931763589382, + "num_tokens": 9063282.0, + "step": 1320 + }, + { + "entropy": 1.9150121569633485, + "epoch": 0.21313675607459787, + "grad_norm": 2.828125, + "learning_rate": 8.284702310676336e-06, + "loss": 1.9562612533569337, + "mean_token_accuracy": 0.5711025543510914, + "num_tokens": 9135759.0, + "step": 1330 + }, + { + "entropy": 1.8666348904371262, + "epoch": 0.21473928807515874, + "grad_norm": 3.09375, + "learning_rate": 8.267836060043854e-06, + "loss": 1.926978302001953, + "mean_token_accuracy": 0.584542565792799, + "num_tokens": 9203135.0, + "step": 1340 + }, + { + "entropy": 1.8651721760630608, + "epoch": 0.21634182007571964, + "grad_norm": 2.90625, + "learning_rate": 8.250969809411369e-06, + "loss": 1.9122041702270507, + "mean_token_accuracy": 0.5834652718156577, + "num_tokens": 9271143.0, + "step": 1350 + }, + { + "entropy": 1.8746020525693894, + "epoch": 0.21794435207628052, + "grad_norm": 2.96875, + "learning_rate": 8.234103558778885e-06, + "loss": 1.9327991485595704, + "mean_token_accuracy": 0.5790342222899199, + "num_tokens": 9340073.0, + "step": 1360 + }, + { + "entropy": 1.8262140899896622, + "epoch": 0.21954688407684142, + "grad_norm": 3.109375, + "learning_rate": 8.2172373081464e-06, + "loss": 1.8879671096801758, + "mean_token_accuracy": 0.5903897985816002, + "num_tokens": 9404239.0, + "step": 1370 + }, + { + "entropy": 1.9080289155244827, + "epoch": 0.2211494160774023, + "grad_norm": 2.828125, + "learning_rate": 8.200371057513916e-06, + "loss": 1.9651382446289063, + "mean_token_accuracy": 0.5774695828557015, + "num_tokens": 9474625.0, + "step": 1380 + }, + { + "entropy": 1.9039118118584155, + "epoch": 0.22275194807796317, + "grad_norm": 2.75, + "learning_rate": 8.18350480688143e-06, + "loss": 1.9804235458374024, + "mean_token_accuracy": 0.5771529369056225, + "num_tokens": 9543238.0, + "step": 1390 + }, + { + "entropy": 1.923792737722397, + "epoch": 0.22435448007852407, + "grad_norm": 2.875, + "learning_rate": 8.166638556248947e-06, + "loss": 1.971097183227539, + "mean_token_accuracy": 0.5753923650830984, + "num_tokens": 9614795.0, + "step": 1400 + }, + { + "entropy": 1.900741594284773, + "epoch": 0.22595701207908495, + "grad_norm": 2.671875, + "learning_rate": 8.149772305616463e-06, + "loss": 1.9472780227661133, + "mean_token_accuracy": 0.5738981295377016, + "num_tokens": 9685473.0, + "step": 1410 + }, + { + "entropy": 1.7839085057377815, + "epoch": 0.22755954407964585, + "grad_norm": 3.109375, + "learning_rate": 8.132906054983978e-06, + "loss": 1.8761466979980468, + "mean_token_accuracy": 0.6003258183598519, + "num_tokens": 9750896.0, + "step": 1420 + }, + { + "entropy": 1.8837758690118789, + "epoch": 0.22916207608020672, + "grad_norm": 2.671875, + "learning_rate": 8.116039804351494e-06, + "loss": 1.9476690292358398, + "mean_token_accuracy": 0.5777528025209904, + "num_tokens": 9820689.0, + "step": 1430 + }, + { + "entropy": 1.8235812917351724, + "epoch": 0.23076460808076762, + "grad_norm": 2.765625, + "learning_rate": 8.099173553719009e-06, + "loss": 1.9014440536499024, + "mean_token_accuracy": 0.5898288045078516, + "num_tokens": 9889123.0, + "step": 1440 + }, + { + "entropy": 1.8353725761175155, + "epoch": 0.2323671400813285, + "grad_norm": 2.921875, + "learning_rate": 8.082307303086523e-06, + "loss": 1.9187559127807616, + "mean_token_accuracy": 0.5885863654315472, + "num_tokens": 9956951.0, + "step": 1450 + }, + { + "entropy": 1.8732848450541497, + "epoch": 0.23396967208188937, + "grad_norm": 3.046875, + "learning_rate": 8.065441052454041e-06, + "loss": 1.918014144897461, + "mean_token_accuracy": 0.582071066647768, + "num_tokens": 10026782.0, + "step": 1460 + }, + { + "entropy": 1.8279739156365395, + "epoch": 0.23557220408245028, + "grad_norm": 2.84375, + "learning_rate": 8.048574801821556e-06, + "loss": 1.8961336135864257, + "mean_token_accuracy": 0.5900806449353695, + "num_tokens": 10092330.0, + "step": 1470 + }, + { + "entropy": 1.866368069499731, + "epoch": 0.23717473608301115, + "grad_norm": 2.515625, + "learning_rate": 8.031708551189072e-06, + "loss": 1.911445426940918, + "mean_token_accuracy": 0.5842766858637333, + "num_tokens": 10159870.0, + "step": 1480 + }, + { + "entropy": 1.8581906087696551, + "epoch": 0.23877726808357205, + "grad_norm": 2.734375, + "learning_rate": 8.014842300556587e-06, + "loss": 1.9097118377685547, + "mean_token_accuracy": 0.5794223874807358, + "num_tokens": 10228867.0, + "step": 1490 + }, + { + "entropy": 1.9131202585995197, + "epoch": 0.24037980008413293, + "grad_norm": 2.875, + "learning_rate": 7.997976049924103e-06, + "loss": 1.9654075622558593, + "mean_token_accuracy": 0.5745159700512886, + "num_tokens": 10298784.0, + "step": 1500 + }, + { + "entropy": 1.8233014531433582, + "epoch": 0.24198233208469383, + "grad_norm": 3.046875, + "learning_rate": 7.98110979929162e-06, + "loss": 1.885025405883789, + "mean_token_accuracy": 0.5910658553242684, + "num_tokens": 10367846.0, + "step": 1510 + }, + { + "entropy": 1.887587697803974, + "epoch": 0.2435848640852547, + "grad_norm": 2.9375, + "learning_rate": 7.964243548659134e-06, + "loss": 1.9407854080200195, + "mean_token_accuracy": 0.5792762577533722, + "num_tokens": 10436720.0, + "step": 1520 + }, + { + "entropy": 1.8616327196359634, + "epoch": 0.24518739608581558, + "grad_norm": 2.859375, + "learning_rate": 7.94737729802665e-06, + "loss": 1.9142820358276367, + "mean_token_accuracy": 0.586741553992033, + "num_tokens": 10505441.0, + "step": 1530 + }, + { + "entropy": 1.8891624853014946, + "epoch": 0.24678992808637648, + "grad_norm": 3.140625, + "learning_rate": 7.930511047394165e-06, + "loss": 1.9602279663085938, + "mean_token_accuracy": 0.5789007391780615, + "num_tokens": 10574984.0, + "step": 1540 + }, + { + "entropy": 1.8885827243328095, + "epoch": 0.24839246008693736, + "grad_norm": 3.1875, + "learning_rate": 7.913644796761681e-06, + "loss": 1.9428186416625977, + "mean_token_accuracy": 0.5796796213835478, + "num_tokens": 10642811.0, + "step": 1550 + }, + { + "entropy": 1.9047790303826333, + "epoch": 0.24999499208749826, + "grad_norm": 3.0625, + "learning_rate": 7.896778546129196e-06, + "loss": 1.9613679885864257, + "mean_token_accuracy": 0.5768419545143842, + "num_tokens": 10710253.0, + "step": 1560 + }, + { + "entropy": 1.9039989359676839, + "epoch": 0.25159752408805913, + "grad_norm": 2.96875, + "learning_rate": 7.879912295496712e-06, + "loss": 1.9505096435546876, + "mean_token_accuracy": 0.5804939832538366, + "num_tokens": 10781429.0, + "step": 1570 + }, + { + "entropy": 1.84059092476964, + "epoch": 0.25320005608862, + "grad_norm": 2.71875, + "learning_rate": 7.863046044864228e-06, + "loss": 1.8826757431030274, + "mean_token_accuracy": 0.5863708309829235, + "num_tokens": 10850423.0, + "step": 1580 + }, + { + "entropy": 1.8954490289092063, + "epoch": 0.2548025880891809, + "grad_norm": 2.953125, + "learning_rate": 7.846179794231743e-06, + "loss": 1.9409446716308594, + "mean_token_accuracy": 0.5763602871447802, + "num_tokens": 10920772.0, + "step": 1590 + }, + { + "entropy": 1.8355828106403351, + "epoch": 0.2564051200897418, + "grad_norm": 3.3125, + "learning_rate": 7.829313543599259e-06, + "loss": 1.8861097335815429, + "mean_token_accuracy": 0.5863994915038347, + "num_tokens": 10988848.0, + "step": 1600 + }, + { + "entropy": 1.8317218028008937, + "epoch": 0.2580076520903027, + "grad_norm": 2.9375, + "learning_rate": 7.812447292966774e-06, + "loss": 1.8998790740966798, + "mean_token_accuracy": 0.5882123652845621, + "num_tokens": 11056733.0, + "step": 1610 + }, + { + "entropy": 1.8588479906320572, + "epoch": 0.25961018409086356, + "grad_norm": 2.796875, + "learning_rate": 7.79558104233429e-06, + "loss": 1.9099315643310546, + "mean_token_accuracy": 0.5861722633242608, + "num_tokens": 11124887.0, + "step": 1620 + }, + { + "entropy": 1.8629890352487564, + "epoch": 0.26121271609142444, + "grad_norm": 2.734375, + "learning_rate": 7.778714791701806e-06, + "loss": 1.912135696411133, + "mean_token_accuracy": 0.5804708641022444, + "num_tokens": 11195170.0, + "step": 1630 + }, + { + "entropy": 1.816237111389637, + "epoch": 0.2628152480919853, + "grad_norm": 3.546875, + "learning_rate": 7.761848541069321e-06, + "loss": 1.8762163162231444, + "mean_token_accuracy": 0.5928201671689749, + "num_tokens": 11264229.0, + "step": 1640 + }, + { + "entropy": 1.8452992513775826, + "epoch": 0.26441778009254624, + "grad_norm": 3.0, + "learning_rate": 7.744982290436837e-06, + "loss": 1.906076431274414, + "mean_token_accuracy": 0.5822683598846197, + "num_tokens": 11331030.0, + "step": 1650 + }, + { + "entropy": 1.8707348950207234, + "epoch": 0.2660203120931071, + "grad_norm": 2.640625, + "learning_rate": 7.728116039804352e-06, + "loss": 1.9349163055419922, + "mean_token_accuracy": 0.5821274347603321, + "num_tokens": 11400398.0, + "step": 1660 + }, + { + "entropy": 1.8705995842814445, + "epoch": 0.267622844093668, + "grad_norm": 2.96875, + "learning_rate": 7.711249789171868e-06, + "loss": 1.9275938034057618, + "mean_token_accuracy": 0.5826949592679739, + "num_tokens": 11470626.0, + "step": 1670 + }, + { + "entropy": 1.8418460339307785, + "epoch": 0.26922537609422886, + "grad_norm": 2.640625, + "learning_rate": 7.694383538539383e-06, + "loss": 1.8914478302001954, + "mean_token_accuracy": 0.586404300481081, + "num_tokens": 11538310.0, + "step": 1680 + }, + { + "entropy": 1.8754889234900474, + "epoch": 0.2708279080947898, + "grad_norm": 2.953125, + "learning_rate": 7.677517287906899e-06, + "loss": 1.913292121887207, + "mean_token_accuracy": 0.5795420207083225, + "num_tokens": 11608431.0, + "step": 1690 + }, + { + "entropy": 1.830224046856165, + "epoch": 0.27243044009535067, + "grad_norm": 2.96875, + "learning_rate": 7.660651037274415e-06, + "loss": 1.906071090698242, + "mean_token_accuracy": 0.5909698359668255, + "num_tokens": 11674388.0, + "step": 1700 + }, + { + "entropy": 1.9046216011047363, + "epoch": 0.27403297209591154, + "grad_norm": 2.65625, + "learning_rate": 7.64378478664193e-06, + "loss": 1.9777605056762695, + "mean_token_accuracy": 0.5790138378739357, + "num_tokens": 11744014.0, + "step": 1710 + }, + { + "entropy": 1.868183906376362, + "epoch": 0.2756355040964724, + "grad_norm": 2.8125, + "learning_rate": 7.626918536009445e-06, + "loss": 1.9261510848999024, + "mean_token_accuracy": 0.583817745372653, + "num_tokens": 11812668.0, + "step": 1720 + }, + { + "entropy": 1.8261702857911586, + "epoch": 0.2772380360970333, + "grad_norm": 2.9375, + "learning_rate": 7.610052285376961e-06, + "loss": 1.908711814880371, + "mean_token_accuracy": 0.592133792489767, + "num_tokens": 11879531.0, + "step": 1730 + }, + { + "entropy": 1.8486380100250244, + "epoch": 0.2788405680975942, + "grad_norm": 2.953125, + "learning_rate": 7.593186034744477e-06, + "loss": 1.8812387466430665, + "mean_token_accuracy": 0.586352600902319, + "num_tokens": 11947698.0, + "step": 1740 + }, + { + "entropy": 1.8807709991931916, + "epoch": 0.2804431000981551, + "grad_norm": 2.53125, + "learning_rate": 7.5763197841119926e-06, + "loss": 1.9278867721557618, + "mean_token_accuracy": 0.5796640980988741, + "num_tokens": 12017344.0, + "step": 1750 + }, + { + "entropy": 1.909091053903103, + "epoch": 0.28204563209871597, + "grad_norm": 2.765625, + "learning_rate": 7.559453533479508e-06, + "loss": 1.9685272216796874, + "mean_token_accuracy": 0.5725758314132691, + "num_tokens": 12088032.0, + "step": 1760 + }, + { + "entropy": 1.8329421862959863, + "epoch": 0.28364816409927684, + "grad_norm": 3.078125, + "learning_rate": 7.5425872828470234e-06, + "loss": 1.8844343185424806, + "mean_token_accuracy": 0.5897137314081192, + "num_tokens": 12155663.0, + "step": 1770 + }, + { + "entropy": 1.8978062078356743, + "epoch": 0.2852506960998377, + "grad_norm": 2.6875, + "learning_rate": 7.525721032214539e-06, + "loss": 1.9374212265014648, + "mean_token_accuracy": 0.5814625948667527, + "num_tokens": 12226721.0, + "step": 1780 + }, + { + "entropy": 1.9012648209929466, + "epoch": 0.28685322810039865, + "grad_norm": 2.6875, + "learning_rate": 7.508854781582054e-06, + "loss": 1.9738529205322266, + "mean_token_accuracy": 0.5740793786942959, + "num_tokens": 12297877.0, + "step": 1790 + }, + { + "entropy": 1.7530992463231088, + "epoch": 0.2884557601009595, + "grad_norm": 2.984375, + "learning_rate": 7.491988530949571e-06, + "loss": 1.8373693466186523, + "mean_token_accuracy": 0.6050250265747309, + "num_tokens": 12362195.0, + "step": 1800 + }, + { + "entropy": 1.8632172025740146, + "epoch": 0.2900582921015204, + "grad_norm": 2.703125, + "learning_rate": 7.475122280317086e-06, + "loss": 1.9394594192504884, + "mean_token_accuracy": 0.5822290167212486, + "num_tokens": 12430326.0, + "step": 1810 + }, + { + "entropy": 1.8754028275609016, + "epoch": 0.2916608241020813, + "grad_norm": 2.703125, + "learning_rate": 7.4582560296846015e-06, + "loss": 1.9305736541748046, + "mean_token_accuracy": 0.5800044223666191, + "num_tokens": 12499188.0, + "step": 1820 + }, + { + "entropy": 1.8500325188040734, + "epoch": 0.29326335610264215, + "grad_norm": 2.6875, + "learning_rate": 7.441389779052117e-06, + "loss": 1.8774026870727538, + "mean_token_accuracy": 0.5874969720840454, + "num_tokens": 12568303.0, + "step": 1830 + }, + { + "entropy": 1.8548648901283742, + "epoch": 0.2948658881032031, + "grad_norm": 2.796875, + "learning_rate": 7.4245235284196324e-06, + "loss": 1.9144201278686523, + "mean_token_accuracy": 0.5831612091511488, + "num_tokens": 12636247.0, + "step": 1840 + }, + { + "entropy": 1.8950597792863846, + "epoch": 0.29646842010376395, + "grad_norm": 2.984375, + "learning_rate": 7.407657277787148e-06, + "loss": 1.9446081161499023, + "mean_token_accuracy": 0.5763969462364912, + "num_tokens": 12705371.0, + "step": 1850 + }, + { + "entropy": 1.7729586772620678, + "epoch": 0.2980709521043248, + "grad_norm": 2.921875, + "learning_rate": 7.390791027154664e-06, + "loss": 1.8617670059204101, + "mean_token_accuracy": 0.6009736400097608, + "num_tokens": 12769491.0, + "step": 1860 + }, + { + "entropy": 1.8554501563310624, + "epoch": 0.2996734841048857, + "grad_norm": 2.96875, + "learning_rate": 7.37392477652218e-06, + "loss": 1.9138887405395508, + "mean_token_accuracy": 0.584252281486988, + "num_tokens": 12838699.0, + "step": 1870 + }, + { + "entropy": 1.8460036411881446, + "epoch": 0.30127601610544663, + "grad_norm": 3.3125, + "learning_rate": 7.357058525889695e-06, + "loss": 1.9024799346923829, + "mean_token_accuracy": 0.588448590785265, + "num_tokens": 12906506.0, + "step": 1880 + }, + { + "entropy": 1.8696325168013572, + "epoch": 0.3028785481060075, + "grad_norm": 3.03125, + "learning_rate": 7.3401922752572105e-06, + "loss": 1.9234735488891601, + "mean_token_accuracy": 0.5802949018776417, + "num_tokens": 12976600.0, + "step": 1890 + }, + { + "entropy": 1.8285135440528393, + "epoch": 0.3044810801065684, + "grad_norm": 2.9375, + "learning_rate": 7.323326024624726e-06, + "loss": 1.900416374206543, + "mean_token_accuracy": 0.5928496561944485, + "num_tokens": 13042794.0, + "step": 1900 + }, + { + "entropy": 1.8411840848624705, + "epoch": 0.30608361210712925, + "grad_norm": 2.65625, + "learning_rate": 7.3064597739922414e-06, + "loss": 1.9251216888427733, + "mean_token_accuracy": 0.5873684994876385, + "num_tokens": 13109346.0, + "step": 1910 + }, + { + "entropy": 1.8464361891150474, + "epoch": 0.30768614410769013, + "grad_norm": 2.6875, + "learning_rate": 7.289593523359758e-06, + "loss": 1.8957292556762695, + "mean_token_accuracy": 0.5861016932874918, + "num_tokens": 13179256.0, + "step": 1920 + }, + { + "entropy": 1.8939056366682052, + "epoch": 0.30928867610825106, + "grad_norm": 2.828125, + "learning_rate": 7.272727272727273e-06, + "loss": 1.9547037124633788, + "mean_token_accuracy": 0.5808741740882397, + "num_tokens": 13248340.0, + "step": 1930 + }, + { + "entropy": 1.8270376928150653, + "epoch": 0.31089120810881193, + "grad_norm": 2.8125, + "learning_rate": 7.255861022094789e-06, + "loss": 1.9034496307373048, + "mean_token_accuracy": 0.592013492435217, + "num_tokens": 13314229.0, + "step": 1940 + }, + { + "entropy": 1.873492280393839, + "epoch": 0.3124937401093728, + "grad_norm": 2.71875, + "learning_rate": 7.238994771462304e-06, + "loss": 1.9518108367919922, + "mean_token_accuracy": 0.5799564849585295, + "num_tokens": 13384091.0, + "step": 1950 + }, + { + "entropy": 1.8474222376942635, + "epoch": 0.3140962721099337, + "grad_norm": 2.734375, + "learning_rate": 7.2221285208298195e-06, + "loss": 1.9209667205810548, + "mean_token_accuracy": 0.5850665267556906, + "num_tokens": 13452310.0, + "step": 1960 + }, + { + "entropy": 1.881131139397621, + "epoch": 0.31569880411049456, + "grad_norm": 2.90625, + "learning_rate": 7.205262270197337e-06, + "loss": 1.904586410522461, + "mean_token_accuracy": 0.5779978223145008, + "num_tokens": 13521281.0, + "step": 1970 + }, + { + "entropy": 1.8433863066136837, + "epoch": 0.3173013361110555, + "grad_norm": 3.0, + "learning_rate": 7.188396019564851e-06, + "loss": 1.9178913116455079, + "mean_token_accuracy": 0.5882460463792085, + "num_tokens": 13590196.0, + "step": 1980 + }, + { + "entropy": 1.8311151549220086, + "epoch": 0.31890386811161636, + "grad_norm": 3.15625, + "learning_rate": 7.171529768932367e-06, + "loss": 1.9058528900146485, + "mean_token_accuracy": 0.5885777480900287, + "num_tokens": 13656542.0, + "step": 1990 + }, + { + "entropy": 1.8787899039685727, + "epoch": 0.32050640011217724, + "grad_norm": 2.921875, + "learning_rate": 7.154663518299882e-06, + "loss": 1.9221023559570312, + "mean_token_accuracy": 0.5850845731794834, + "num_tokens": 13725850.0, + "step": 2000 + }, + { + "entropy": 1.8783811405301094, + "epoch": 0.3221089321127381, + "grad_norm": 3.296875, + "learning_rate": 7.137797267667398e-06, + "loss": 1.9362503051757813, + "mean_token_accuracy": 0.5805343955755233, + "num_tokens": 13794498.0, + "step": 2010 + }, + { + "entropy": 1.8898996517062188, + "epoch": 0.323711464113299, + "grad_norm": 2.84375, + "learning_rate": 7.120931017034913e-06, + "loss": 1.9480035781860352, + "mean_token_accuracy": 0.5791302386671304, + "num_tokens": 13864359.0, + "step": 2020 + }, + { + "entropy": 1.7916542790830134, + "epoch": 0.3253139961138599, + "grad_norm": 2.953125, + "learning_rate": 7.10406476640243e-06, + "loss": 1.8486898422241211, + "mean_token_accuracy": 0.5971706237643957, + "num_tokens": 13931370.0, + "step": 2030 + }, + { + "entropy": 1.865858867764473, + "epoch": 0.3269165281144208, + "grad_norm": 2.640625, + "learning_rate": 7.087198515769945e-06, + "loss": 1.9063629150390624, + "mean_token_accuracy": 0.5809760253876448, + "num_tokens": 14000913.0, + "step": 2040 + }, + { + "entropy": 1.8851099610328674, + "epoch": 0.32851906011498166, + "grad_norm": 3.03125, + "learning_rate": 7.07033226513746e-06, + "loss": 1.938603401184082, + "mean_token_accuracy": 0.5784780897200108, + "num_tokens": 14069557.0, + "step": 2050 + }, + { + "entropy": 1.9176052078604697, + "epoch": 0.33012159211554254, + "grad_norm": 2.703125, + "learning_rate": 7.053466014504976e-06, + "loss": 1.9561405181884766, + "mean_token_accuracy": 0.5729512359946967, + "num_tokens": 14141660.0, + "step": 2060 + }, + { + "entropy": 1.8795430816709995, + "epoch": 0.33172412411610347, + "grad_norm": 2.609375, + "learning_rate": 7.036599763872491e-06, + "loss": 1.930160140991211, + "mean_token_accuracy": 0.5816898390650749, + "num_tokens": 14209466.0, + "step": 2070 + }, + { + "entropy": 1.8757113859057426, + "epoch": 0.33332665611666434, + "grad_norm": 3.4375, + "learning_rate": 7.019733513240007e-06, + "loss": 1.9418207168579102, + "mean_token_accuracy": 0.5815501570701599, + "num_tokens": 14279109.0, + "step": 2080 + }, + { + "entropy": 1.9274300292134285, + "epoch": 0.3349291881172252, + "grad_norm": 2.796875, + "learning_rate": 7.002867262607524e-06, + "loss": 1.9555702209472656, + "mean_token_accuracy": 0.5733671579509974, + "num_tokens": 14351031.0, + "step": 2090 + }, + { + "entropy": 1.8100173577666283, + "epoch": 0.3365317201177861, + "grad_norm": 3.0625, + "learning_rate": 6.986001011975039e-06, + "loss": 1.868393325805664, + "mean_token_accuracy": 0.5990545708686114, + "num_tokens": 14415016.0, + "step": 2100 + }, + { + "entropy": 1.8206559799611568, + "epoch": 0.33813425211834697, + "grad_norm": 2.75, + "learning_rate": 6.969134761342554e-06, + "loss": 1.8662199020385741, + "mean_token_accuracy": 0.5882071249186993, + "num_tokens": 14483513.0, + "step": 2110 + }, + { + "entropy": 1.8442789621651172, + "epoch": 0.3397367841189079, + "grad_norm": 2.96875, + "learning_rate": 6.952268510710069e-06, + "loss": 1.9212806701660157, + "mean_token_accuracy": 0.5897382352501154, + "num_tokens": 14550786.0, + "step": 2120 + }, + { + "entropy": 1.833724681288004, + "epoch": 0.34133931611946877, + "grad_norm": 3.140625, + "learning_rate": 6.935402260077585e-06, + "loss": 1.8911985397338866, + "mean_token_accuracy": 0.5906524024903774, + "num_tokens": 14619432.0, + "step": 2130 + }, + { + "entropy": 1.8132789947092534, + "epoch": 0.34294184812002965, + "grad_norm": 2.75, + "learning_rate": 6.9185360094451e-06, + "loss": 1.9041032791137695, + "mean_token_accuracy": 0.5969481058418751, + "num_tokens": 14684677.0, + "step": 2140 + }, + { + "entropy": 1.875623344630003, + "epoch": 0.3445443801205905, + "grad_norm": 3.09375, + "learning_rate": 6.901669758812617e-06, + "loss": 1.931635856628418, + "mean_token_accuracy": 0.5816898342221976, + "num_tokens": 14753856.0, + "step": 2150 + }, + { + "entropy": 1.9176507964730263, + "epoch": 0.3461469121211514, + "grad_norm": 3.03125, + "learning_rate": 6.884803508180133e-06, + "loss": 1.970164680480957, + "mean_token_accuracy": 0.5698140971362591, + "num_tokens": 14829029.0, + "step": 2160 + }, + { + "entropy": 1.867189484834671, + "epoch": 0.3477494441217123, + "grad_norm": 2.671875, + "learning_rate": 6.867937257547647e-06, + "loss": 1.9144952774047852, + "mean_token_accuracy": 0.5880186032503844, + "num_tokens": 14896701.0, + "step": 2170 + }, + { + "entropy": 1.8927301332354545, + "epoch": 0.3493519761222732, + "grad_norm": 2.8125, + "learning_rate": 6.851071006915163e-06, + "loss": 1.9222103118896485, + "mean_token_accuracy": 0.5790928430855274, + "num_tokens": 14969268.0, + "step": 2180 + }, + { + "entropy": 1.863588874042034, + "epoch": 0.3509545081228341, + "grad_norm": 3.046875, + "learning_rate": 6.834204756282678e-06, + "loss": 1.9339506149291992, + "mean_token_accuracy": 0.5797934107482433, + "num_tokens": 15037697.0, + "step": 2190 + }, + { + "entropy": 1.8574811123311519, + "epoch": 0.35255704012339495, + "grad_norm": 3.265625, + "learning_rate": 6.817338505650195e-06, + "loss": 1.9404556274414062, + "mean_token_accuracy": 0.5782970868051052, + "num_tokens": 15108557.0, + "step": 2200 + }, + { + "entropy": 1.8824990913271904, + "epoch": 0.3541595721239558, + "grad_norm": 2.765625, + "learning_rate": 6.800472255017711e-06, + "loss": 1.9320140838623048, + "mean_token_accuracy": 0.5829423192888499, + "num_tokens": 15177520.0, + "step": 2210 + }, + { + "entropy": 1.8936009645462035, + "epoch": 0.35576210412451675, + "grad_norm": 3.015625, + "learning_rate": 6.783606004385226e-06, + "loss": 1.9541767120361329, + "mean_token_accuracy": 0.5792703408747911, + "num_tokens": 15250167.0, + "step": 2220 + }, + { + "entropy": 1.858391472697258, + "epoch": 0.35736463612507763, + "grad_norm": 2.90625, + "learning_rate": 6.766739753752741e-06, + "loss": 1.9123594284057617, + "mean_token_accuracy": 0.5811858758330345, + "num_tokens": 15319761.0, + "step": 2230 + }, + { + "entropy": 1.825581496208906, + "epoch": 0.3589671681256385, + "grad_norm": 2.859375, + "learning_rate": 6.749873503120256e-06, + "loss": 1.8857872009277343, + "mean_token_accuracy": 0.5892566710710525, + "num_tokens": 15386569.0, + "step": 2240 + }, + { + "entropy": 1.8254388399422168, + "epoch": 0.3605697001261994, + "grad_norm": 2.828125, + "learning_rate": 6.733007252487772e-06, + "loss": 1.8876169204711915, + "mean_token_accuracy": 0.5917335107922554, + "num_tokens": 15452970.0, + "step": 2250 + }, + { + "entropy": 1.834766410291195, + "epoch": 0.3621722321267603, + "grad_norm": 2.875, + "learning_rate": 6.716141001855289e-06, + "loss": 1.8933477401733398, + "mean_token_accuracy": 0.5864639330655337, + "num_tokens": 15522607.0, + "step": 2260 + }, + { + "entropy": 1.861782355606556, + "epoch": 0.3637747641273212, + "grad_norm": 2.796875, + "learning_rate": 6.699274751222804e-06, + "loss": 1.945047378540039, + "mean_token_accuracy": 0.5846532471477985, + "num_tokens": 15592431.0, + "step": 2270 + }, + { + "entropy": 1.8507339976727963, + "epoch": 0.36537729612788206, + "grad_norm": 2.9375, + "learning_rate": 6.68240850059032e-06, + "loss": 1.8829397201538085, + "mean_token_accuracy": 0.5838924575597048, + "num_tokens": 15660961.0, + "step": 2280 + }, + { + "entropy": 1.8665653564035893, + "epoch": 0.36697982812844293, + "grad_norm": 2.875, + "learning_rate": 6.665542249957835e-06, + "loss": 1.9256706237792969, + "mean_token_accuracy": 0.5818109232932329, + "num_tokens": 15732019.0, + "step": 2290 + }, + { + "entropy": 1.8203022845089436, + "epoch": 0.3685823601290038, + "grad_norm": 2.9375, + "learning_rate": 6.64867599932535e-06, + "loss": 1.8997062683105468, + "mean_token_accuracy": 0.5923520758748054, + "num_tokens": 15799902.0, + "step": 2300 + }, + { + "entropy": 1.7739195488393307, + "epoch": 0.37018489212956474, + "grad_norm": 2.609375, + "learning_rate": 6.631809748692865e-06, + "loss": 1.8644765853881835, + "mean_token_accuracy": 0.6021988991647959, + "num_tokens": 15864613.0, + "step": 2310 + }, + { + "entropy": 1.8488979011774063, + "epoch": 0.3717874241301256, + "grad_norm": 2.78125, + "learning_rate": 6.6149434980603824e-06, + "loss": 1.8930198669433593, + "mean_token_accuracy": 0.5810744848102332, + "num_tokens": 15932591.0, + "step": 2320 + }, + { + "entropy": 1.832446900755167, + "epoch": 0.3733899561306865, + "grad_norm": 3.421875, + "learning_rate": 6.598077247427898e-06, + "loss": 1.9018989562988282, + "mean_token_accuracy": 0.5864586193114519, + "num_tokens": 16001143.0, + "step": 2330 + }, + { + "entropy": 1.825374136865139, + "epoch": 0.37499248813124736, + "grad_norm": 3.4375, + "learning_rate": 6.581210996795413e-06, + "loss": 1.8895555496215821, + "mean_token_accuracy": 0.5906005047261715, + "num_tokens": 16069332.0, + "step": 2340 + }, + { + "entropy": 1.8373838283121586, + "epoch": 0.37659502013180823, + "grad_norm": 2.921875, + "learning_rate": 6.564344746162929e-06, + "loss": 1.924220085144043, + "mean_token_accuracy": 0.5885270729660987, + "num_tokens": 16135459.0, + "step": 2350 + }, + { + "entropy": 1.863863729685545, + "epoch": 0.37819755213236916, + "grad_norm": 2.96875, + "learning_rate": 6.547478495530443e-06, + "loss": 1.9460079193115234, + "mean_token_accuracy": 0.5869121756404638, + "num_tokens": 16202998.0, + "step": 2360 + }, + { + "entropy": 1.89323253557086, + "epoch": 0.37980008413293004, + "grad_norm": 2.6875, + "learning_rate": 6.530612244897959e-06, + "loss": 1.929962158203125, + "mean_token_accuracy": 0.5782414205372334, + "num_tokens": 16271840.0, + "step": 2370 + }, + { + "entropy": 1.8196432545781136, + "epoch": 0.3814026161334909, + "grad_norm": 2.8125, + "learning_rate": 6.513745994265476e-06, + "loss": 1.861398696899414, + "mean_token_accuracy": 0.5919506680220366, + "num_tokens": 16338327.0, + "step": 2380 + }, + { + "entropy": 1.8163256019353866, + "epoch": 0.3830051481340518, + "grad_norm": 2.671875, + "learning_rate": 6.4968797436329914e-06, + "loss": 1.8722827911376954, + "mean_token_accuracy": 0.5892269656062126, + "num_tokens": 16404422.0, + "step": 2390 + }, + { + "entropy": 1.8725082725286484, + "epoch": 0.38460768013461266, + "grad_norm": 3.140625, + "learning_rate": 6.480013493000507e-06, + "loss": 1.9314796447753906, + "mean_token_accuracy": 0.5786304190754891, + "num_tokens": 16474783.0, + "step": 2400 + }, + { + "entropy": 1.780524667352438, + "epoch": 0.3862102121351736, + "grad_norm": 2.796875, + "learning_rate": 6.463147242368022e-06, + "loss": 1.8242341995239257, + "mean_token_accuracy": 0.6014976523816585, + "num_tokens": 16538442.0, + "step": 2410 + }, + { + "entropy": 1.8381457321345807, + "epoch": 0.38781274413573447, + "grad_norm": 3.109375, + "learning_rate": 6.446280991735537e-06, + "loss": 1.9019351959228517, + "mean_token_accuracy": 0.5861563563346863, + "num_tokens": 16604392.0, + "step": 2420 + }, + { + "entropy": 1.862081041932106, + "epoch": 0.38941527613629534, + "grad_norm": 2.984375, + "learning_rate": 6.429414741103054e-06, + "loss": 1.9009132385253906, + "mean_token_accuracy": 0.5874000541865826, + "num_tokens": 16672112.0, + "step": 2430 + }, + { + "entropy": 1.8587382271885873, + "epoch": 0.3910178081368562, + "grad_norm": 2.65625, + "learning_rate": 6.4125484904705695e-06, + "loss": 1.895541000366211, + "mean_token_accuracy": 0.5839630082249642, + "num_tokens": 16741056.0, + "step": 2440 + }, + { + "entropy": 1.8325805217027664, + "epoch": 0.39262034013741715, + "grad_norm": 3.109375, + "learning_rate": 6.395682239838085e-06, + "loss": 1.9115198135375977, + "mean_token_accuracy": 0.5885306518524885, + "num_tokens": 16807450.0, + "step": 2450 + }, + { + "entropy": 1.9020752415060997, + "epoch": 0.394222872137978, + "grad_norm": 3.046875, + "learning_rate": 6.3788159892056e-06, + "loss": 1.9247831344604491, + "mean_token_accuracy": 0.5807623241096735, + "num_tokens": 16874716.0, + "step": 2460 + }, + { + "entropy": 1.824447751790285, + "epoch": 0.3958254041385389, + "grad_norm": 3.40625, + "learning_rate": 6.361949738573116e-06, + "loss": 1.9011926651000977, + "mean_token_accuracy": 0.5913017760962248, + "num_tokens": 16940753.0, + "step": 2470 + }, + { + "entropy": 1.8734217897057532, + "epoch": 0.39742793613909977, + "grad_norm": 2.984375, + "learning_rate": 6.345083487940631e-06, + "loss": 1.9072031021118163, + "mean_token_accuracy": 0.5824517548084259, + "num_tokens": 17011670.0, + "step": 2480 + }, + { + "entropy": 1.882775531709194, + "epoch": 0.39903046813966064, + "grad_norm": 2.96875, + "learning_rate": 6.328217237308148e-06, + "loss": 1.9223293304443358, + "mean_token_accuracy": 0.5763808127492667, + "num_tokens": 17081373.0, + "step": 2490 + }, + { + "entropy": 1.8312932297587394, + "epoch": 0.4006330001402216, + "grad_norm": 2.96875, + "learning_rate": 6.311350986675663e-06, + "loss": 1.8834392547607421, + "mean_token_accuracy": 0.5890156481415033, + "num_tokens": 17148423.0, + "step": 2500 + }, + { + "entropy": 1.8463823966681958, + "epoch": 0.40223553214078245, + "grad_norm": 3.109375, + "learning_rate": 6.2944847360431785e-06, + "loss": 1.8855321884155274, + "mean_token_accuracy": 0.5882707864046097, + "num_tokens": 17215730.0, + "step": 2510 + }, + { + "entropy": 1.8465096279978752, + "epoch": 0.4038380641413433, + "grad_norm": 2.78125, + "learning_rate": 6.277618485410694e-06, + "loss": 1.9089530944824218, + "mean_token_accuracy": 0.5890189308673144, + "num_tokens": 17283905.0, + "step": 2520 + }, + { + "entropy": 1.789325623959303, + "epoch": 0.4054405961419042, + "grad_norm": 2.703125, + "learning_rate": 6.260752234778209e-06, + "loss": 1.8789407730102539, + "mean_token_accuracy": 0.6008992414921522, + "num_tokens": 17347380.0, + "step": 2530 + }, + { + "entropy": 1.9213061038404704, + "epoch": 0.40704312814246507, + "grad_norm": 2.640625, + "learning_rate": 6.243885984145725e-06, + "loss": 1.9735231399536133, + "mean_token_accuracy": 0.5743049286305905, + "num_tokens": 17419874.0, + "step": 2540 + }, + { + "entropy": 1.8554375410079955, + "epoch": 0.408645660143026, + "grad_norm": 2.765625, + "learning_rate": 6.227019733513241e-06, + "loss": 1.9063846588134765, + "mean_token_accuracy": 0.5845744274556637, + "num_tokens": 17489879.0, + "step": 2550 + }, + { + "entropy": 1.8458185702562333, + "epoch": 0.4102481921435869, + "grad_norm": 2.859375, + "learning_rate": 6.210153482880757e-06, + "loss": 1.8916229248046874, + "mean_token_accuracy": 0.586842879652977, + "num_tokens": 17557261.0, + "step": 2560 + }, + { + "entropy": 1.8558879360556602, + "epoch": 0.41185072414414775, + "grad_norm": 2.984375, + "learning_rate": 6.193287232248272e-06, + "loss": 1.9234607696533204, + "mean_token_accuracy": 0.5813111860305071, + "num_tokens": 17625792.0, + "step": 2570 + }, + { + "entropy": 1.8427653566002846, + "epoch": 0.4134532561447086, + "grad_norm": 3.171875, + "learning_rate": 6.1764209816157875e-06, + "loss": 1.912209701538086, + "mean_token_accuracy": 0.588098294660449, + "num_tokens": 17693209.0, + "step": 2580 + }, + { + "entropy": 1.80966489687562, + "epoch": 0.4150557881452695, + "grad_norm": 2.65625, + "learning_rate": 6.159554730983303e-06, + "loss": 1.8694408416748047, + "mean_token_accuracy": 0.5923740215599537, + "num_tokens": 17761999.0, + "step": 2590 + }, + { + "entropy": 1.8303676053881646, + "epoch": 0.41665832014583043, + "grad_norm": 2.796875, + "learning_rate": 6.142688480350818e-06, + "loss": 1.9028659820556642, + "mean_token_accuracy": 0.5906838051974773, + "num_tokens": 17830724.0, + "step": 2600 + }, + { + "entropy": 1.9149459421634674, + "epoch": 0.4182608521463913, + "grad_norm": 2.90625, + "learning_rate": 6.125822229718335e-06, + "loss": 1.96752872467041, + "mean_token_accuracy": 0.574995793774724, + "num_tokens": 17900433.0, + "step": 2610 + }, + { + "entropy": 1.808657655864954, + "epoch": 0.4198633841469522, + "grad_norm": 3.125, + "learning_rate": 6.10895597908585e-06, + "loss": 1.8747655868530273, + "mean_token_accuracy": 0.5923097655177116, + "num_tokens": 17967759.0, + "step": 2620 + }, + { + "entropy": 1.8198106482625007, + "epoch": 0.42146591614751305, + "grad_norm": 3.0, + "learning_rate": 6.092089728453366e-06, + "loss": 1.8572179794311523, + "mean_token_accuracy": 0.5907534711062908, + "num_tokens": 18033433.0, + "step": 2630 + }, + { + "entropy": 1.8586710482835769, + "epoch": 0.423068448148074, + "grad_norm": 2.59375, + "learning_rate": 6.075223477820881e-06, + "loss": 1.925465202331543, + "mean_token_accuracy": 0.5850045710802079, + "num_tokens": 18102813.0, + "step": 2640 + }, + { + "entropy": 1.8994694516062736, + "epoch": 0.42467098014863486, + "grad_norm": 2.875, + "learning_rate": 6.0583572271883965e-06, + "loss": 1.9421836853027343, + "mean_token_accuracy": 0.574908834323287, + "num_tokens": 18175905.0, + "step": 2650 + }, + { + "entropy": 1.8027544744312762, + "epoch": 0.42627351214919573, + "grad_norm": 2.96875, + "learning_rate": 6.041490976555913e-06, + "loss": 1.8755990982055664, + "mean_token_accuracy": 0.5987543936818838, + "num_tokens": 18242701.0, + "step": 2660 + }, + { + "entropy": 1.8242316350340844, + "epoch": 0.4278760441497566, + "grad_norm": 3.046875, + "learning_rate": 6.024624725923428e-06, + "loss": 1.9025865554809571, + "mean_token_accuracy": 0.5926646631211042, + "num_tokens": 18309115.0, + "step": 2670 + }, + { + "entropy": 1.8555085845291615, + "epoch": 0.4294785761503175, + "grad_norm": 2.8125, + "learning_rate": 6.007758475290944e-06, + "loss": 1.9329267501831056, + "mean_token_accuracy": 0.5813577599823475, + "num_tokens": 18380520.0, + "step": 2680 + }, + { + "entropy": 1.8312315061688422, + "epoch": 0.4310811081508784, + "grad_norm": 2.875, + "learning_rate": 5.990892224658459e-06, + "loss": 1.9179895401000977, + "mean_token_accuracy": 0.5868359692394733, + "num_tokens": 18449662.0, + "step": 2690 + }, + { + "entropy": 1.862812553346157, + "epoch": 0.4326836401514393, + "grad_norm": 2.78125, + "learning_rate": 5.9740259740259746e-06, + "loss": 1.912089729309082, + "mean_token_accuracy": 0.5835570193827152, + "num_tokens": 18517870.0, + "step": 2700 + }, + { + "entropy": 1.8445881478488446, + "epoch": 0.43428617215200016, + "grad_norm": 3.15625, + "learning_rate": 5.95715972339349e-06, + "loss": 1.9296016693115234, + "mean_token_accuracy": 0.5899791497737169, + "num_tokens": 18585466.0, + "step": 2710 + }, + { + "entropy": 1.8312980830669403, + "epoch": 0.43588870415256104, + "grad_norm": 2.90625, + "learning_rate": 5.940293472761006e-06, + "loss": 1.8721866607666016, + "mean_token_accuracy": 0.5901631616055966, + "num_tokens": 18650210.0, + "step": 2720 + }, + { + "entropy": 1.810354943573475, + "epoch": 0.4374912361531219, + "grad_norm": 3.09375, + "learning_rate": 5.923427222128522e-06, + "loss": 1.848921775817871, + "mean_token_accuracy": 0.5946807909756899, + "num_tokens": 18714978.0, + "step": 2730 + }, + { + "entropy": 1.882994955778122, + "epoch": 0.43909376815368284, + "grad_norm": 2.9375, + "learning_rate": 5.906560971496037e-06, + "loss": 1.9537870407104492, + "mean_token_accuracy": 0.5792578358203173, + "num_tokens": 18782585.0, + "step": 2740 + }, + { + "entropy": 1.881935852020979, + "epoch": 0.4406963001542437, + "grad_norm": 2.984375, + "learning_rate": 5.889694720863553e-06, + "loss": 1.927790069580078, + "mean_token_accuracy": 0.5773686055094004, + "num_tokens": 18852965.0, + "step": 2750 + }, + { + "entropy": 1.8925419092178344, + "epoch": 0.4422988321548046, + "grad_norm": 2.921875, + "learning_rate": 5.872828470231068e-06, + "loss": 1.942881202697754, + "mean_token_accuracy": 0.5796595361083746, + "num_tokens": 18921226.0, + "step": 2760 + }, + { + "entropy": 1.7830379381775856, + "epoch": 0.44390136415536546, + "grad_norm": 2.890625, + "learning_rate": 5.8559622195985836e-06, + "loss": 1.8563718795776367, + "mean_token_accuracy": 0.596837505698204, + "num_tokens": 18986799.0, + "step": 2770 + }, + { + "entropy": 1.8433029428124428, + "epoch": 0.44550389615592634, + "grad_norm": 3.0625, + "learning_rate": 5.8390959689661e-06, + "loss": 1.911284828186035, + "mean_token_accuracy": 0.5852627001702786, + "num_tokens": 19056284.0, + "step": 2780 + }, + { + "entropy": 1.9118391960859298, + "epoch": 0.44710642815648727, + "grad_norm": 2.84375, + "learning_rate": 5.822229718333615e-06, + "loss": 1.9613269805908202, + "mean_token_accuracy": 0.5728709891438484, + "num_tokens": 19127350.0, + "step": 2790 + }, + { + "entropy": 1.8576175361871718, + "epoch": 0.44870896015704814, + "grad_norm": 2.8125, + "learning_rate": 5.805363467701131e-06, + "loss": 1.891185188293457, + "mean_token_accuracy": 0.5836143758147955, + "num_tokens": 19199605.0, + "step": 2800 + }, + { + "entropy": 1.8335268676280976, + "epoch": 0.450311492157609, + "grad_norm": 2.875, + "learning_rate": 5.788497217068646e-06, + "loss": 1.8964523315429687, + "mean_token_accuracy": 0.591438103839755, + "num_tokens": 19269442.0, + "step": 2810 + }, + { + "entropy": 1.865811436623335, + "epoch": 0.4519140241581699, + "grad_norm": 2.984375, + "learning_rate": 5.771630966436162e-06, + "loss": 1.9232105255126952, + "mean_token_accuracy": 0.5796146471053362, + "num_tokens": 19338371.0, + "step": 2820 + }, + { + "entropy": 1.8866727635264398, + "epoch": 0.4535165561587308, + "grad_norm": 2.90625, + "learning_rate": 5.754764715803677e-06, + "loss": 1.9480762481689453, + "mean_token_accuracy": 0.5735677890479565, + "num_tokens": 19407321.0, + "step": 2830 + }, + { + "entropy": 1.8429814919829368, + "epoch": 0.4551190881592917, + "grad_norm": 2.78125, + "learning_rate": 5.737898465171193e-06, + "loss": 1.9041376113891602, + "mean_token_accuracy": 0.586334315687418, + "num_tokens": 19476059.0, + "step": 2840 + }, + { + "entropy": 1.857156838476658, + "epoch": 0.45672162015985257, + "grad_norm": 3.15625, + "learning_rate": 5.721032214538709e-06, + "loss": 1.915693473815918, + "mean_token_accuracy": 0.5839728847146034, + "num_tokens": 19544213.0, + "step": 2850 + }, + { + "entropy": 1.864296567440033, + "epoch": 0.45832415216041344, + "grad_norm": 2.78125, + "learning_rate": 5.704165963906224e-06, + "loss": 1.905698585510254, + "mean_token_accuracy": 0.5868626192212105, + "num_tokens": 19611371.0, + "step": 2860 + }, + { + "entropy": 1.8460254110395908, + "epoch": 0.4599266841609743, + "grad_norm": 2.953125, + "learning_rate": 5.68729971327374e-06, + "loss": 1.905874252319336, + "mean_token_accuracy": 0.584000188484788, + "num_tokens": 19680184.0, + "step": 2870 + }, + { + "entropy": 1.8271512754261494, + "epoch": 0.46152921616153525, + "grad_norm": 2.734375, + "learning_rate": 5.670433462641255e-06, + "loss": 1.9141311645507812, + "mean_token_accuracy": 0.5897762086242437, + "num_tokens": 19749348.0, + "step": 2880 + }, + { + "entropy": 1.889108157157898, + "epoch": 0.4631317481620961, + "grad_norm": 2.828125, + "learning_rate": 5.6535672120087715e-06, + "loss": 1.9251979827880858, + "mean_token_accuracy": 0.5813889559358358, + "num_tokens": 19820299.0, + "step": 2890 + }, + { + "entropy": 1.8363960474729537, + "epoch": 0.464734280162657, + "grad_norm": 2.796875, + "learning_rate": 5.636700961376287e-06, + "loss": 1.8978172302246095, + "mean_token_accuracy": 0.5845275580883026, + "num_tokens": 19887799.0, + "step": 2900 + }, + { + "entropy": 1.877197700738907, + "epoch": 0.4663368121632179, + "grad_norm": 2.921875, + "learning_rate": 5.619834710743802e-06, + "loss": 1.9254043579101563, + "mean_token_accuracy": 0.581829895451665, + "num_tokens": 19956245.0, + "step": 2910 + }, + { + "entropy": 1.845394493639469, + "epoch": 0.46793934416377875, + "grad_norm": 2.71875, + "learning_rate": 5.602968460111318e-06, + "loss": 1.9196294784545898, + "mean_token_accuracy": 0.590549175068736, + "num_tokens": 20023326.0, + "step": 2920 + }, + { + "entropy": 1.8792662344872952, + "epoch": 0.4695418761643397, + "grad_norm": 3.15625, + "learning_rate": 5.586102209478833e-06, + "loss": 1.9388086318969726, + "mean_token_accuracy": 0.5791543461382389, + "num_tokens": 20094220.0, + "step": 2930 + }, + { + "entropy": 1.8718859672546386, + "epoch": 0.47114440816490055, + "grad_norm": 2.84375, + "learning_rate": 5.569235958846349e-06, + "loss": 1.9154325485229493, + "mean_token_accuracy": 0.5803413491696119, + "num_tokens": 20163723.0, + "step": 2940 + }, + { + "entropy": 1.9110450595617294, + "epoch": 0.4727469401654614, + "grad_norm": 3.046875, + "learning_rate": 5.552369708213865e-06, + "loss": 1.9459632873535155, + "mean_token_accuracy": 0.5762019760906696, + "num_tokens": 20235101.0, + "step": 2950 + }, + { + "entropy": 1.8887835063040257, + "epoch": 0.4743494721660223, + "grad_norm": 2.78125, + "learning_rate": 5.5355034575813805e-06, + "loss": 1.9171342849731445, + "mean_token_accuracy": 0.5793264780193568, + "num_tokens": 20304564.0, + "step": 2960 + }, + { + "entropy": 1.9018400639295578, + "epoch": 0.4759520041665832, + "grad_norm": 2.8125, + "learning_rate": 5.518637206948896e-06, + "loss": 1.9300537109375, + "mean_token_accuracy": 0.5778403412550688, + "num_tokens": 20373203.0, + "step": 2970 + }, + { + "entropy": 1.7895628839731217, + "epoch": 0.4775545361671441, + "grad_norm": 2.984375, + "learning_rate": 5.501770956316411e-06, + "loss": 1.8884479522705078, + "mean_token_accuracy": 0.5980903979390859, + "num_tokens": 20437375.0, + "step": 2980 + }, + { + "entropy": 1.8900270894169808, + "epoch": 0.479157068167705, + "grad_norm": 2.984375, + "learning_rate": 5.484904705683927e-06, + "loss": 1.931725311279297, + "mean_token_accuracy": 0.57785121537745, + "num_tokens": 20504799.0, + "step": 2990 + }, + { + "entropy": 1.8562400430440902, + "epoch": 0.48075960016826585, + "grad_norm": 2.921875, + "learning_rate": 5.468038455051442e-06, + "loss": 1.9025707244873047, + "mean_token_accuracy": 0.5844077534973622, + "num_tokens": 20575140.0, + "step": 3000 + }, + { + "entropy": 1.8393971025943756, + "epoch": 0.48236213216882673, + "grad_norm": 2.984375, + "learning_rate": 5.4511722044189586e-06, + "loss": 1.8763370513916016, + "mean_token_accuracy": 0.5868513479828834, + "num_tokens": 20642816.0, + "step": 3010 + }, + { + "entropy": 1.8503380201756954, + "epoch": 0.48396466416938766, + "grad_norm": 2.953125, + "learning_rate": 5.434305953786474e-06, + "loss": 1.927678108215332, + "mean_token_accuracy": 0.5839967612177134, + "num_tokens": 20713553.0, + "step": 3020 + }, + { + "entropy": 1.8344844073057174, + "epoch": 0.48556719616994853, + "grad_norm": 2.578125, + "learning_rate": 5.4174397031539895e-06, + "loss": 1.885507583618164, + "mean_token_accuracy": 0.5878067553043366, + "num_tokens": 20783309.0, + "step": 3030 + }, + { + "entropy": 1.8160213552415372, + "epoch": 0.4871697281705094, + "grad_norm": 2.90625, + "learning_rate": 5.400573452521505e-06, + "loss": 1.8787452697753906, + "mean_token_accuracy": 0.5898300107568503, + "num_tokens": 20850485.0, + "step": 3040 + }, + { + "entropy": 1.8818404108285904, + "epoch": 0.4887722601710703, + "grad_norm": 2.875, + "learning_rate": 5.38370720188902e-06, + "loss": 1.9195350646972655, + "mean_token_accuracy": 0.5788391418755054, + "num_tokens": 20920923.0, + "step": 3050 + }, + { + "entropy": 1.8369183473289012, + "epoch": 0.49037479217163116, + "grad_norm": 2.8125, + "learning_rate": 5.366840951256536e-06, + "loss": 1.9277904510498047, + "mean_token_accuracy": 0.5910786610096693, + "num_tokens": 20989809.0, + "step": 3060 + }, + { + "entropy": 1.7957973316311837, + "epoch": 0.4919773241721921, + "grad_norm": 2.671875, + "learning_rate": 5.349974700624052e-06, + "loss": 1.828994369506836, + "mean_token_accuracy": 0.5928265832364559, + "num_tokens": 21055057.0, + "step": 3070 + }, + { + "entropy": 1.8879523053765297, + "epoch": 0.49357985617275296, + "grad_norm": 2.921875, + "learning_rate": 5.3331084499915675e-06, + "loss": 1.9327852249145507, + "mean_token_accuracy": 0.5796903323382139, + "num_tokens": 21123165.0, + "step": 3080 + }, + { + "entropy": 1.900216892361641, + "epoch": 0.49518238817331384, + "grad_norm": 2.890625, + "learning_rate": 5.316242199359083e-06, + "loss": 1.9367078781127929, + "mean_token_accuracy": 0.5748460162431002, + "num_tokens": 21192984.0, + "step": 3090 + }, + { + "entropy": 1.9121940307319165, + "epoch": 0.4967849201738747, + "grad_norm": 3.0625, + "learning_rate": 5.2993759487265984e-06, + "loss": 1.9564058303833007, + "mean_token_accuracy": 0.577494065463543, + "num_tokens": 21262418.0, + "step": 3100 + }, + { + "entropy": 1.7955867640674115, + "epoch": 0.4983874521744356, + "grad_norm": 2.765625, + "learning_rate": 5.282509698094114e-06, + "loss": 1.848398780822754, + "mean_token_accuracy": 0.5978333078324795, + "num_tokens": 21327925.0, + "step": 3110 + }, + { + "entropy": 1.8336094319820404, + "epoch": 0.4999899841749965, + "grad_norm": 2.875, + "learning_rate": 5.26564344746163e-06, + "loss": 1.8984619140625, + "mean_token_accuracy": 0.5879204269498587, + "num_tokens": 21393942.0, + "step": 3120 + }, + { + "entropy": 1.8322543017566204, + "epoch": 0.5015925161755573, + "grad_norm": 2.734375, + "learning_rate": 5.248777196829146e-06, + "loss": 1.8764625549316407, + "mean_token_accuracy": 0.5878861173987389, + "num_tokens": 21462106.0, + "step": 3130 + }, + { + "entropy": 1.7941844053566456, + "epoch": 0.5031950481761183, + "grad_norm": 3.15625, + "learning_rate": 5.231910946196661e-06, + "loss": 1.8972002029418946, + "mean_token_accuracy": 0.592497718334198, + "num_tokens": 21529516.0, + "step": 3140 + }, + { + "entropy": 1.8582370266318322, + "epoch": 0.5047975801766792, + "grad_norm": 2.609375, + "learning_rate": 5.2150446955641765e-06, + "loss": 1.9111324310302735, + "mean_token_accuracy": 0.5843363590538502, + "num_tokens": 21599252.0, + "step": 3150 + }, + { + "entropy": 1.830164335668087, + "epoch": 0.50640011217724, + "grad_norm": 2.984375, + "learning_rate": 5.198178444931692e-06, + "loss": 1.8945926666259765, + "mean_token_accuracy": 0.5922736659646034, + "num_tokens": 21665924.0, + "step": 3160 + }, + { + "entropy": 1.9132280111312867, + "epoch": 0.5080026441778009, + "grad_norm": 2.953125, + "learning_rate": 5.1813121942992074e-06, + "loss": 1.9597476959228515, + "mean_token_accuracy": 0.5744776498526335, + "num_tokens": 21739563.0, + "step": 3170 + }, + { + "entropy": 1.8639399752020835, + "epoch": 0.5096051761783618, + "grad_norm": 3.078125, + "learning_rate": 5.164445943666724e-06, + "loss": 1.920318031311035, + "mean_token_accuracy": 0.5860882043838501, + "num_tokens": 21809001.0, + "step": 3180 + }, + { + "entropy": 1.8148264288902283, + "epoch": 0.5112077081789227, + "grad_norm": 2.859375, + "learning_rate": 5.147579693034239e-06, + "loss": 1.8659805297851562, + "mean_token_accuracy": 0.5945463448762893, + "num_tokens": 21875311.0, + "step": 3190 + }, + { + "entropy": 1.7990412935614586, + "epoch": 0.5128102401794836, + "grad_norm": 2.90625, + "learning_rate": 5.130713442401755e-06, + "loss": 1.865718460083008, + "mean_token_accuracy": 0.5927905708551406, + "num_tokens": 21941984.0, + "step": 3200 + }, + { + "entropy": 1.859583093225956, + "epoch": 0.5144127721800444, + "grad_norm": 3.140625, + "learning_rate": 5.11384719176927e-06, + "loss": 1.9168724060058593, + "mean_token_accuracy": 0.5831061247736216, + "num_tokens": 22010341.0, + "step": 3210 + }, + { + "entropy": 1.8411869630217552, + "epoch": 0.5160153041806054, + "grad_norm": 3.09375, + "learning_rate": 5.0969809411367855e-06, + "loss": 1.9032838821411133, + "mean_token_accuracy": 0.5862697619944811, + "num_tokens": 22078255.0, + "step": 3220 + }, + { + "entropy": 1.8429126217961311, + "epoch": 0.5176178361811662, + "grad_norm": 2.6875, + "learning_rate": 5.080114690504301e-06, + "loss": 1.8853023529052735, + "mean_token_accuracy": 0.5895553410053254, + "num_tokens": 22145488.0, + "step": 3230 + }, + { + "entropy": 1.885491119325161, + "epoch": 0.5192203681817271, + "grad_norm": 3.109375, + "learning_rate": 5.063248439871817e-06, + "loss": 1.9154699325561524, + "mean_token_accuracy": 0.5776723183691501, + "num_tokens": 22215282.0, + "step": 3240 + }, + { + "entropy": 1.8708842188119887, + "epoch": 0.520822900182288, + "grad_norm": 2.90625, + "learning_rate": 5.046382189239333e-06, + "loss": 1.9112281799316406, + "mean_token_accuracy": 0.5818539954721927, + "num_tokens": 22282862.0, + "step": 3250 + }, + { + "entropy": 1.8231175325810909, + "epoch": 0.5224254321828489, + "grad_norm": 2.53125, + "learning_rate": 5.029515938606848e-06, + "loss": 1.8751091003417968, + "mean_token_accuracy": 0.5905714184045792, + "num_tokens": 22350290.0, + "step": 3260 + }, + { + "entropy": 1.8113183729350566, + "epoch": 0.5240279641834098, + "grad_norm": 3.046875, + "learning_rate": 5.012649687974364e-06, + "loss": 1.897282600402832, + "mean_token_accuracy": 0.5939208552241325, + "num_tokens": 22417435.0, + "step": 3270 + }, + { + "entropy": 1.8155881494283677, + "epoch": 0.5256304961839706, + "grad_norm": 2.734375, + "learning_rate": 4.99578343734188e-06, + "loss": 1.8791780471801758, + "mean_token_accuracy": 0.5893838539719581, + "num_tokens": 22486402.0, + "step": 3280 + }, + { + "entropy": 1.903946729004383, + "epoch": 0.5272330281845315, + "grad_norm": 2.703125, + "learning_rate": 4.978917186709395e-06, + "loss": 1.9404390335083008, + "mean_token_accuracy": 0.5766887858510017, + "num_tokens": 22556511.0, + "step": 3290 + }, + { + "entropy": 1.8609099626541137, + "epoch": 0.5288355601850925, + "grad_norm": 2.59375, + "learning_rate": 4.96205093607691e-06, + "loss": 1.9064706802368163, + "mean_token_accuracy": 0.5823613096028566, + "num_tokens": 22625934.0, + "step": 3300 + }, + { + "entropy": 1.8036007642745973, + "epoch": 0.5304380921856533, + "grad_norm": 2.90625, + "learning_rate": 4.945184685444426e-06, + "loss": 1.8651018142700195, + "mean_token_accuracy": 0.5906497817486525, + "num_tokens": 22695222.0, + "step": 3310 + }, + { + "entropy": 1.8944967277348042, + "epoch": 0.5320406241862142, + "grad_norm": 2.828125, + "learning_rate": 4.928318434811942e-06, + "loss": 1.9579627990722657, + "mean_token_accuracy": 0.5770516272634267, + "num_tokens": 22764978.0, + "step": 3320 + }, + { + "entropy": 1.827626084536314, + "epoch": 0.5336431561867752, + "grad_norm": 2.890625, + "learning_rate": 4.911452184179457e-06, + "loss": 1.8864225387573241, + "mean_token_accuracy": 0.5910983499139547, + "num_tokens": 22831462.0, + "step": 3330 + }, + { + "entropy": 1.8530403889715672, + "epoch": 0.535245688187336, + "grad_norm": 3.09375, + "learning_rate": 4.8945859335469734e-06, + "loss": 1.893623161315918, + "mean_token_accuracy": 0.5863074962049722, + "num_tokens": 22902702.0, + "step": 3340 + }, + { + "entropy": 1.8771760039031506, + "epoch": 0.5368482201878969, + "grad_norm": 3.15625, + "learning_rate": 4.877719682914489e-06, + "loss": 1.9317413330078126, + "mean_token_accuracy": 0.5844787560403347, + "num_tokens": 22972838.0, + "step": 3350 + }, + { + "entropy": 1.8093865230679511, + "epoch": 0.5384507521884577, + "grad_norm": 3.125, + "learning_rate": 4.8608534322820035e-06, + "loss": 1.8382366180419922, + "mean_token_accuracy": 0.5946755729615688, + "num_tokens": 23039852.0, + "step": 3360 + }, + { + "entropy": 1.8227481991052628, + "epoch": 0.5400532841890187, + "grad_norm": 2.71875, + "learning_rate": 4.84398718164952e-06, + "loss": 1.8795154571533204, + "mean_token_accuracy": 0.5877833373844623, + "num_tokens": 23109196.0, + "step": 3370 + }, + { + "entropy": 1.7931410901248455, + "epoch": 0.5416558161895796, + "grad_norm": 3.15625, + "learning_rate": 4.827120931017035e-06, + "loss": 1.8591276168823243, + "mean_token_accuracy": 0.5954682864248753, + "num_tokens": 23174720.0, + "step": 3380 + }, + { + "entropy": 1.810715451091528, + "epoch": 0.5432583481901404, + "grad_norm": 2.8125, + "learning_rate": 4.810254680384551e-06, + "loss": 1.884342575073242, + "mean_token_accuracy": 0.5896463751792907, + "num_tokens": 23241017.0, + "step": 3390 + }, + { + "entropy": 1.8730248123407365, + "epoch": 0.5448608801907013, + "grad_norm": 2.765625, + "learning_rate": 4.793388429752067e-06, + "loss": 1.9179431915283203, + "mean_token_accuracy": 0.5850767388939857, + "num_tokens": 23309999.0, + "step": 3400 + }, + { + "entropy": 1.8814474999904633, + "epoch": 0.5464634121912622, + "grad_norm": 2.796875, + "learning_rate": 4.7765221791195824e-06, + "loss": 1.9308740615844726, + "mean_token_accuracy": 0.5797040991485118, + "num_tokens": 23380827.0, + "step": 3410 + }, + { + "entropy": 1.8731833666563034, + "epoch": 0.5480659441918231, + "grad_norm": 2.703125, + "learning_rate": 4.759655928487097e-06, + "loss": 1.937031364440918, + "mean_token_accuracy": 0.5782224997878075, + "num_tokens": 23450401.0, + "step": 3420 + }, + { + "entropy": 1.8374334543943405, + "epoch": 0.549668476192384, + "grad_norm": 2.765625, + "learning_rate": 4.742789677854613e-06, + "loss": 1.8725309371948242, + "mean_token_accuracy": 0.58929248675704, + "num_tokens": 23517968.0, + "step": 3430 + }, + { + "entropy": 1.888993863761425, + "epoch": 0.5512710081929448, + "grad_norm": 3.0, + "learning_rate": 4.725923427222129e-06, + "loss": 1.9277978897094727, + "mean_token_accuracy": 0.5754599675536156, + "num_tokens": 23589020.0, + "step": 3440 + }, + { + "entropy": 1.8315568715333939, + "epoch": 0.5528735401935058, + "grad_norm": 2.734375, + "learning_rate": 4.709057176589644e-06, + "loss": 1.8970975875854492, + "mean_token_accuracy": 0.5894832197576761, + "num_tokens": 23655586.0, + "step": 3450 + }, + { + "entropy": 1.8086498454213142, + "epoch": 0.5544760721940666, + "grad_norm": 3.171875, + "learning_rate": 4.6921909259571605e-06, + "loss": 1.8923715591430663, + "mean_token_accuracy": 0.5938830282539129, + "num_tokens": 23721568.0, + "step": 3460 + }, + { + "entropy": 1.8702815189957618, + "epoch": 0.5560786041946275, + "grad_norm": 2.734375, + "learning_rate": 4.675324675324676e-06, + "loss": 1.912058448791504, + "mean_token_accuracy": 0.5766608223319054, + "num_tokens": 23792474.0, + "step": 3470 + }, + { + "entropy": 1.8467231094837189, + "epoch": 0.5576811361951884, + "grad_norm": 2.765625, + "learning_rate": 4.658458424692191e-06, + "loss": 1.9086189270019531, + "mean_token_accuracy": 0.5887116767466068, + "num_tokens": 23857874.0, + "step": 3480 + }, + { + "entropy": 1.8486532710492611, + "epoch": 0.5592836681957493, + "grad_norm": 2.90625, + "learning_rate": 4.641592174059707e-06, + "loss": 1.8958675384521484, + "mean_token_accuracy": 0.5825589634478092, + "num_tokens": 23927414.0, + "step": 3490 + }, + { + "entropy": 1.8610327310860157, + "epoch": 0.5608862001963102, + "grad_norm": 2.90625, + "learning_rate": 4.624725923427222e-06, + "loss": 1.9029279708862306, + "mean_token_accuracy": 0.5831367287784814, + "num_tokens": 23995081.0, + "step": 3500 + }, + { + "entropy": 1.8732392929494381, + "epoch": 0.562488732196871, + "grad_norm": 2.796875, + "learning_rate": 4.607859672794739e-06, + "loss": 1.9352720260620118, + "mean_token_accuracy": 0.5802403304725885, + "num_tokens": 24064995.0, + "step": 3510 + }, + { + "entropy": 1.8747488550841809, + "epoch": 0.5640912641974319, + "grad_norm": 2.9375, + "learning_rate": 4.590993422162254e-06, + "loss": 1.9249692916870118, + "mean_token_accuracy": 0.5802885986864567, + "num_tokens": 24137888.0, + "step": 3520 + }, + { + "entropy": 1.8377225518226623, + "epoch": 0.5656937961979929, + "grad_norm": 2.90625, + "learning_rate": 4.5741271715297695e-06, + "loss": 1.8629825592041016, + "mean_token_accuracy": 0.587927145510912, + "num_tokens": 24205350.0, + "step": 3530 + }, + { + "entropy": 1.862581817060709, + "epoch": 0.5672963281985537, + "grad_norm": 2.96875, + "learning_rate": 4.557260920897285e-06, + "loss": 1.933199119567871, + "mean_token_accuracy": 0.5822917714715004, + "num_tokens": 24274637.0, + "step": 3540 + }, + { + "entropy": 1.8414349660277367, + "epoch": 0.5688988601991146, + "grad_norm": 2.953125, + "learning_rate": 4.5403946702648e-06, + "loss": 1.8869396209716798, + "mean_token_accuracy": 0.5826878268271685, + "num_tokens": 24343142.0, + "step": 3550 + }, + { + "entropy": 1.8475224629044533, + "epoch": 0.5705013921996754, + "grad_norm": 2.8125, + "learning_rate": 4.523528419632316e-06, + "loss": 1.9176437377929687, + "mean_token_accuracy": 0.5875023037195206, + "num_tokens": 24409182.0, + "step": 3560 + }, + { + "entropy": 1.8767376720905304, + "epoch": 0.5721039242002364, + "grad_norm": 2.953125, + "learning_rate": 4.506662168999832e-06, + "loss": 1.9157390594482422, + "mean_token_accuracy": 0.5804454453289509, + "num_tokens": 24478294.0, + "step": 3570 + }, + { + "entropy": 1.8539368592202663, + "epoch": 0.5737064562007973, + "grad_norm": 3.0625, + "learning_rate": 4.489795918367348e-06, + "loss": 1.9065256118774414, + "mean_token_accuracy": 0.5849509820342064, + "num_tokens": 24547908.0, + "step": 3580 + }, + { + "entropy": 1.8835485830903054, + "epoch": 0.5753089882013581, + "grad_norm": 2.890625, + "learning_rate": 4.472929667734863e-06, + "loss": 1.9412755966186523, + "mean_token_accuracy": 0.5793763112276793, + "num_tokens": 24618029.0, + "step": 3590 + }, + { + "entropy": 1.869025533646345, + "epoch": 0.576911520201919, + "grad_norm": 3.0, + "learning_rate": 4.4560634171023785e-06, + "loss": 1.913435935974121, + "mean_token_accuracy": 0.5836379799991847, + "num_tokens": 24686681.0, + "step": 3600 + }, + { + "entropy": 1.832342056185007, + "epoch": 0.5785140522024799, + "grad_norm": 2.8125, + "learning_rate": 4.439197166469894e-06, + "loss": 1.8887754440307618, + "mean_token_accuracy": 0.5894854720681906, + "num_tokens": 24754343.0, + "step": 3610 + }, + { + "entropy": 1.8395591847598554, + "epoch": 0.5801165842030408, + "grad_norm": 2.9375, + "learning_rate": 4.422330915837409e-06, + "loss": 1.9002063751220704, + "mean_token_accuracy": 0.5841349400579929, + "num_tokens": 24822232.0, + "step": 3620 + }, + { + "entropy": 1.8473046600818634, + "epoch": 0.5817191162036017, + "grad_norm": 2.765625, + "learning_rate": 4.405464665204926e-06, + "loss": 1.9122318267822265, + "mean_token_accuracy": 0.5860261067748069, + "num_tokens": 24891946.0, + "step": 3630 + }, + { + "entropy": 1.8386360064148903, + "epoch": 0.5833216482041625, + "grad_norm": 2.734375, + "learning_rate": 4.388598414572441e-06, + "loss": 1.8892175674438476, + "mean_token_accuracy": 0.5868268880993128, + "num_tokens": 24957974.0, + "step": 3640 + }, + { + "entropy": 1.8272941276431083, + "epoch": 0.5849241802047235, + "grad_norm": 2.90625, + "learning_rate": 4.371732163939957e-06, + "loss": 1.8862226486206055, + "mean_token_accuracy": 0.5928803119808436, + "num_tokens": 25024594.0, + "step": 3650 + }, + { + "entropy": 1.8759762145578862, + "epoch": 0.5865267122052843, + "grad_norm": 2.859375, + "learning_rate": 4.354865913307472e-06, + "loss": 1.931888961791992, + "mean_token_accuracy": 0.5769379314035177, + "num_tokens": 25094620.0, + "step": 3660 + }, + { + "entropy": 1.8491356074810028, + "epoch": 0.5881292442058452, + "grad_norm": 3.046875, + "learning_rate": 4.3379996626749875e-06, + "loss": 1.914997673034668, + "mean_token_accuracy": 0.5848333861678838, + "num_tokens": 25164112.0, + "step": 3670 + }, + { + "entropy": 1.8535669289529324, + "epoch": 0.5897317762064062, + "grad_norm": 2.921875, + "learning_rate": 4.321133412042503e-06, + "loss": 1.9279930114746093, + "mean_token_accuracy": 0.5833786878734827, + "num_tokens": 25233582.0, + "step": 3680 + }, + { + "entropy": 1.8587990984320641, + "epoch": 0.591334308206967, + "grad_norm": 2.796875, + "learning_rate": 4.304267161410019e-06, + "loss": 1.9063568115234375, + "mean_token_accuracy": 0.580756638199091, + "num_tokens": 25304298.0, + "step": 3690 + }, + { + "entropy": 1.847524681687355, + "epoch": 0.5929368402075279, + "grad_norm": 2.6875, + "learning_rate": 4.287400910777535e-06, + "loss": 1.8817052841186523, + "mean_token_accuracy": 0.5853028532117606, + "num_tokens": 25371793.0, + "step": 3700 + }, + { + "entropy": 1.7811777122318744, + "epoch": 0.5945393722080888, + "grad_norm": 2.78125, + "learning_rate": 4.27053466014505e-06, + "loss": 1.8650287628173827, + "mean_token_accuracy": 0.5995365470647812, + "num_tokens": 25437476.0, + "step": 3710 + }, + { + "entropy": 1.8484894543886186, + "epoch": 0.5961419042086497, + "grad_norm": 2.796875, + "learning_rate": 4.2536684095125656e-06, + "loss": 1.8962841033935547, + "mean_token_accuracy": 0.584038731828332, + "num_tokens": 25505490.0, + "step": 3720 + }, + { + "entropy": 1.8471211820840836, + "epoch": 0.5977444362092106, + "grad_norm": 2.96875, + "learning_rate": 4.236802158880081e-06, + "loss": 1.8902151107788085, + "mean_token_accuracy": 0.5868056703358888, + "num_tokens": 25573449.0, + "step": 3730 + }, + { + "entropy": 1.8298789858818054, + "epoch": 0.5993469682097714, + "grad_norm": 2.625, + "learning_rate": 4.219935908247597e-06, + "loss": 1.8739568710327148, + "mean_token_accuracy": 0.5878440275788307, + "num_tokens": 25640814.0, + "step": 3740 + }, + { + "entropy": 1.7966152891516685, + "epoch": 0.6009495002103323, + "grad_norm": 2.671875, + "learning_rate": 4.203069657615113e-06, + "loss": 1.8355796813964844, + "mean_token_accuracy": 0.5947290062904358, + "num_tokens": 25707760.0, + "step": 3750 + }, + { + "entropy": 1.920249554514885, + "epoch": 0.6025520322108933, + "grad_norm": 3.015625, + "learning_rate": 4.186203406982628e-06, + "loss": 1.9803874969482422, + "mean_token_accuracy": 0.571630297228694, + "num_tokens": 25778390.0, + "step": 3760 + }, + { + "entropy": 1.8823009133338928, + "epoch": 0.6041545642114541, + "grad_norm": 2.96875, + "learning_rate": 4.169337156350144e-06, + "loss": 1.9324773788452148, + "mean_token_accuracy": 0.5821881555020809, + "num_tokens": 25850137.0, + "step": 3770 + }, + { + "entropy": 1.8832687310874463, + "epoch": 0.605757096212015, + "grad_norm": 2.78125, + "learning_rate": 4.152470905717659e-06, + "loss": 1.925374984741211, + "mean_token_accuracy": 0.5806475289165973, + "num_tokens": 25920517.0, + "step": 3780 + }, + { + "entropy": 1.8682250641286373, + "epoch": 0.6073596282125758, + "grad_norm": 2.78125, + "learning_rate": 4.1356046550851746e-06, + "loss": 1.9021371841430663, + "mean_token_accuracy": 0.5809845846146345, + "num_tokens": 25990445.0, + "step": 3790 + }, + { + "entropy": 1.8415281191468238, + "epoch": 0.6089621602131368, + "grad_norm": 2.96875, + "learning_rate": 4.118738404452691e-06, + "loss": 1.8859903335571289, + "mean_token_accuracy": 0.5898528724908829, + "num_tokens": 26054835.0, + "step": 3800 + }, + { + "entropy": 1.8924433827400207, + "epoch": 0.6105646922136977, + "grad_norm": 3.0, + "learning_rate": 4.101872153820206e-06, + "loss": 1.9476310729980468, + "mean_token_accuracy": 0.5753111571073533, + "num_tokens": 26124843.0, + "step": 3810 + }, + { + "entropy": 1.8384124413132668, + "epoch": 0.6121672242142585, + "grad_norm": 3.015625, + "learning_rate": 4.085005903187722e-06, + "loss": 1.8944419860839843, + "mean_token_accuracy": 0.5857999432832003, + "num_tokens": 26192380.0, + "step": 3820 + }, + { + "entropy": 1.9009744957089425, + "epoch": 0.6137697562148194, + "grad_norm": 2.703125, + "learning_rate": 4.068139652555237e-06, + "loss": 1.9390430450439453, + "mean_token_accuracy": 0.5778307665139437, + "num_tokens": 26263193.0, + "step": 3830 + }, + { + "entropy": 1.8429256439208985, + "epoch": 0.6153722882153803, + "grad_norm": 3.0625, + "learning_rate": 4.051273401922753e-06, + "loss": 1.9250511169433593, + "mean_token_accuracy": 0.5913165871053934, + "num_tokens": 26328954.0, + "step": 3840 + }, + { + "entropy": 1.8886337257921695, + "epoch": 0.6169748202159412, + "grad_norm": 2.84375, + "learning_rate": 4.034407151290268e-06, + "loss": 1.9294986724853516, + "mean_token_accuracy": 0.5790377296507359, + "num_tokens": 26398415.0, + "step": 3850 + }, + { + "entropy": 1.8385604590177536, + "epoch": 0.6185773522165021, + "grad_norm": 2.671875, + "learning_rate": 4.017540900657784e-06, + "loss": 1.8972116470336915, + "mean_token_accuracy": 0.5878618601709604, + "num_tokens": 26467637.0, + "step": 3860 + }, + { + "entropy": 1.9100745365023613, + "epoch": 0.6201798842170629, + "grad_norm": 2.875, + "learning_rate": 4.0006746500253e-06, + "loss": 1.981870460510254, + "mean_token_accuracy": 0.5758707467466593, + "num_tokens": 26535877.0, + "step": 3870 + }, + { + "entropy": 1.9046914495527745, + "epoch": 0.6217824162176239, + "grad_norm": 2.765625, + "learning_rate": 3.983808399392815e-06, + "loss": 1.9622364044189453, + "mean_token_accuracy": 0.5798295836895704, + "num_tokens": 26607844.0, + "step": 3880 + }, + { + "entropy": 1.8779021188616754, + "epoch": 0.6233849482181847, + "grad_norm": 2.78125, + "learning_rate": 3.966942148760331e-06, + "loss": 1.9223901748657226, + "mean_token_accuracy": 0.5800982560962439, + "num_tokens": 26677111.0, + "step": 3890 + }, + { + "entropy": 1.8721623986959457, + "epoch": 0.6249874802187456, + "grad_norm": 3.078125, + "learning_rate": 3.950075898127846e-06, + "loss": 1.9060981750488282, + "mean_token_accuracy": 0.5845723442733288, + "num_tokens": 26745167.0, + "step": 3900 + }, + { + "entropy": 1.8842680297791958, + "epoch": 0.6265900122193065, + "grad_norm": 3.03125, + "learning_rate": 3.933209647495362e-06, + "loss": 1.9447505950927735, + "mean_token_accuracy": 0.5788533575832844, + "num_tokens": 26814737.0, + "step": 3910 + }, + { + "entropy": 1.8723237678408622, + "epoch": 0.6281925442198674, + "grad_norm": 2.8125, + "learning_rate": 3.916343396862878e-06, + "loss": 1.9061521530151366, + "mean_token_accuracy": 0.5807973630726337, + "num_tokens": 26885009.0, + "step": 3920 + }, + { + "entropy": 1.80911298468709, + "epoch": 0.6297950762204283, + "grad_norm": 2.875, + "learning_rate": 3.899477146230393e-06, + "loss": 1.87274227142334, + "mean_token_accuracy": 0.5926986195147037, + "num_tokens": 26953328.0, + "step": 3930 + }, + { + "entropy": 1.7731514915823936, + "epoch": 0.6313976082209891, + "grad_norm": 2.734375, + "learning_rate": 3.882610895597909e-06, + "loss": 1.8349546432495116, + "mean_token_accuracy": 0.5997257970273495, + "num_tokens": 27018951.0, + "step": 3940 + }, + { + "entropy": 1.8358073227107525, + "epoch": 0.63300014022155, + "grad_norm": 2.796875, + "learning_rate": 3.865744644965424e-06, + "loss": 1.8858596801757812, + "mean_token_accuracy": 0.5876387380063534, + "num_tokens": 27086370.0, + "step": 3950 + }, + { + "entropy": 1.8649689063429833, + "epoch": 0.634602672222111, + "grad_norm": 2.78125, + "learning_rate": 3.84887839433294e-06, + "loss": 1.9156248092651367, + "mean_token_accuracy": 0.5837285362184048, + "num_tokens": 27155131.0, + "step": 3960 + }, + { + "entropy": 1.8766348496079446, + "epoch": 0.6362052042226718, + "grad_norm": 2.609375, + "learning_rate": 3.832012143700456e-06, + "loss": 1.9406532287597655, + "mean_token_accuracy": 0.5804110359400511, + "num_tokens": 27225164.0, + "step": 3970 + }, + { + "entropy": 1.8654035195708274, + "epoch": 0.6378077362232327, + "grad_norm": 2.96875, + "learning_rate": 3.8151458930679715e-06, + "loss": 1.893624496459961, + "mean_token_accuracy": 0.5834946081042289, + "num_tokens": 27295122.0, + "step": 3980 + }, + { + "entropy": 1.9283578127622605, + "epoch": 0.6394102682237935, + "grad_norm": 2.796875, + "learning_rate": 3.798279642435487e-06, + "loss": 1.9532358169555664, + "mean_token_accuracy": 0.5727495942264795, + "num_tokens": 27366790.0, + "step": 3990 + }, + { + "entropy": 1.85790873169899, + "epoch": 0.6410128002243545, + "grad_norm": 3.171875, + "learning_rate": 3.781413391803003e-06, + "loss": 1.8932832717895507, + "mean_token_accuracy": 0.5854593172669411, + "num_tokens": 27435744.0, + "step": 4000 + }, + { + "entropy": 1.8387853726744652, + "epoch": 0.6426153322249154, + "grad_norm": 2.859375, + "learning_rate": 3.7645471411705182e-06, + "loss": 1.886970329284668, + "mean_token_accuracy": 0.5850338660180568, + "num_tokens": 27505901.0, + "step": 4010 + }, + { + "entropy": 1.9090761132538319, + "epoch": 0.6442178642254762, + "grad_norm": 2.703125, + "learning_rate": 3.7476808905380337e-06, + "loss": 1.9542694091796875, + "mean_token_accuracy": 0.5762960381805897, + "num_tokens": 27576637.0, + "step": 4020 + }, + { + "entropy": 1.8114977724850179, + "epoch": 0.6458203962260372, + "grad_norm": 2.90625, + "learning_rate": 3.7308146399055496e-06, + "loss": 1.8777925491333007, + "mean_token_accuracy": 0.5919980019330978, + "num_tokens": 27645081.0, + "step": 4030 + }, + { + "entropy": 1.8588636852800846, + "epoch": 0.647422928226598, + "grad_norm": 2.765625, + "learning_rate": 3.713948389273065e-06, + "loss": 1.9345163345336913, + "mean_token_accuracy": 0.5856771107763052, + "num_tokens": 27716654.0, + "step": 4040 + }, + { + "entropy": 1.8071557343006135, + "epoch": 0.6490254602271589, + "grad_norm": 3.109375, + "learning_rate": 3.6970821386405805e-06, + "loss": 1.8865779876708983, + "mean_token_accuracy": 0.5963869657367468, + "num_tokens": 27781907.0, + "step": 4050 + }, + { + "entropy": 1.8470010980963707, + "epoch": 0.6506279922277198, + "grad_norm": 2.765625, + "learning_rate": 3.6802158880080963e-06, + "loss": 1.915898323059082, + "mean_token_accuracy": 0.58299705311656, + "num_tokens": 27849927.0, + "step": 4060 + }, + { + "entropy": 1.8855213105678559, + "epoch": 0.6522305242282807, + "grad_norm": 2.96875, + "learning_rate": 3.6633496373756118e-06, + "loss": 1.9473722457885743, + "mean_token_accuracy": 0.5766369249671698, + "num_tokens": 27921805.0, + "step": 4070 + }, + { + "entropy": 1.8205056108534337, + "epoch": 0.6538330562288416, + "grad_norm": 2.8125, + "learning_rate": 3.6464833867431272e-06, + "loss": 1.868624496459961, + "mean_token_accuracy": 0.5868720389902592, + "num_tokens": 27989223.0, + "step": 4080 + }, + { + "entropy": 1.8420435622334481, + "epoch": 0.6554355882294025, + "grad_norm": 2.96875, + "learning_rate": 3.629617136110643e-06, + "loss": 1.8833166122436524, + "mean_token_accuracy": 0.5918383941054344, + "num_tokens": 28055631.0, + "step": 4090 + }, + { + "entropy": 1.8503759488463403, + "epoch": 0.6570381202299633, + "grad_norm": 2.796875, + "learning_rate": 3.6127508854781586e-06, + "loss": 1.9118997573852539, + "mean_token_accuracy": 0.588625418022275, + "num_tokens": 28123056.0, + "step": 4100 + }, + { + "entropy": 1.8946703895926476, + "epoch": 0.6586406522305243, + "grad_norm": 2.671875, + "learning_rate": 3.595884634845674e-06, + "loss": 1.924124526977539, + "mean_token_accuracy": 0.5777648027986289, + "num_tokens": 28193705.0, + "step": 4110 + }, + { + "entropy": 1.857002855092287, + "epoch": 0.6602431842310851, + "grad_norm": 2.96875, + "learning_rate": 3.57901838421319e-06, + "loss": 1.921992874145508, + "mean_token_accuracy": 0.585981798171997, + "num_tokens": 28262128.0, + "step": 4120 + }, + { + "entropy": 1.9027111411094666, + "epoch": 0.661845716231646, + "grad_norm": 2.875, + "learning_rate": 3.5621521335807053e-06, + "loss": 1.9402679443359374, + "mean_token_accuracy": 0.5746401138603687, + "num_tokens": 28334805.0, + "step": 4130 + }, + { + "entropy": 1.835258822888136, + "epoch": 0.6634482482322069, + "grad_norm": 3.109375, + "learning_rate": 3.5452858829482208e-06, + "loss": 1.9036882400512696, + "mean_token_accuracy": 0.5913544304668903, + "num_tokens": 28400153.0, + "step": 4140 + }, + { + "entropy": 1.8309452913701534, + "epoch": 0.6650507802327678, + "grad_norm": 2.921875, + "learning_rate": 3.5284196323157366e-06, + "loss": 1.898341178894043, + "mean_token_accuracy": 0.589846097677946, + "num_tokens": 28468130.0, + "step": 4150 + }, + { + "entropy": 1.7959118835628032, + "epoch": 0.6666533122333287, + "grad_norm": 3.265625, + "learning_rate": 3.511553381683252e-06, + "loss": 1.8534683227539062, + "mean_token_accuracy": 0.5948268000036478, + "num_tokens": 28536745.0, + "step": 4160 + }, + { + "entropy": 1.7979405872523784, + "epoch": 0.6682558442338895, + "grad_norm": 2.671875, + "learning_rate": 3.494687131050768e-06, + "loss": 1.8724647521972657, + "mean_token_accuracy": 0.5950725205242634, + "num_tokens": 28604849.0, + "step": 4170 + }, + { + "entropy": 1.8711092531681062, + "epoch": 0.6698583762344504, + "grad_norm": 2.671875, + "learning_rate": 3.4778208804182834e-06, + "loss": 1.9114248275756835, + "mean_token_accuracy": 0.583422476798296, + "num_tokens": 28673688.0, + "step": 4180 + }, + { + "entropy": 1.892587960511446, + "epoch": 0.6714609082350114, + "grad_norm": 3.015625, + "learning_rate": 3.460954629785799e-06, + "loss": 1.9401971817016601, + "mean_token_accuracy": 0.5767185620963573, + "num_tokens": 28743571.0, + "step": 4190 + }, + { + "entropy": 1.8978145480155946, + "epoch": 0.6730634402355722, + "grad_norm": 2.578125, + "learning_rate": 3.4440883791533147e-06, + "loss": 1.9398605346679687, + "mean_token_accuracy": 0.5744742628186941, + "num_tokens": 28814004.0, + "step": 4200 + }, + { + "entropy": 1.8904282376170158, + "epoch": 0.6746659722361331, + "grad_norm": 2.890625, + "learning_rate": 3.42722212852083e-06, + "loss": 1.9394752502441406, + "mean_token_accuracy": 0.579630171135068, + "num_tokens": 28888419.0, + "step": 4210 + }, + { + "entropy": 1.821556354314089, + "epoch": 0.6762685042366939, + "grad_norm": 2.703125, + "learning_rate": 3.4103558778883456e-06, + "loss": 1.8964935302734376, + "mean_token_accuracy": 0.5887388437986374, + "num_tokens": 28954657.0, + "step": 4220 + }, + { + "entropy": 1.7560958586633206, + "epoch": 0.6778710362372549, + "grad_norm": 3.015625, + "learning_rate": 3.3934896272558615e-06, + "loss": 1.8420488357543945, + "mean_token_accuracy": 0.6024362590163946, + "num_tokens": 29020174.0, + "step": 4230 + }, + { + "entropy": 1.8498153872787952, + "epoch": 0.6794735682378158, + "grad_norm": 2.734375, + "learning_rate": 3.376623376623377e-06, + "loss": 1.9001192092895507, + "mean_token_accuracy": 0.578879962861538, + "num_tokens": 29090973.0, + "step": 4240 + }, + { + "entropy": 1.8873520374298096, + "epoch": 0.6810761002383766, + "grad_norm": 2.78125, + "learning_rate": 3.3597571259908924e-06, + "loss": 1.9619806289672852, + "mean_token_accuracy": 0.5761936750262976, + "num_tokens": 29161675.0, + "step": 4250 + }, + { + "entropy": 1.8612853288650513, + "epoch": 0.6826786322389375, + "grad_norm": 2.875, + "learning_rate": 3.3428908753584083e-06, + "loss": 1.9202625274658203, + "mean_token_accuracy": 0.5825033310800791, + "num_tokens": 29229458.0, + "step": 4260 + }, + { + "entropy": 1.8753561928868294, + "epoch": 0.6842811642394984, + "grad_norm": 2.5625, + "learning_rate": 3.3260246247259237e-06, + "loss": 1.9131437301635743, + "mean_token_accuracy": 0.5833288405090571, + "num_tokens": 29298070.0, + "step": 4270 + }, + { + "entropy": 1.7987837627530099, + "epoch": 0.6858836962400593, + "grad_norm": 2.796875, + "learning_rate": 3.309158374093439e-06, + "loss": 1.8557870864868165, + "mean_token_accuracy": 0.5962090812623501, + "num_tokens": 29363949.0, + "step": 4280 + }, + { + "entropy": 1.8736120633780957, + "epoch": 0.6874862282406202, + "grad_norm": 3.15625, + "learning_rate": 3.292292123460955e-06, + "loss": 1.9229534149169922, + "mean_token_accuracy": 0.5777334328740835, + "num_tokens": 29435535.0, + "step": 4290 + }, + { + "entropy": 1.879132377356291, + "epoch": 0.689088760241181, + "grad_norm": 2.71875, + "learning_rate": 3.2754258728284705e-06, + "loss": 1.9337249755859376, + "mean_token_accuracy": 0.5785136640071868, + "num_tokens": 29504164.0, + "step": 4300 + }, + { + "entropy": 1.8871854215860366, + "epoch": 0.690691292241742, + "grad_norm": 2.828125, + "learning_rate": 3.258559622195986e-06, + "loss": 1.9246557235717774, + "mean_token_accuracy": 0.5791821502149105, + "num_tokens": 29572809.0, + "step": 4310 + }, + { + "entropy": 1.867794480919838, + "epoch": 0.6922938242423028, + "grad_norm": 2.859375, + "learning_rate": 3.241693371563502e-06, + "loss": 1.9245538711547852, + "mean_token_accuracy": 0.585183822363615, + "num_tokens": 29640253.0, + "step": 4320 + }, + { + "entropy": 1.8319248631596565, + "epoch": 0.6938963562428637, + "grad_norm": 3.515625, + "learning_rate": 3.2248271209310173e-06, + "loss": 1.9025934219360352, + "mean_token_accuracy": 0.5880723088979721, + "num_tokens": 29708282.0, + "step": 4330 + }, + { + "entropy": 1.8819570675492288, + "epoch": 0.6954988882434247, + "grad_norm": 2.859375, + "learning_rate": 3.2079608702985327e-06, + "loss": 1.9437738418579102, + "mean_token_accuracy": 0.578991699591279, + "num_tokens": 29775422.0, + "step": 4340 + }, + { + "entropy": 1.8296003483235836, + "epoch": 0.6971014202439855, + "grad_norm": 2.8125, + "learning_rate": 3.1910946196660486e-06, + "loss": 1.9154447555541991, + "mean_token_accuracy": 0.5931405682116747, + "num_tokens": 29844864.0, + "step": 4350 + }, + { + "entropy": 1.7935851775109768, + "epoch": 0.6987039522445464, + "grad_norm": 2.6875, + "learning_rate": 3.174228369033564e-06, + "loss": 1.8333089828491211, + "mean_token_accuracy": 0.5960859883576631, + "num_tokens": 29912079.0, + "step": 4360 + }, + { + "entropy": 1.8747857213020325, + "epoch": 0.7003064842451072, + "grad_norm": 3.125, + "learning_rate": 3.1573621184010795e-06, + "loss": 1.931085205078125, + "mean_token_accuracy": 0.578602023422718, + "num_tokens": 29982490.0, + "step": 4370 + }, + { + "entropy": 1.8096578311175109, + "epoch": 0.7019090162456681, + "grad_norm": 3.15625, + "learning_rate": 3.1404958677685953e-06, + "loss": 1.87011775970459, + "mean_token_accuracy": 0.601407915353775, + "num_tokens": 30048423.0, + "step": 4380 + }, + { + "entropy": 1.8461608737707138, + "epoch": 0.7035115482462291, + "grad_norm": 2.921875, + "learning_rate": 3.123629617136111e-06, + "loss": 1.8947601318359375, + "mean_token_accuracy": 0.5874628745019436, + "num_tokens": 30116819.0, + "step": 4390 + }, + { + "entropy": 1.8574359193444252, + "epoch": 0.7051140802467899, + "grad_norm": 2.90625, + "learning_rate": 3.1067633665036267e-06, + "loss": 1.9097387313842773, + "mean_token_accuracy": 0.5841793101280928, + "num_tokens": 30185743.0, + "step": 4400 + }, + { + "entropy": 1.796320417523384, + "epoch": 0.7067166122473508, + "grad_norm": 3.03125, + "learning_rate": 3.089897115871142e-06, + "loss": 1.8411190032958984, + "mean_token_accuracy": 0.5962270520627498, + "num_tokens": 30250772.0, + "step": 4410 + }, + { + "entropy": 1.8903830736875533, + "epoch": 0.7083191442479116, + "grad_norm": 2.96875, + "learning_rate": 3.0730308652386576e-06, + "loss": 1.9433307647705078, + "mean_token_accuracy": 0.5788230109959841, + "num_tokens": 30322456.0, + "step": 4420 + }, + { + "entropy": 1.8906516812741756, + "epoch": 0.7099216762484726, + "grad_norm": 2.984375, + "learning_rate": 3.0561646146061734e-06, + "loss": 1.949106788635254, + "mean_token_accuracy": 0.5781694982200861, + "num_tokens": 30390641.0, + "step": 4430 + }, + { + "entropy": 1.7996213018894196, + "epoch": 0.7115242082490335, + "grad_norm": 2.828125, + "learning_rate": 3.039298363973689e-06, + "loss": 1.8633939743041992, + "mean_token_accuracy": 0.5935101728886366, + "num_tokens": 30457494.0, + "step": 4440 + }, + { + "entropy": 1.8619076699018478, + "epoch": 0.7131267402495943, + "grad_norm": 3.0625, + "learning_rate": 3.0224321133412043e-06, + "loss": 1.934151840209961, + "mean_token_accuracy": 0.5796947993338109, + "num_tokens": 30526804.0, + "step": 4450 + }, + { + "entropy": 1.7975557282567025, + "epoch": 0.7147292722501553, + "grad_norm": 3.0625, + "learning_rate": 3.00556586270872e-06, + "loss": 1.8760286331176759, + "mean_token_accuracy": 0.5939261231571435, + "num_tokens": 30592498.0, + "step": 4460 + }, + { + "entropy": 1.8446580216288566, + "epoch": 0.7163318042507162, + "grad_norm": 3.015625, + "learning_rate": 2.9886996120762357e-06, + "loss": 1.8737337112426757, + "mean_token_accuracy": 0.5900526240468025, + "num_tokens": 30659144.0, + "step": 4470 + }, + { + "entropy": 1.853717105090618, + "epoch": 0.717934336251277, + "grad_norm": 2.90625, + "learning_rate": 2.971833361443751e-06, + "loss": 1.8939176559448243, + "mean_token_accuracy": 0.5849155694246292, + "num_tokens": 30727301.0, + "step": 4480 + }, + { + "entropy": 1.9284060686826705, + "epoch": 0.7195368682518379, + "grad_norm": 2.890625, + "learning_rate": 2.954967110811267e-06, + "loss": 1.9791166305541992, + "mean_token_accuracy": 0.5722882691770792, + "num_tokens": 30799212.0, + "step": 4490 + }, + { + "entropy": 1.797964096814394, + "epoch": 0.7211394002523988, + "grad_norm": 3.296875, + "learning_rate": 2.9381008601787824e-06, + "loss": 1.8483495712280273, + "mean_token_accuracy": 0.5944610767066478, + "num_tokens": 30866936.0, + "step": 4500 + }, + { + "entropy": 1.9101737931370735, + "epoch": 0.7227419322529597, + "grad_norm": 2.6875, + "learning_rate": 2.921234609546298e-06, + "loss": 1.9717981338500976, + "mean_token_accuracy": 0.5766745507717133, + "num_tokens": 30939177.0, + "step": 4510 + }, + { + "entropy": 1.899225589632988, + "epoch": 0.7243444642535206, + "grad_norm": 2.75, + "learning_rate": 2.9043683589138137e-06, + "loss": 1.942579460144043, + "mean_token_accuracy": 0.5780370756983757, + "num_tokens": 31010329.0, + "step": 4520 + }, + { + "entropy": 1.8574948862195015, + "epoch": 0.7259469962540814, + "grad_norm": 3.3125, + "learning_rate": 2.887502108281329e-06, + "loss": 1.9129962921142578, + "mean_token_accuracy": 0.5802199840545654, + "num_tokens": 31079276.0, + "step": 4530 + }, + { + "entropy": 1.8531811386346817, + "epoch": 0.7275495282546424, + "grad_norm": 3.09375, + "learning_rate": 2.8706358576488446e-06, + "loss": 1.8707773208618164, + "mean_token_accuracy": 0.5845135271549224, + "num_tokens": 31149692.0, + "step": 4540 + }, + { + "entropy": 1.8767701983451843, + "epoch": 0.7291520602552032, + "grad_norm": 3.0625, + "learning_rate": 2.8537696070163605e-06, + "loss": 1.932365608215332, + "mean_token_accuracy": 0.5770091351121664, + "num_tokens": 31220597.0, + "step": 4550 + }, + { + "entropy": 1.837051272392273, + "epoch": 0.7307545922557641, + "grad_norm": 3.28125, + "learning_rate": 2.836903356383876e-06, + "loss": 1.9031335830688476, + "mean_token_accuracy": 0.5894479807466269, + "num_tokens": 31288332.0, + "step": 4560 + }, + { + "entropy": 1.8541367374360562, + "epoch": 0.732357124256325, + "grad_norm": 3.0625, + "learning_rate": 2.8200371057513914e-06, + "loss": 1.906287956237793, + "mean_token_accuracy": 0.5865425508469343, + "num_tokens": 31356335.0, + "step": 4570 + }, + { + "entropy": 1.8312122486531734, + "epoch": 0.7339596562568859, + "grad_norm": 3.015625, + "learning_rate": 2.8031708551189073e-06, + "loss": 1.91760311126709, + "mean_token_accuracy": 0.5876502882689237, + "num_tokens": 31426261.0, + "step": 4580 + }, + { + "entropy": 1.8157531134784222, + "epoch": 0.7355621882574468, + "grad_norm": 2.65625, + "learning_rate": 2.7863046044864227e-06, + "loss": 1.8741228103637695, + "mean_token_accuracy": 0.5895969212055207, + "num_tokens": 31493295.0, + "step": 4590 + }, + { + "entropy": 1.862128420174122, + "epoch": 0.7371647202580076, + "grad_norm": 2.71875, + "learning_rate": 2.769438353853938e-06, + "loss": 1.9290430068969726, + "mean_token_accuracy": 0.5843666099011898, + "num_tokens": 31562680.0, + "step": 4600 + }, + { + "entropy": 1.7904260948300361, + "epoch": 0.7387672522585685, + "grad_norm": 2.75, + "learning_rate": 2.752572103221454e-06, + "loss": 1.8450855255126952, + "mean_token_accuracy": 0.5969460435211659, + "num_tokens": 31627777.0, + "step": 4610 + }, + { + "entropy": 1.8627801559865476, + "epoch": 0.7403697842591295, + "grad_norm": 3.0625, + "learning_rate": 2.7357058525889695e-06, + "loss": 1.9175779342651367, + "mean_token_accuracy": 0.5832827895879745, + "num_tokens": 31696447.0, + "step": 4620 + }, + { + "entropy": 1.7895106546580792, + "epoch": 0.7419723162596903, + "grad_norm": 2.703125, + "learning_rate": 2.7188396019564854e-06, + "loss": 1.8732286453247071, + "mean_token_accuracy": 0.6013656318187713, + "num_tokens": 31762723.0, + "step": 4630 + }, + { + "entropy": 1.8264621473848819, + "epoch": 0.7435748482602512, + "grad_norm": 3.1875, + "learning_rate": 2.701973351324001e-06, + "loss": 1.8991756439208984, + "mean_token_accuracy": 0.5955311380326748, + "num_tokens": 31828494.0, + "step": 4640 + }, + { + "entropy": 1.907581451535225, + "epoch": 0.745177380260812, + "grad_norm": 2.84375, + "learning_rate": 2.6851071006915163e-06, + "loss": 1.9553951263427733, + "mean_token_accuracy": 0.5728101029992103, + "num_tokens": 31899128.0, + "step": 4650 + }, + { + "entropy": 1.8415529295802116, + "epoch": 0.746779912261373, + "grad_norm": 3.0625, + "learning_rate": 2.668240850059032e-06, + "loss": 1.9126508712768555, + "mean_token_accuracy": 0.5831385359168053, + "num_tokens": 31966401.0, + "step": 4660 + }, + { + "entropy": 1.8863008134067059, + "epoch": 0.7483824442619339, + "grad_norm": 2.78125, + "learning_rate": 2.6513745994265476e-06, + "loss": 1.9363431930541992, + "mean_token_accuracy": 0.5793558444827795, + "num_tokens": 32036907.0, + "step": 4670 + }, + { + "entropy": 1.823825690150261, + "epoch": 0.7499849762624947, + "grad_norm": 2.953125, + "learning_rate": 2.634508348794063e-06, + "loss": 1.888538932800293, + "mean_token_accuracy": 0.5882374349981546, + "num_tokens": 32103892.0, + "step": 4680 + }, + { + "entropy": 1.8487841807305814, + "epoch": 0.7515875082630556, + "grad_norm": 3.5, + "learning_rate": 2.617642098161579e-06, + "loss": 1.883224868774414, + "mean_token_accuracy": 0.5878496304154396, + "num_tokens": 32170790.0, + "step": 4690 + }, + { + "entropy": 1.8293106988072396, + "epoch": 0.7531900402636165, + "grad_norm": 3.09375, + "learning_rate": 2.6007758475290944e-06, + "loss": 1.8674444198608398, + "mean_token_accuracy": 0.590721195936203, + "num_tokens": 32237508.0, + "step": 4700 + }, + { + "entropy": 1.8427614197134972, + "epoch": 0.7547925722641774, + "grad_norm": 2.984375, + "learning_rate": 2.58390959689661e-06, + "loss": 1.931664276123047, + "mean_token_accuracy": 0.585081772506237, + "num_tokens": 32304433.0, + "step": 4710 + }, + { + "entropy": 1.8339687652885914, + "epoch": 0.7563951042647383, + "grad_norm": 3.109375, + "learning_rate": 2.5670433462641257e-06, + "loss": 1.8750896453857422, + "mean_token_accuracy": 0.5903500825166702, + "num_tokens": 32372527.0, + "step": 4720 + }, + { + "entropy": 1.8150975018739701, + "epoch": 0.7579976362652991, + "grad_norm": 2.984375, + "learning_rate": 2.550177095631641e-06, + "loss": 1.8547204971313476, + "mean_token_accuracy": 0.5925628982484341, + "num_tokens": 32439016.0, + "step": 4730 + }, + { + "entropy": 1.8668007925152779, + "epoch": 0.7596001682658601, + "grad_norm": 2.703125, + "learning_rate": 2.5333108449991566e-06, + "loss": 1.9275758743286133, + "mean_token_accuracy": 0.5824221331626177, + "num_tokens": 32508193.0, + "step": 4740 + }, + { + "entropy": 1.8837640598416328, + "epoch": 0.7612027002664209, + "grad_norm": 2.8125, + "learning_rate": 2.5164445943666725e-06, + "loss": 1.962438201904297, + "mean_token_accuracy": 0.5787151921540499, + "num_tokens": 32578305.0, + "step": 4750 + }, + { + "entropy": 1.823105738312006, + "epoch": 0.7628052322669818, + "grad_norm": 2.765625, + "learning_rate": 2.499578343734188e-06, + "loss": 1.877680778503418, + "mean_token_accuracy": 0.5912107802927494, + "num_tokens": 32646587.0, + "step": 4760 + }, + { + "entropy": 1.8075616881251335, + "epoch": 0.7644077642675428, + "grad_norm": 2.921875, + "learning_rate": 2.4827120931017038e-06, + "loss": 1.8594440460205077, + "mean_token_accuracy": 0.5939967796206475, + "num_tokens": 32712758.0, + "step": 4770 + }, + { + "entropy": 1.8341136962175368, + "epoch": 0.7660102962681036, + "grad_norm": 3.1875, + "learning_rate": 2.4658458424692192e-06, + "loss": 1.8853012084960938, + "mean_token_accuracy": 0.5875776465982199, + "num_tokens": 32779513.0, + "step": 4780 + }, + { + "entropy": 1.8376713953912258, + "epoch": 0.7676128282686645, + "grad_norm": 2.765625, + "learning_rate": 2.4489795918367347e-06, + "loss": 1.8840415954589844, + "mean_token_accuracy": 0.5857285771518945, + "num_tokens": 32848465.0, + "step": 4790 + }, + { + "entropy": 1.857311400771141, + "epoch": 0.7692153602692253, + "grad_norm": 3.046875, + "learning_rate": 2.4321133412042505e-06, + "loss": 1.915665626525879, + "mean_token_accuracy": 0.5853688273578882, + "num_tokens": 32916117.0, + "step": 4800 + }, + { + "entropy": 1.8423568911850452, + "epoch": 0.7708178922697863, + "grad_norm": 2.578125, + "learning_rate": 2.4152470905717664e-06, + "loss": 1.9055423736572266, + "mean_token_accuracy": 0.5882505733519793, + "num_tokens": 32984183.0, + "step": 4810 + }, + { + "entropy": 1.835125819593668, + "epoch": 0.7724204242703472, + "grad_norm": 2.8125, + "learning_rate": 2.3983808399392814e-06, + "loss": 1.9093955993652343, + "mean_token_accuracy": 0.5893498983234167, + "num_tokens": 33052722.0, + "step": 4820 + }, + { + "entropy": 1.8223372012376786, + "epoch": 0.774022956270908, + "grad_norm": 3.078125, + "learning_rate": 2.3815145893067973e-06, + "loss": 1.8917829513549804, + "mean_token_accuracy": 0.5907191261649132, + "num_tokens": 33121558.0, + "step": 4830 + }, + { + "entropy": 1.8190541714429855, + "epoch": 0.7756254882714689, + "grad_norm": 2.828125, + "learning_rate": 2.364648338674313e-06, + "loss": 1.8961849212646484, + "mean_token_accuracy": 0.5917201146483422, + "num_tokens": 33190162.0, + "step": 4840 + }, + { + "entropy": 1.8600998565554618, + "epoch": 0.7772280202720299, + "grad_norm": 2.734375, + "learning_rate": 2.347782088041828e-06, + "loss": 1.8977258682250977, + "mean_token_accuracy": 0.5800701964646577, + "num_tokens": 33260039.0, + "step": 4850 + }, + { + "entropy": 1.7787691496312619, + "epoch": 0.7788305522725907, + "grad_norm": 2.875, + "learning_rate": 2.330915837409344e-06, + "loss": 1.8467596054077149, + "mean_token_accuracy": 0.60061249807477, + "num_tokens": 33323447.0, + "step": 4860 + }, + { + "entropy": 1.8592630334198474, + "epoch": 0.7804330842731516, + "grad_norm": 3.015625, + "learning_rate": 2.31404958677686e-06, + "loss": 1.8995931625366211, + "mean_token_accuracy": 0.585073859244585, + "num_tokens": 33392525.0, + "step": 4870 + }, + { + "entropy": 1.844882532209158, + "epoch": 0.7820356162737124, + "grad_norm": 2.890625, + "learning_rate": 2.297183336144375e-06, + "loss": 1.9015777587890625, + "mean_token_accuracy": 0.5880853958427906, + "num_tokens": 33460290.0, + "step": 4880 + }, + { + "entropy": 1.845929805189371, + "epoch": 0.7836381482742734, + "grad_norm": 2.84375, + "learning_rate": 2.280317085511891e-06, + "loss": 1.917535400390625, + "mean_token_accuracy": 0.583182455226779, + "num_tokens": 33530065.0, + "step": 4890 + }, + { + "entropy": 1.8467786006629467, + "epoch": 0.7852406802748343, + "grad_norm": 3.03125, + "learning_rate": 2.2634508348794067e-06, + "loss": 1.9201107025146484, + "mean_token_accuracy": 0.589047035202384, + "num_tokens": 33598388.0, + "step": 4900 + }, + { + "entropy": 1.8024130068719386, + "epoch": 0.7868432122753951, + "grad_norm": 2.921875, + "learning_rate": 2.246584584246922e-06, + "loss": 1.8678976058959962, + "mean_token_accuracy": 0.5942654374986887, + "num_tokens": 33668337.0, + "step": 4910 + }, + { + "entropy": 1.8550097823143006, + "epoch": 0.788445744275956, + "grad_norm": 2.984375, + "learning_rate": 2.2297183336144376e-06, + "loss": 1.8873409271240233, + "mean_token_accuracy": 0.5828581850975751, + "num_tokens": 33737823.0, + "step": 4920 + }, + { + "entropy": 1.8001336701214314, + "epoch": 0.7900482762765169, + "grad_norm": 3.171875, + "learning_rate": 2.2128520829819535e-06, + "loss": 1.8621498107910157, + "mean_token_accuracy": 0.5943111833184958, + "num_tokens": 33802844.0, + "step": 4930 + }, + { + "entropy": 1.8257937185466289, + "epoch": 0.7916508082770778, + "grad_norm": 3.046875, + "learning_rate": 2.195985832349469e-06, + "loss": 1.9113481521606446, + "mean_token_accuracy": 0.5905203901231288, + "num_tokens": 33872731.0, + "step": 4940 + }, + { + "entropy": 1.8191636987030506, + "epoch": 0.7932533402776387, + "grad_norm": 2.859375, + "learning_rate": 2.1791195817169844e-06, + "loss": 1.8932903289794922, + "mean_token_accuracy": 0.5920727163553238, + "num_tokens": 33940292.0, + "step": 4950 + }, + { + "entropy": 1.786191315948963, + "epoch": 0.7948558722781995, + "grad_norm": 3.234375, + "learning_rate": 2.1622533310845003e-06, + "loss": 1.843625831604004, + "mean_token_accuracy": 0.5974460437893867, + "num_tokens": 34005383.0, + "step": 4960 + }, + { + "entropy": 1.8545084938406944, + "epoch": 0.7964584042787605, + "grad_norm": 2.75, + "learning_rate": 2.1453870804520157e-06, + "loss": 1.9135856628417969, + "mean_token_accuracy": 0.5837194677442312, + "num_tokens": 34075792.0, + "step": 4970 + }, + { + "entropy": 1.871314498782158, + "epoch": 0.7980609362793213, + "grad_norm": 2.875, + "learning_rate": 2.128520829819531e-06, + "loss": 1.9335128784179687, + "mean_token_accuracy": 0.5792830560356379, + "num_tokens": 34146039.0, + "step": 4980 + }, + { + "entropy": 1.8142223849892616, + "epoch": 0.7996634682798822, + "grad_norm": 3.078125, + "learning_rate": 2.111654579187047e-06, + "loss": 1.8717248916625977, + "mean_token_accuracy": 0.5912014968693257, + "num_tokens": 34213756.0, + "step": 4990 + }, + { + "entropy": 1.8623139068484307, + "epoch": 0.8012660002804431, + "grad_norm": 2.75, + "learning_rate": 2.0947883285545625e-06, + "loss": 1.8864187240600585, + "mean_token_accuracy": 0.5801310263574123, + "num_tokens": 34282925.0, + "step": 5000 + }, + { + "entropy": 1.9080819115042686, + "epoch": 0.802868532281004, + "grad_norm": 2.9375, + "learning_rate": 2.0779220779220784e-06, + "loss": 1.9643669128417969, + "mean_token_accuracy": 0.5719181023538112, + "num_tokens": 34354376.0, + "step": 5010 + }, + { + "entropy": 1.8581409096717834, + "epoch": 0.8044710642815649, + "grad_norm": 2.75, + "learning_rate": 2.061055827289594e-06, + "loss": 1.9315671920776367, + "mean_token_accuracy": 0.5858389385044575, + "num_tokens": 34425581.0, + "step": 5020 + }, + { + "entropy": 1.8516808852553368, + "epoch": 0.8060735962821257, + "grad_norm": 2.875, + "learning_rate": 2.0441895766571092e-06, + "loss": 1.888896369934082, + "mean_token_accuracy": 0.5831075765192508, + "num_tokens": 34493276.0, + "step": 5030 + }, + { + "entropy": 1.928952093422413, + "epoch": 0.8076761282826866, + "grad_norm": 2.71875, + "learning_rate": 2.027323326024625e-06, + "loss": 1.9985475540161133, + "mean_token_accuracy": 0.5690149266272784, + "num_tokens": 34568048.0, + "step": 5040 + }, + { + "entropy": 1.8902572244405746, + "epoch": 0.8092786602832476, + "grad_norm": 2.90625, + "learning_rate": 2.0104570753921406e-06, + "loss": 1.9458175659179688, + "mean_token_accuracy": 0.579744578525424, + "num_tokens": 34637422.0, + "step": 5050 + }, + { + "entropy": 1.9189714342355728, + "epoch": 0.8108811922838084, + "grad_norm": 2.921875, + "learning_rate": 1.993590824759656e-06, + "loss": 1.9847827911376954, + "mean_token_accuracy": 0.5708337672054767, + "num_tokens": 34708012.0, + "step": 5060 + }, + { + "entropy": 1.8283629328012467, + "epoch": 0.8124837242843693, + "grad_norm": 2.84375, + "learning_rate": 1.976724574127172e-06, + "loss": 1.8860307693481446, + "mean_token_accuracy": 0.5901182591915131, + "num_tokens": 34776908.0, + "step": 5070 + }, + { + "entropy": 1.8660996221005917, + "epoch": 0.8140862562849301, + "grad_norm": 2.875, + "learning_rate": 1.9598583234946873e-06, + "loss": 1.9303264617919922, + "mean_token_accuracy": 0.5811872001737356, + "num_tokens": 34845348.0, + "step": 5080 + }, + { + "entropy": 1.8891982451081275, + "epoch": 0.8156887882854911, + "grad_norm": 2.921875, + "learning_rate": 1.9429920728622028e-06, + "loss": 1.9262752532958984, + "mean_token_accuracy": 0.5712238024920225, + "num_tokens": 34917326.0, + "step": 5090 + }, + { + "entropy": 1.8782757677137851, + "epoch": 0.817291320286052, + "grad_norm": 2.734375, + "learning_rate": 1.9261258222297187e-06, + "loss": 1.9394376754760743, + "mean_token_accuracy": 0.5833391141146421, + "num_tokens": 34987233.0, + "step": 5100 + }, + { + "entropy": 1.836723731458187, + "epoch": 0.8188938522866128, + "grad_norm": 2.90625, + "learning_rate": 1.909259571597234e-06, + "loss": 1.8888580322265625, + "mean_token_accuracy": 0.5898743011057377, + "num_tokens": 35054390.0, + "step": 5110 + }, + { + "entropy": 1.8592747025191785, + "epoch": 0.8204963842871738, + "grad_norm": 2.75, + "learning_rate": 1.8923933209647496e-06, + "loss": 1.918815803527832, + "mean_token_accuracy": 0.5785074956715107, + "num_tokens": 35124783.0, + "step": 5120 + }, + { + "entropy": 1.802715352922678, + "epoch": 0.8220989162877346, + "grad_norm": 2.890625, + "learning_rate": 1.8755270703322654e-06, + "loss": 1.9023824691772462, + "mean_token_accuracy": 0.5942517884075642, + "num_tokens": 35192523.0, + "step": 5130 + }, + { + "entropy": 1.8116022616624832, + "epoch": 0.8237014482882955, + "grad_norm": 2.71875, + "learning_rate": 1.858660819699781e-06, + "loss": 1.871706771850586, + "mean_token_accuracy": 0.5937372047454119, + "num_tokens": 35260988.0, + "step": 5140 + }, + { + "entropy": 1.8106999851763248, + "epoch": 0.8253039802888564, + "grad_norm": 2.90625, + "learning_rate": 1.8417945690672963e-06, + "loss": 1.8581106185913085, + "mean_token_accuracy": 0.5908673726022243, + "num_tokens": 35327982.0, + "step": 5150 + }, + { + "entropy": 1.8918942615389824, + "epoch": 0.8269065122894173, + "grad_norm": 3.046875, + "learning_rate": 1.8249283184348122e-06, + "loss": 1.9652887344360352, + "mean_token_accuracy": 0.5815871477127075, + "num_tokens": 35397549.0, + "step": 5160 + }, + { + "entropy": 1.8354435302317142, + "epoch": 0.8285090442899782, + "grad_norm": 2.859375, + "learning_rate": 1.8080620678023279e-06, + "loss": 1.8734939575195313, + "mean_token_accuracy": 0.5881023917347192, + "num_tokens": 35466217.0, + "step": 5170 + }, + { + "entropy": 1.8706816494464875, + "epoch": 0.830111576290539, + "grad_norm": 2.828125, + "learning_rate": 1.791195817169843e-06, + "loss": 1.9151870727539062, + "mean_token_accuracy": 0.5856871705502271, + "num_tokens": 35536080.0, + "step": 5180 + }, + { + "entropy": 1.8841779872775077, + "epoch": 0.8317141082910999, + "grad_norm": 3.09375, + "learning_rate": 1.774329566537359e-06, + "loss": 1.942214584350586, + "mean_token_accuracy": 0.5793454956263304, + "num_tokens": 35608158.0, + "step": 5190 + }, + { + "entropy": 1.848481398075819, + "epoch": 0.8333166402916609, + "grad_norm": 2.9375, + "learning_rate": 1.7574633159048746e-06, + "loss": 1.9140625, + "mean_token_accuracy": 0.5864365387707948, + "num_tokens": 35676010.0, + "step": 5200 + }, + { + "entropy": 1.7927502900362016, + "epoch": 0.8349191722922217, + "grad_norm": 3.265625, + "learning_rate": 1.74059706527239e-06, + "loss": 1.8784461975097657, + "mean_token_accuracy": 0.5948816005140543, + "num_tokens": 35743360.0, + "step": 5210 + }, + { + "entropy": 1.8374880060553551, + "epoch": 0.8365217042927826, + "grad_norm": 2.90625, + "learning_rate": 1.7237308146399057e-06, + "loss": 1.8934955596923828, + "mean_token_accuracy": 0.5864754639565944, + "num_tokens": 35812001.0, + "step": 5220 + }, + { + "entropy": 1.9162466913461684, + "epoch": 0.8381242362933434, + "grad_norm": 3.015625, + "learning_rate": 1.7068645640074214e-06, + "loss": 1.989309310913086, + "mean_token_accuracy": 0.5767899330705404, + "num_tokens": 35883914.0, + "step": 5230 + }, + { + "entropy": 1.8502981126308442, + "epoch": 0.8397267682939044, + "grad_norm": 3.046875, + "learning_rate": 1.689998313374937e-06, + "loss": 1.9092756271362306, + "mean_token_accuracy": 0.5868145152926445, + "num_tokens": 35951633.0, + "step": 5240 + }, + { + "entropy": 1.8918419994413853, + "epoch": 0.8413293002944653, + "grad_norm": 2.703125, + "learning_rate": 1.6731320627424525e-06, + "loss": 1.9371561050415038, + "mean_token_accuracy": 0.5782828338444232, + "num_tokens": 36022240.0, + "step": 5250 + }, + { + "entropy": 1.858195111900568, + "epoch": 0.8429318322950261, + "grad_norm": 3.09375, + "learning_rate": 1.6562658121099682e-06, + "loss": 1.9401632308959962, + "mean_token_accuracy": 0.5855324938893318, + "num_tokens": 36089230.0, + "step": 5260 + }, + { + "entropy": 1.9209559485316277, + "epoch": 0.844534364295587, + "grad_norm": 3.03125, + "learning_rate": 1.6393995614774838e-06, + "loss": 1.9692459106445312, + "mean_token_accuracy": 0.5720816683024168, + "num_tokens": 36158834.0, + "step": 5270 + }, + { + "entropy": 1.8659572064876557, + "epoch": 0.846136896296148, + "grad_norm": 3.046875, + "learning_rate": 1.6225333108449993e-06, + "loss": 1.890706443786621, + "mean_token_accuracy": 0.5827448319643735, + "num_tokens": 36229727.0, + "step": 5280 + }, + { + "entropy": 1.8979657620191575, + "epoch": 0.8477394282967088, + "grad_norm": 3.21875, + "learning_rate": 1.605667060212515e-06, + "loss": 1.9233301162719727, + "mean_token_accuracy": 0.5765829961746931, + "num_tokens": 36298130.0, + "step": 5290 + }, + { + "entropy": 1.8676749154925347, + "epoch": 0.8493419602972697, + "grad_norm": 3.203125, + "learning_rate": 1.5888008095800306e-06, + "loss": 1.9104772567749024, + "mean_token_accuracy": 0.5829620614647866, + "num_tokens": 36368506.0, + "step": 5300 + }, + { + "entropy": 1.894658635556698, + "epoch": 0.8509444922978305, + "grad_norm": 3.0, + "learning_rate": 1.571934558947546e-06, + "loss": 1.939011001586914, + "mean_token_accuracy": 0.5757065914571285, + "num_tokens": 36437738.0, + "step": 5310 + }, + { + "entropy": 1.8023961640894413, + "epoch": 0.8525470242983915, + "grad_norm": 2.53125, + "learning_rate": 1.5550683083150617e-06, + "loss": 1.8812477111816406, + "mean_token_accuracy": 0.5942258331924677, + "num_tokens": 36504934.0, + "step": 5320 + }, + { + "entropy": 1.8514545068144799, + "epoch": 0.8541495562989524, + "grad_norm": 2.90625, + "learning_rate": 1.5382020576825774e-06, + "loss": 1.8894914627075194, + "mean_token_accuracy": 0.580870047956705, + "num_tokens": 36573640.0, + "step": 5330 + }, + { + "entropy": 1.87561452165246, + "epoch": 0.8557520882995132, + "grad_norm": 2.765625, + "learning_rate": 1.5213358070500928e-06, + "loss": 1.9121932983398438, + "mean_token_accuracy": 0.5853728234767914, + "num_tokens": 36643224.0, + "step": 5340 + }, + { + "entropy": 1.8336880192160607, + "epoch": 0.8573546203000741, + "grad_norm": 2.765625, + "learning_rate": 1.5044695564176085e-06, + "loss": 1.8873580932617187, + "mean_token_accuracy": 0.5845846958458424, + "num_tokens": 36712263.0, + "step": 5350 + }, + { + "entropy": 1.8954137042164803, + "epoch": 0.858957152300635, + "grad_norm": 3.015625, + "learning_rate": 1.4876033057851241e-06, + "loss": 1.9475727081298828, + "mean_token_accuracy": 0.5799393642693758, + "num_tokens": 36781564.0, + "step": 5360 + }, + { + "entropy": 1.8517433993518353, + "epoch": 0.8605596843011959, + "grad_norm": 3.0625, + "learning_rate": 1.4707370551526398e-06, + "loss": 1.9321174621582031, + "mean_token_accuracy": 0.5853044964373112, + "num_tokens": 36849659.0, + "step": 5370 + }, + { + "entropy": 1.8717767260968685, + "epoch": 0.8621622163017568, + "grad_norm": 2.765625, + "learning_rate": 1.4538708045201552e-06, + "loss": 1.9176826477050781, + "mean_token_accuracy": 0.5836375288665294, + "num_tokens": 36919377.0, + "step": 5380 + }, + { + "entropy": 1.8603466354310512, + "epoch": 0.8637647483023176, + "grad_norm": 3.046875, + "learning_rate": 1.437004553887671e-06, + "loss": 1.9242975234985351, + "mean_token_accuracy": 0.5835679214447737, + "num_tokens": 36989574.0, + "step": 5390 + }, + { + "entropy": 1.880230689048767, + "epoch": 0.8653672803028786, + "grad_norm": 2.71875, + "learning_rate": 1.4201383032551866e-06, + "loss": 1.9477001190185548, + "mean_token_accuracy": 0.5788057737052441, + "num_tokens": 37058313.0, + "step": 5400 + }, + { + "entropy": 1.8516812466084958, + "epoch": 0.8669698123034394, + "grad_norm": 2.625, + "learning_rate": 1.403272052622702e-06, + "loss": 1.9131620407104493, + "mean_token_accuracy": 0.5865709237754345, + "num_tokens": 37127963.0, + "step": 5410 + }, + { + "entropy": 1.875563132762909, + "epoch": 0.8685723443040003, + "grad_norm": 3.0625, + "learning_rate": 1.3864058019902177e-06, + "loss": 1.9106025695800781, + "mean_token_accuracy": 0.57944978736341, + "num_tokens": 37198896.0, + "step": 5420 + }, + { + "entropy": 1.8328796833753587, + "epoch": 0.8701748763045613, + "grad_norm": 2.78125, + "learning_rate": 1.3695395513577333e-06, + "loss": 1.8999868392944337, + "mean_token_accuracy": 0.5887691337615252, + "num_tokens": 37266555.0, + "step": 5430 + }, + { + "entropy": 1.8776087261736394, + "epoch": 0.8717774083051221, + "grad_norm": 3.0625, + "learning_rate": 1.3526733007252488e-06, + "loss": 1.9219970703125, + "mean_token_accuracy": 0.5799083292484284, + "num_tokens": 37337861.0, + "step": 5440 + }, + { + "entropy": 1.9137307450175285, + "epoch": 0.873379940305683, + "grad_norm": 2.6875, + "learning_rate": 1.3358070500927644e-06, + "loss": 1.9520265579223632, + "mean_token_accuracy": 0.5753105688840151, + "num_tokens": 37408519.0, + "step": 5450 + }, + { + "entropy": 1.8622472748160361, + "epoch": 0.8749824723062438, + "grad_norm": 3.234375, + "learning_rate": 1.31894079946028e-06, + "loss": 1.899980354309082, + "mean_token_accuracy": 0.5822821542620659, + "num_tokens": 37476669.0, + "step": 5460 + }, + { + "entropy": 1.8612875059247016, + "epoch": 0.8765850043068047, + "grad_norm": 2.765625, + "learning_rate": 1.3020745488277958e-06, + "loss": 1.911638069152832, + "mean_token_accuracy": 0.5831337984651327, + "num_tokens": 37546286.0, + "step": 5470 + }, + { + "entropy": 1.8205799013376236, + "epoch": 0.8781875363073657, + "grad_norm": 2.84375, + "learning_rate": 1.2852082981953112e-06, + "loss": 1.8920133590698243, + "mean_token_accuracy": 0.5909904126077891, + "num_tokens": 37617453.0, + "step": 5480 + }, + { + "entropy": 1.8669624656438828, + "epoch": 0.8797900683079265, + "grad_norm": 3.171875, + "learning_rate": 1.2683420475628269e-06, + "loss": 1.9207725524902344, + "mean_token_accuracy": 0.5827377833425998, + "num_tokens": 37687831.0, + "step": 5490 + }, + { + "entropy": 1.8486655861139298, + "epoch": 0.8813926003084874, + "grad_norm": 2.78125, + "learning_rate": 1.2514757969303425e-06, + "loss": 1.881540870666504, + "mean_token_accuracy": 0.5863363519310951, + "num_tokens": 37755218.0, + "step": 5500 + }, + { + "entropy": 1.844733679294586, + "epoch": 0.8829951323090482, + "grad_norm": 3.015625, + "learning_rate": 1.2346095462978582e-06, + "loss": 1.910595703125, + "mean_token_accuracy": 0.5841306939721107, + "num_tokens": 37824810.0, + "step": 5510 + }, + { + "entropy": 1.812732180953026, + "epoch": 0.8845976643096092, + "grad_norm": 2.921875, + "learning_rate": 1.2177432956653736e-06, + "loss": 1.8701910018920898, + "mean_token_accuracy": 0.5909413047134876, + "num_tokens": 37890678.0, + "step": 5520 + }, + { + "entropy": 1.892632459849119, + "epoch": 0.8862001963101701, + "grad_norm": 3.078125, + "learning_rate": 1.2008770450328893e-06, + "loss": 1.9323055267333984, + "mean_token_accuracy": 0.5802071906626225, + "num_tokens": 37961740.0, + "step": 5530 + }, + { + "entropy": 1.7887009508907794, + "epoch": 0.8878027283107309, + "grad_norm": 2.859375, + "learning_rate": 1.184010794400405e-06, + "loss": 1.8533214569091796, + "mean_token_accuracy": 0.5976423732936382, + "num_tokens": 38026435.0, + "step": 5540 + }, + { + "entropy": 1.8592222422361373, + "epoch": 0.8894052603112919, + "grad_norm": 2.8125, + "learning_rate": 1.1671445437679204e-06, + "loss": 1.9166738510131835, + "mean_token_accuracy": 0.5843483276665211, + "num_tokens": 38093902.0, + "step": 5550 + }, + { + "entropy": 1.8362051859498023, + "epoch": 0.8910077923118527, + "grad_norm": 2.75, + "learning_rate": 1.150278293135436e-06, + "loss": 1.8831689834594727, + "mean_token_accuracy": 0.5898423045873642, + "num_tokens": 38159392.0, + "step": 5560 + }, + { + "entropy": 1.8008449248969556, + "epoch": 0.8926103243124136, + "grad_norm": 2.984375, + "learning_rate": 1.1334120425029517e-06, + "loss": 1.8782642364501954, + "mean_token_accuracy": 0.5952645525336265, + "num_tokens": 38227315.0, + "step": 5570 + }, + { + "entropy": 1.8333451189100742, + "epoch": 0.8942128563129745, + "grad_norm": 3.015625, + "learning_rate": 1.1165457918704672e-06, + "loss": 1.881983757019043, + "mean_token_accuracy": 0.5908118661493063, + "num_tokens": 38293521.0, + "step": 5580 + }, + { + "entropy": 1.802014109492302, + "epoch": 0.8958153883135354, + "grad_norm": 2.734375, + "learning_rate": 1.0996795412379828e-06, + "loss": 1.8676593780517579, + "mean_token_accuracy": 0.5922972787171602, + "num_tokens": 38360577.0, + "step": 5590 + }, + { + "entropy": 1.8505891650915145, + "epoch": 0.8974179203140963, + "grad_norm": 2.953125, + "learning_rate": 1.0828132906054985e-06, + "loss": 1.9128009796142578, + "mean_token_accuracy": 0.5826629012823105, + "num_tokens": 38428404.0, + "step": 5600 + }, + { + "entropy": 1.8447042502462865, + "epoch": 0.8990204523146571, + "grad_norm": 3.046875, + "learning_rate": 1.065947039973014e-06, + "loss": 1.8999542236328124, + "mean_token_accuracy": 0.5868043266236782, + "num_tokens": 38494320.0, + "step": 5610 + }, + { + "entropy": 1.8725532680749892, + "epoch": 0.900622984315218, + "grad_norm": 3.046875, + "learning_rate": 1.0490807893405296e-06, + "loss": 1.9284070968627929, + "mean_token_accuracy": 0.5785685375332832, + "num_tokens": 38565397.0, + "step": 5620 + }, + { + "entropy": 1.788800986111164, + "epoch": 0.902225516315779, + "grad_norm": 2.921875, + "learning_rate": 1.0322145387080453e-06, + "loss": 1.8518497467041015, + "mean_token_accuracy": 0.5960364241153002, + "num_tokens": 38631352.0, + "step": 5630 + }, + { + "entropy": 1.8663704261183738, + "epoch": 0.9038280483163398, + "grad_norm": 2.8125, + "learning_rate": 1.015348288075561e-06, + "loss": 1.9133312225341796, + "mean_token_accuracy": 0.580243082344532, + "num_tokens": 38703513.0, + "step": 5640 + }, + { + "entropy": 1.8696530893445016, + "epoch": 0.9054305803169007, + "grad_norm": 3.078125, + "learning_rate": 9.984820374430764e-07, + "loss": 1.9072153091430664, + "mean_token_accuracy": 0.5836617544293403, + "num_tokens": 38771109.0, + "step": 5650 + }, + { + "entropy": 1.8581979684531689, + "epoch": 0.9070331123174616, + "grad_norm": 3.203125, + "learning_rate": 9.81615786810592e-07, + "loss": 1.9353012084960937, + "mean_token_accuracy": 0.58640504963696, + "num_tokens": 38840595.0, + "step": 5660 + }, + { + "entropy": 1.8999485149979591, + "epoch": 0.9086356443180225, + "grad_norm": 2.796875, + "learning_rate": 9.647495361781077e-07, + "loss": 1.9293127059936523, + "mean_token_accuracy": 0.5741612739861012, + "num_tokens": 38911161.0, + "step": 5670 + }, + { + "entropy": 1.8909139916300775, + "epoch": 0.9102381763185834, + "grad_norm": 2.765625, + "learning_rate": 9.478832855456233e-07, + "loss": 1.943209457397461, + "mean_token_accuracy": 0.5803187564015388, + "num_tokens": 38979663.0, + "step": 5680 + }, + { + "entropy": 1.8345882877707482, + "epoch": 0.9118407083191442, + "grad_norm": 3.0625, + "learning_rate": 9.310170349131389e-07, + "loss": 1.8940544128417969, + "mean_token_accuracy": 0.5828323502093553, + "num_tokens": 39050266.0, + "step": 5690 + }, + { + "entropy": 1.8535967022180557, + "epoch": 0.9134432403197051, + "grad_norm": 2.859375, + "learning_rate": 9.141507842806545e-07, + "loss": 1.9206104278564453, + "mean_token_accuracy": 0.5844812471419573, + "num_tokens": 39118119.0, + "step": 5700 + }, + { + "entropy": 1.8310791112482547, + "epoch": 0.9150457723202661, + "grad_norm": 3.125, + "learning_rate": 8.9728453364817e-07, + "loss": 1.8925857543945312, + "mean_token_accuracy": 0.591274730116129, + "num_tokens": 39184397.0, + "step": 5710 + }, + { + "entropy": 1.8014666005969047, + "epoch": 0.9166483043208269, + "grad_norm": 2.8125, + "learning_rate": 8.804182830156857e-07, + "loss": 1.8689302444458007, + "mean_token_accuracy": 0.5962204236537219, + "num_tokens": 39250312.0, + "step": 5720 + }, + { + "entropy": 1.8697697766125203, + "epoch": 0.9182508363213878, + "grad_norm": 2.890625, + "learning_rate": 8.635520323832012e-07, + "loss": 1.9139211654663086, + "mean_token_accuracy": 0.5804957438260316, + "num_tokens": 39320105.0, + "step": 5730 + }, + { + "entropy": 1.8676650166511535, + "epoch": 0.9198533683219486, + "grad_norm": 3.0625, + "learning_rate": 8.466857817507169e-07, + "loss": 1.9182451248168946, + "mean_token_accuracy": 0.5818617131561041, + "num_tokens": 39387355.0, + "step": 5740 + }, + { + "entropy": 1.8362473480403423, + "epoch": 0.9214559003225096, + "grad_norm": 3.046875, + "learning_rate": 8.298195311182325e-07, + "loss": 1.8849479675292968, + "mean_token_accuracy": 0.5921859316527843, + "num_tokens": 39453933.0, + "step": 5750 + }, + { + "entropy": 1.8936782360076905, + "epoch": 0.9230584323230705, + "grad_norm": 3.765625, + "learning_rate": 8.12953280485748e-07, + "loss": 1.94251708984375, + "mean_token_accuracy": 0.5773557811975479, + "num_tokens": 39523210.0, + "step": 5760 + }, + { + "entropy": 1.8765916638076305, + "epoch": 0.9246609643236313, + "grad_norm": 3.09375, + "learning_rate": 7.960870298532637e-07, + "loss": 1.9312158584594727, + "mean_token_accuracy": 0.5803747341036797, + "num_tokens": 39595139.0, + "step": 5770 + }, + { + "entropy": 1.8852329470217228, + "epoch": 0.9262634963241922, + "grad_norm": 2.8125, + "learning_rate": 7.792207792207792e-07, + "loss": 1.9441871643066406, + "mean_token_accuracy": 0.5775437675416469, + "num_tokens": 39666767.0, + "step": 5780 + }, + { + "entropy": 1.8359283626079559, + "epoch": 0.9278660283247531, + "grad_norm": 2.828125, + "learning_rate": 7.62354528588295e-07, + "loss": 1.8917394638061524, + "mean_token_accuracy": 0.5900426283478737, + "num_tokens": 39733631.0, + "step": 5790 + }, + { + "entropy": 1.8818567633628844, + "epoch": 0.929468560325314, + "grad_norm": 3.03125, + "learning_rate": 7.454882779558105e-07, + "loss": 1.9617582321166993, + "mean_token_accuracy": 0.5813181817531585, + "num_tokens": 39803663.0, + "step": 5800 + }, + { + "entropy": 1.839282288402319, + "epoch": 0.9310710923258749, + "grad_norm": 2.90625, + "learning_rate": 7.28622027323326e-07, + "loss": 1.9438974380493164, + "mean_token_accuracy": 0.5880431465804576, + "num_tokens": 39874603.0, + "step": 5810 + }, + { + "entropy": 1.8572435699403287, + "epoch": 0.9326736243264357, + "grad_norm": 2.890625, + "learning_rate": 7.117557766908418e-07, + "loss": 1.9279142379760743, + "mean_token_accuracy": 0.5857914865016938, + "num_tokens": 39944298.0, + "step": 5820 + }, + { + "entropy": 1.832928830385208, + "epoch": 0.9342761563269967, + "grad_norm": 3.015625, + "learning_rate": 6.948895260583573e-07, + "loss": 1.8981988906860352, + "mean_token_accuracy": 0.5886461935937405, + "num_tokens": 40013681.0, + "step": 5830 + }, + { + "entropy": 1.8386280879378318, + "epoch": 0.9358786883275575, + "grad_norm": 2.75, + "learning_rate": 6.780232754258729e-07, + "loss": 1.8867168426513672, + "mean_token_accuracy": 0.5850611589848995, + "num_tokens": 40081484.0, + "step": 5840 + }, + { + "entropy": 1.826653690636158, + "epoch": 0.9374812203281184, + "grad_norm": 3.09375, + "learning_rate": 6.611570247933885e-07, + "loss": 1.8758953094482422, + "mean_token_accuracy": 0.5902330029755831, + "num_tokens": 40147306.0, + "step": 5850 + }, + { + "entropy": 1.8569243192672729, + "epoch": 0.9390837523286794, + "grad_norm": 2.6875, + "learning_rate": 6.442907741609041e-07, + "loss": 1.9039873123168944, + "mean_token_accuracy": 0.5841257013380527, + "num_tokens": 40217272.0, + "step": 5860 + }, + { + "entropy": 1.8387947618961333, + "epoch": 0.9406862843292402, + "grad_norm": 2.734375, + "learning_rate": 6.274245235284197e-07, + "loss": 1.9156251907348634, + "mean_token_accuracy": 0.5889201257377863, + "num_tokens": 40284855.0, + "step": 5870 + }, + { + "entropy": 1.8239695675671102, + "epoch": 0.9422888163298011, + "grad_norm": 2.96875, + "learning_rate": 6.105582728959353e-07, + "loss": 1.8879735946655274, + "mean_token_accuracy": 0.591039814800024, + "num_tokens": 40351023.0, + "step": 5880 + }, + { + "entropy": 1.7918000653386117, + "epoch": 0.9438913483303619, + "grad_norm": 2.96875, + "learning_rate": 5.936920222634509e-07, + "loss": 1.8323190689086915, + "mean_token_accuracy": 0.5958353541791439, + "num_tokens": 40415218.0, + "step": 5890 + }, + { + "entropy": 1.8455849036574363, + "epoch": 0.9454938803309229, + "grad_norm": 2.84375, + "learning_rate": 5.768257716309665e-07, + "loss": 1.8803943634033202, + "mean_token_accuracy": 0.5886748474091291, + "num_tokens": 40482300.0, + "step": 5900 + }, + { + "entropy": 1.8429814413189889, + "epoch": 0.9470964123314838, + "grad_norm": 2.765625, + "learning_rate": 5.599595209984821e-07, + "loss": 1.903822135925293, + "mean_token_accuracy": 0.590263594314456, + "num_tokens": 40549426.0, + "step": 5910 + }, + { + "entropy": 1.8786433458328247, + "epoch": 0.9486989443320446, + "grad_norm": 2.953125, + "learning_rate": 5.430932703659976e-07, + "loss": 1.9294485092163085, + "mean_token_accuracy": 0.5808563213795424, + "num_tokens": 40615905.0, + "step": 5920 + }, + { + "entropy": 1.8201900728046894, + "epoch": 0.9503014763326055, + "grad_norm": 2.984375, + "learning_rate": 5.262270197335133e-07, + "loss": 1.8631288528442382, + "mean_token_accuracy": 0.5932236719876528, + "num_tokens": 40682893.0, + "step": 5930 + }, + { + "entropy": 1.890999047458172, + "epoch": 0.9519040083331664, + "grad_norm": 3.0625, + "learning_rate": 5.093607691010289e-07, + "loss": 1.9362371444702149, + "mean_token_accuracy": 0.5777292806655169, + "num_tokens": 40750336.0, + "step": 5940 + }, + { + "entropy": 1.8011705093085766, + "epoch": 0.9535065403337273, + "grad_norm": 2.8125, + "learning_rate": 4.924945184685445e-07, + "loss": 1.8577068328857422, + "mean_token_accuracy": 0.5976326711475849, + "num_tokens": 40816136.0, + "step": 5950 + }, + { + "entropy": 1.8403367862105369, + "epoch": 0.9551090723342882, + "grad_norm": 2.9375, + "learning_rate": 4.7562826783606005e-07, + "loss": 1.9028356552124024, + "mean_token_accuracy": 0.5863358832895755, + "num_tokens": 40884407.0, + "step": 5960 + }, + { + "entropy": 1.8023689292371272, + "epoch": 0.956711604334849, + "grad_norm": 2.6875, + "learning_rate": 4.5876201720357566e-07, + "loss": 1.8804582595825194, + "mean_token_accuracy": 0.595867944508791, + "num_tokens": 40953133.0, + "step": 5970 + }, + { + "entropy": 1.8206961631774903, + "epoch": 0.95831413633541, + "grad_norm": 2.96875, + "learning_rate": 4.4189576657109127e-07, + "loss": 1.900848388671875, + "mean_token_accuracy": 0.5918670609593392, + "num_tokens": 41020901.0, + "step": 5980 + }, + { + "entropy": 1.831934504956007, + "epoch": 0.9599166683359708, + "grad_norm": 3.046875, + "learning_rate": 4.250295159386069e-07, + "loss": 1.8994543075561523, + "mean_token_accuracy": 0.588746365904808, + "num_tokens": 41089718.0, + "step": 5990 + }, + { + "entropy": 1.8424327656626702, + "epoch": 0.9615192003365317, + "grad_norm": 2.96875, + "learning_rate": 4.0816326530612243e-07, + "loss": 1.8773746490478516, + "mean_token_accuracy": 0.5854915887117386, + "num_tokens": 41159377.0, + "step": 6000 + }, + { + "entropy": 1.775068336725235, + "epoch": 0.9631217323370926, + "grad_norm": 3.015625, + "learning_rate": 3.912970146736381e-07, + "loss": 1.8501998901367187, + "mean_token_accuracy": 0.5993145342916251, + "num_tokens": 41224229.0, + "step": 6010 + }, + { + "entropy": 1.8535139806568623, + "epoch": 0.9647242643376535, + "grad_norm": 2.65625, + "learning_rate": 3.744307640411537e-07, + "loss": 1.9236631393432617, + "mean_token_accuracy": 0.5842952460050583, + "num_tokens": 41293701.0, + "step": 6020 + }, + { + "entropy": 1.8634035423398019, + "epoch": 0.9663267963382144, + "grad_norm": 2.796875, + "learning_rate": 3.575645134086693e-07, + "loss": 1.901199722290039, + "mean_token_accuracy": 0.5833870090544224, + "num_tokens": 41362831.0, + "step": 6030 + }, + { + "entropy": 1.8594874814152718, + "epoch": 0.9679293283387753, + "grad_norm": 2.828125, + "learning_rate": 3.406982627761849e-07, + "loss": 1.908275032043457, + "mean_token_accuracy": 0.5845970336347819, + "num_tokens": 41430854.0, + "step": 6040 + }, + { + "entropy": 1.8676169857382774, + "epoch": 0.9695318603393361, + "grad_norm": 2.875, + "learning_rate": 3.2383201214370046e-07, + "loss": 1.9279546737670898, + "mean_token_accuracy": 0.5803915828466415, + "num_tokens": 41500576.0, + "step": 6050 + }, + { + "entropy": 1.9107640534639359, + "epoch": 0.9711343923398971, + "grad_norm": 3.21875, + "learning_rate": 3.0696576151121607e-07, + "loss": 1.960128402709961, + "mean_token_accuracy": 0.5723039723932744, + "num_tokens": 41570197.0, + "step": 6060 + }, + { + "entropy": 1.852728360891342, + "epoch": 0.9727369243404579, + "grad_norm": 2.828125, + "learning_rate": 2.900995108787317e-07, + "loss": 1.9074199676513672, + "mean_token_accuracy": 0.5860687278211116, + "num_tokens": 41637859.0, + "step": 6070 + }, + { + "entropy": 1.8482899755239486, + "epoch": 0.9743394563410188, + "grad_norm": 2.875, + "learning_rate": 2.732332602462473e-07, + "loss": 1.9062208175659179, + "mean_token_accuracy": 0.5821232583373785, + "num_tokens": 41707538.0, + "step": 6080 + }, + { + "entropy": 1.8607835844159126, + "epoch": 0.9759419883415797, + "grad_norm": 2.734375, + "learning_rate": 2.563670096137629e-07, + "loss": 1.9100791931152343, + "mean_token_accuracy": 0.5810363296419382, + "num_tokens": 41777379.0, + "step": 6090 + }, + { + "entropy": 1.8598943777382373, + "epoch": 0.9775445203421406, + "grad_norm": 2.859375, + "learning_rate": 2.395007589812785e-07, + "loss": 1.9228595733642577, + "mean_token_accuracy": 0.5857084028422832, + "num_tokens": 41843062.0, + "step": 6100 + }, + { + "entropy": 1.8349303409457207, + "epoch": 0.9791470523427015, + "grad_norm": 2.875, + "learning_rate": 2.2263450834879408e-07, + "loss": 1.8791229248046875, + "mean_token_accuracy": 0.590753136947751, + "num_tokens": 41910227.0, + "step": 6110 + }, + { + "entropy": 1.7299993604421615, + "epoch": 0.9807495843432623, + "grad_norm": 2.890625, + "learning_rate": 2.057682577163097e-07, + "loss": 1.8145219802856445, + "mean_token_accuracy": 0.6077951058745384, + "num_tokens": 41974421.0, + "step": 6120 + }, + { + "entropy": 1.9011049136519431, + "epoch": 0.9823521163438232, + "grad_norm": 2.9375, + "learning_rate": 1.8890200708382527e-07, + "loss": 1.9380456924438476, + "mean_token_accuracy": 0.5766681145876646, + "num_tokens": 42045797.0, + "step": 6130 + }, + { + "entropy": 1.8033956550061703, + "epoch": 0.9839546483443842, + "grad_norm": 2.8125, + "learning_rate": 1.7203575645134088e-07, + "loss": 1.8644264221191407, + "mean_token_accuracy": 0.5904493033885956, + "num_tokens": 42112332.0, + "step": 6140 + }, + { + "entropy": 1.8577637001872063, + "epoch": 0.985557180344945, + "grad_norm": 3.25, + "learning_rate": 1.5516950581885649e-07, + "loss": 1.9165401458740234, + "mean_token_accuracy": 0.5841254234313965, + "num_tokens": 42181504.0, + "step": 6150 + }, + { + "entropy": 1.8639202922582627, + "epoch": 0.9871597123455059, + "grad_norm": 3.09375, + "learning_rate": 1.383032551863721e-07, + "loss": 1.9187042236328125, + "mean_token_accuracy": 0.5859580975025892, + "num_tokens": 42247650.0, + "step": 6160 + }, + { + "entropy": 1.9042673364281655, + "epoch": 0.9887622443460667, + "grad_norm": 3.109375, + "learning_rate": 1.2143700455388767e-07, + "loss": 1.9495109558105468, + "mean_token_accuracy": 0.5765886418521404, + "num_tokens": 42318033.0, + "step": 6170 + }, + { + "entropy": 1.8380866006016732, + "epoch": 0.9903647763466277, + "grad_norm": 2.6875, + "learning_rate": 1.0457075392140328e-07, + "loss": 1.9063236236572265, + "mean_token_accuracy": 0.5875726152211428, + "num_tokens": 42386158.0, + "step": 6180 + }, + { + "entropy": 1.871013981103897, + "epoch": 0.9919673083471886, + "grad_norm": 2.921875, + "learning_rate": 8.770450328891888e-08, + "loss": 1.9296409606933593, + "mean_token_accuracy": 0.5827766362577677, + "num_tokens": 42456203.0, + "step": 6190 + }, + { + "entropy": 1.84286737293005, + "epoch": 0.9935698403477494, + "grad_norm": 3.1875, + "learning_rate": 7.083825265643448e-08, + "loss": 1.9174055099487304, + "mean_token_accuracy": 0.5898357950150966, + "num_tokens": 42523293.0, + "step": 6200 + }, + { + "entropy": 1.8976394981145859, + "epoch": 0.9951723723483104, + "grad_norm": 2.875, + "learning_rate": 5.397200202395008e-08, + "loss": 1.966810417175293, + "mean_token_accuracy": 0.5724526148289442, + "num_tokens": 42595973.0, + "step": 6210 + }, + { + "entropy": 1.8372694373130798, + "epoch": 0.9967749043488712, + "grad_norm": 2.984375, + "learning_rate": 3.710575139146568e-08, + "loss": 1.8860355377197267, + "mean_token_accuracy": 0.5865787580609322, + "num_tokens": 42663904.0, + "step": 6220 + }, + { + "entropy": 1.8298647806048394, + "epoch": 0.9983774363494321, + "grad_norm": 2.5625, + "learning_rate": 2.023950075898128e-08, + "loss": 1.8837800979614259, + "mean_token_accuracy": 0.5879415862262249, + "num_tokens": 42732619.0, + "step": 6230 + }, + { + "entropy": 1.8752723596990108, + "epoch": 0.999979968349993, + "grad_norm": 2.984375, + "learning_rate": 3.37325012649688e-09, + "loss": 1.9457399368286132, + "mean_token_accuracy": 0.5779070716351271, + "num_tokens": 42805759.0, + "step": 6240 + } + ], + "logging_steps": 10, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.338653249407058e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-6241/training_args.bin b/checkpoint-6241/training_args.bin new file mode 100644 index 0000000..496fced --- /dev/null +++ b/checkpoint-6241/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203999aa8f9c803df98193e503cdab1a4f9929e25d137d21695250f74c2fa7ad +size 5713 diff --git a/checkpoint-7136/chat_template.jinja b/checkpoint-7136/chat_template.jinja new file mode 100644 index 0000000..755ba94 --- /dev/null +++ b/checkpoint-7136/chat_template.jinja @@ -0,0 +1,13 @@ +{%- if messages[0]['role'] == 'system' %} + {%- set loop_messages = messages[1:] %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} +{%- else %} + {%- set loop_messages = messages %} + {{- '<|im_start|>system\n' + "You are a knowledgeable assistant. For multiple-choice questions, write ONLY the letter of the correct answer inside \\boxed{}." + '<|im_end|>\n' }} +{%- endif %} +{%- for message in loop_messages %} + {{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>\n' }} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-7136/config.json b/checkpoint-7136/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-7136/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-7136/generation_config.json b/checkpoint-7136/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-7136/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-7136/model.safetensors b/checkpoint-7136/model.safetensors new file mode 100644 index 0000000..0bc0212 --- /dev/null +++ b/checkpoint-7136/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:530f552bb2a5eaf1eaa4ff6ca6f9e4197c9f515754ef8fe0e7c6526fedd160d6 +size 3441185608 diff --git a/checkpoint-7136/optimizer.pt b/checkpoint-7136/optimizer.pt new file mode 100644 index 0000000..53327f3 --- /dev/null +++ b/checkpoint-7136/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a2433f98e74e6c0a46cff3df29cb206fbe0d750801ef640c8d3379f56c20316 +size 6882572207 diff --git a/checkpoint-7136/rng_state.pth b/checkpoint-7136/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-7136/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-7136/scheduler.pt b/checkpoint-7136/scheduler.pt new file mode 100644 index 0000000..b30989c --- /dev/null +++ b/checkpoint-7136/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77b2bcd6a6fe7f9904d15dd735af201ee67fe2aaf90996e4ebb4a9c178aacf7d +size 1465 diff --git a/checkpoint-7136/tokenizer.json b/checkpoint-7136/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-7136/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-7136/tokenizer_config.json b/checkpoint-7136/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-7136/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-7136/trainer_state.json b/checkpoint-7136/trainer_state.json new file mode 100644 index 0000000..3ce6a37 --- /dev/null +++ b/checkpoint-7136/trainer_state.json @@ -0,0 +1,7164 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 7136, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.1749843299388885, + "epoch": 0.0014013943874154784, + "grad_norm": 77.5, + "learning_rate": 1.264044943820225e-07, + "loss": 3.7286407470703127, + "mean_token_accuracy": 0.4398000232875347, + "num_tokens": 58634.0, + "step": 10 + }, + { + "entropy": 1.138375636935234, + "epoch": 0.002802788774830957, + "grad_norm": 89.0, + "learning_rate": 2.668539325842697e-07, + "loss": 3.726331329345703, + "mean_token_accuracy": 0.44621490836143496, + "num_tokens": 117434.0, + "step": 20 + }, + { + "entropy": 1.1546230360865593, + "epoch": 0.004204183162246435, + "grad_norm": 77.0, + "learning_rate": 4.073033707865169e-07, + "loss": 3.6857357025146484, + "mean_token_accuracy": 0.44239638149738314, + "num_tokens": 177224.0, + "step": 30 + }, + { + "entropy": 1.165491023659706, + "epoch": 0.005605577549661914, + "grad_norm": 81.0, + "learning_rate": 5.477528089887641e-07, + "loss": 3.7168804168701173, + "mean_token_accuracy": 0.441532301902771, + "num_tokens": 238330.0, + "step": 40 + }, + { + "entropy": 1.2151292562484741, + "epoch": 0.007006971937077392, + "grad_norm": 65.0, + "learning_rate": 6.882022471910113e-07, + "loss": 3.7297904968261717, + "mean_token_accuracy": 0.4368342585861683, + "num_tokens": 296693.0, + "step": 50 + }, + { + "entropy": 1.2036117509007453, + "epoch": 0.00840836632449287, + "grad_norm": 63.0, + "learning_rate": 8.286516853932584e-07, + "loss": 3.6518295288085936, + "mean_token_accuracy": 0.44340850710868834, + "num_tokens": 355315.0, + "step": 60 + }, + { + "entropy": 1.217386195063591, + "epoch": 0.009809760711908349, + "grad_norm": 59.0, + "learning_rate": 9.691011235955058e-07, + "loss": 3.5424434661865236, + "mean_token_accuracy": 0.44781318232417106, + "num_tokens": 414287.0, + "step": 70 + }, + { + "entropy": 1.2282691702246666, + "epoch": 0.011211155099323827, + "grad_norm": 57.0, + "learning_rate": 1.1095505617977528e-06, + "loss": 3.553877258300781, + "mean_token_accuracy": 0.4493464961647987, + "num_tokens": 470247.0, + "step": 80 + }, + { + "entropy": 1.2959989964962007, + "epoch": 0.012612549486739306, + "grad_norm": 47.25, + "learning_rate": 1.25e-06, + "loss": 3.4337196350097656, + "mean_token_accuracy": 0.4504865989089012, + "num_tokens": 530809.0, + "step": 90 + }, + { + "entropy": 1.3231378316879272, + "epoch": 0.014013943874154784, + "grad_norm": 42.5, + "learning_rate": 1.3904494382022474e-06, + "loss": 3.357102966308594, + "mean_token_accuracy": 0.46152718737721443, + "num_tokens": 589432.0, + "step": 100 + }, + { + "entropy": 1.376977691054344, + "epoch": 0.015415338261570263, + "grad_norm": 30.75, + "learning_rate": 1.5308988764044946e-06, + "loss": 3.2706432342529297, + "mean_token_accuracy": 0.4622736141085625, + "num_tokens": 647158.0, + "step": 110 + }, + { + "entropy": 1.4021880328655243, + "epoch": 0.01681673264898574, + "grad_norm": 25.0, + "learning_rate": 1.6713483146067417e-06, + "loss": 3.1713953018188477, + "mean_token_accuracy": 0.4704868413507938, + "num_tokens": 707400.0, + "step": 120 + }, + { + "entropy": 1.436927217245102, + "epoch": 0.01821812703640122, + "grad_norm": 27.375, + "learning_rate": 1.811797752808989e-06, + "loss": 3.0929262161254885, + "mean_token_accuracy": 0.47034821286797523, + "num_tokens": 764052.0, + "step": 130 + }, + { + "entropy": 1.5048964083194734, + "epoch": 0.019619521423816698, + "grad_norm": 18.875, + "learning_rate": 1.9522471910112362e-06, + "loss": 3.021409606933594, + "mean_token_accuracy": 0.4734907761216164, + "num_tokens": 819260.0, + "step": 140 + }, + { + "entropy": 1.6020113319158553, + "epoch": 0.021020915811232176, + "grad_norm": 18.25, + "learning_rate": 2.0926966292134835e-06, + "loss": 2.9581523895263673, + "mean_token_accuracy": 0.46810011118650435, + "num_tokens": 878716.0, + "step": 150 + }, + { + "entropy": 1.6423213809728623, + "epoch": 0.022422310198647655, + "grad_norm": 16.0, + "learning_rate": 2.2331460674157303e-06, + "loss": 2.784856414794922, + "mean_token_accuracy": 0.4787336468696594, + "num_tokens": 938918.0, + "step": 160 + }, + { + "entropy": 1.6930634200572967, + "epoch": 0.023823704586063133, + "grad_norm": 13.4375, + "learning_rate": 2.3735955056179776e-06, + "loss": 2.6741771697998047, + "mean_token_accuracy": 0.48545088768005373, + "num_tokens": 996413.0, + "step": 170 + }, + { + "entropy": 1.764124980568886, + "epoch": 0.02522509897347861, + "grad_norm": 12.75, + "learning_rate": 2.514044943820225e-06, + "loss": 2.544516754150391, + "mean_token_accuracy": 0.4911739453673363, + "num_tokens": 1055251.0, + "step": 180 + }, + { + "entropy": 1.7424744457006454, + "epoch": 0.02662649336089409, + "grad_norm": 9.75, + "learning_rate": 2.654494382022472e-06, + "loss": 2.343752861022949, + "mean_token_accuracy": 0.5158485703170299, + "num_tokens": 1111837.0, + "step": 190 + }, + { + "entropy": 1.787026074528694, + "epoch": 0.02802788774830957, + "grad_norm": 8.5625, + "learning_rate": 2.7949438202247194e-06, + "loss": 2.3229976654052735, + "mean_token_accuracy": 0.5173665106296539, + "num_tokens": 1172000.0, + "step": 200 + }, + { + "entropy": 1.845227986574173, + "epoch": 0.029429282135725047, + "grad_norm": 7.28125, + "learning_rate": 2.9353932584269666e-06, + "loss": 2.2990942001342773, + "mean_token_accuracy": 0.5192534171044827, + "num_tokens": 1231079.0, + "step": 210 + }, + { + "entropy": 1.8506489276885987, + "epoch": 0.030830676523140525, + "grad_norm": 6.625, + "learning_rate": 3.075842696629214e-06, + "loss": 2.2143314361572264, + "mean_token_accuracy": 0.527560542523861, + "num_tokens": 1291223.0, + "step": 220 + }, + { + "entropy": 1.976783648133278, + "epoch": 0.032232070910556, + "grad_norm": 6.75, + "learning_rate": 3.2162921348314607e-06, + "loss": 2.259410285949707, + "mean_token_accuracy": 0.520425470918417, + "num_tokens": 1349689.0, + "step": 230 + }, + { + "entropy": 1.9874911963939668, + "epoch": 0.03363346529797148, + "grad_norm": 5.96875, + "learning_rate": 3.3567415730337084e-06, + "loss": 2.2152097702026365, + "mean_token_accuracy": 0.5251795709133148, + "num_tokens": 1413724.0, + "step": 240 + }, + { + "entropy": 2.0247445702552795, + "epoch": 0.03503485968538696, + "grad_norm": 7.34375, + "learning_rate": 3.4971910112359553e-06, + "loss": 2.192658042907715, + "mean_token_accuracy": 0.5360996119678021, + "num_tokens": 1470666.0, + "step": 250 + }, + { + "entropy": 2.0707518488168715, + "epoch": 0.03643625407280244, + "grad_norm": 5.375, + "learning_rate": 3.637640449438202e-06, + "loss": 2.203626823425293, + "mean_token_accuracy": 0.5276296675205231, + "num_tokens": 1531575.0, + "step": 260 + }, + { + "entropy": 2.0428607910871506, + "epoch": 0.037837648460217914, + "grad_norm": 4.875, + "learning_rate": 3.77808988764045e-06, + "loss": 2.181136894226074, + "mean_token_accuracy": 0.5354902669787407, + "num_tokens": 1589503.0, + "step": 270 + }, + { + "entropy": 2.094152969121933, + "epoch": 0.039239042847633396, + "grad_norm": 4.25, + "learning_rate": 3.918539325842697e-06, + "loss": 2.2115604400634767, + "mean_token_accuracy": 0.5282930836081505, + "num_tokens": 1650565.0, + "step": 280 + }, + { + "entropy": 2.070644298195839, + "epoch": 0.04064043723504887, + "grad_norm": 4.6875, + "learning_rate": 4.058988764044944e-06, + "loss": 2.190484809875488, + "mean_token_accuracy": 0.5283051446080208, + "num_tokens": 1712022.0, + "step": 290 + }, + { + "entropy": 2.076260048151016, + "epoch": 0.04204183162246435, + "grad_norm": 3.9375, + "learning_rate": 4.199438202247192e-06, + "loss": 2.1462358474731444, + "mean_token_accuracy": 0.533646783977747, + "num_tokens": 1777142.0, + "step": 300 + }, + { + "entropy": 2.0128800898790358, + "epoch": 0.04344322600987983, + "grad_norm": 3.578125, + "learning_rate": 4.339887640449438e-06, + "loss": 2.0999853134155275, + "mean_token_accuracy": 0.5459584936499595, + "num_tokens": 1833136.0, + "step": 310 + }, + { + "entropy": 1.9699875265359879, + "epoch": 0.04484462039729531, + "grad_norm": 3.609375, + "learning_rate": 4.480337078651686e-06, + "loss": 2.054058837890625, + "mean_token_accuracy": 0.548506161570549, + "num_tokens": 1890622.0, + "step": 320 + }, + { + "entropy": 2.0611583977937697, + "epoch": 0.046246014784710784, + "grad_norm": 3.640625, + "learning_rate": 4.6207865168539325e-06, + "loss": 2.1545442581176757, + "mean_token_accuracy": 0.5369547970592976, + "num_tokens": 1950235.0, + "step": 330 + }, + { + "entropy": 2.0399067103862762, + "epoch": 0.047647409172126266, + "grad_norm": 4.03125, + "learning_rate": 4.761235955056181e-06, + "loss": 2.139397621154785, + "mean_token_accuracy": 0.5447163872420788, + "num_tokens": 2008662.0, + "step": 340 + }, + { + "entropy": 1.9834542274475098, + "epoch": 0.04904880355954174, + "grad_norm": 3.5, + "learning_rate": 4.901685393258427e-06, + "loss": 2.0456113815307617, + "mean_token_accuracy": 0.5515513971447945, + "num_tokens": 2065547.0, + "step": 350 + }, + { + "entropy": 1.9932841449975967, + "epoch": 0.05045019794695722, + "grad_norm": 3.71875, + "learning_rate": 4.9977876106194695e-06, + "loss": 2.0840469360351563, + "mean_token_accuracy": 0.5537005968391895, + "num_tokens": 2122846.0, + "step": 360 + }, + { + "entropy": 1.9792243987321854, + "epoch": 0.0518515923343727, + "grad_norm": 3.765625, + "learning_rate": 4.990412979351033e-06, + "loss": 2.048763084411621, + "mean_token_accuracy": 0.5508734963834285, + "num_tokens": 2176941.0, + "step": 370 + }, + { + "entropy": 1.9960775971412659, + "epoch": 0.05325298672178818, + "grad_norm": 3.890625, + "learning_rate": 4.983038348082596e-06, + "loss": 2.0834062576293944, + "mean_token_accuracy": 0.5490451008081436, + "num_tokens": 2234758.0, + "step": 380 + }, + { + "entropy": 2.0350757420063017, + "epoch": 0.054654381109203655, + "grad_norm": 3.640625, + "learning_rate": 4.975663716814159e-06, + "loss": 2.102910041809082, + "mean_token_accuracy": 0.5434986114501953, + "num_tokens": 2291715.0, + "step": 390 + }, + { + "entropy": 2.029018145799637, + "epoch": 0.05605577549661914, + "grad_norm": 3.53125, + "learning_rate": 4.9682890855457235e-06, + "loss": 2.0678232192993162, + "mean_token_accuracy": 0.5505224116146564, + "num_tokens": 2350005.0, + "step": 400 + }, + { + "entropy": 2.0029339730739593, + "epoch": 0.05745716988403461, + "grad_norm": 4.46875, + "learning_rate": 4.960914454277287e-06, + "loss": 2.094471740722656, + "mean_token_accuracy": 0.5440472796559334, + "num_tokens": 2407677.0, + "step": 410 + }, + { + "entropy": 2.0633452206850054, + "epoch": 0.058858564271450094, + "grad_norm": 3.484375, + "learning_rate": 4.95353982300885e-06, + "loss": 2.13787899017334, + "mean_token_accuracy": 0.5416313037276268, + "num_tokens": 2467567.0, + "step": 420 + }, + { + "entropy": 1.9949743568897247, + "epoch": 0.06025995865886557, + "grad_norm": 4.28125, + "learning_rate": 4.946165191740413e-06, + "loss": 2.062998962402344, + "mean_token_accuracy": 0.547588687390089, + "num_tokens": 2525614.0, + "step": 430 + }, + { + "entropy": 2.019722428917885, + "epoch": 0.06166135304628105, + "grad_norm": 3.46875, + "learning_rate": 4.938790560471977e-06, + "loss": 2.0874080657958984, + "mean_token_accuracy": 0.5463835179805756, + "num_tokens": 2582910.0, + "step": 440 + }, + { + "entropy": 1.9916650235652924, + "epoch": 0.06306274743369653, + "grad_norm": 3.8125, + "learning_rate": 4.93141592920354e-06, + "loss": 2.0598018646240233, + "mean_token_accuracy": 0.5500055000185966, + "num_tokens": 2640183.0, + "step": 450 + }, + { + "entropy": 2.0060368895530702, + "epoch": 0.064464141821112, + "grad_norm": 3.59375, + "learning_rate": 4.924041297935104e-06, + "loss": 2.0795265197753907, + "mean_token_accuracy": 0.5481240846216678, + "num_tokens": 2698117.0, + "step": 460 + }, + { + "entropy": 2.033749130368233, + "epoch": 0.06586553620852749, + "grad_norm": 3.75, + "learning_rate": 4.9166666666666665e-06, + "loss": 2.095734405517578, + "mean_token_accuracy": 0.5404889442026615, + "num_tokens": 2756561.0, + "step": 470 + }, + { + "entropy": 2.0706971049308778, + "epoch": 0.06726693059594296, + "grad_norm": 3.390625, + "learning_rate": 4.909292035398231e-06, + "loss": 2.1171714782714846, + "mean_token_accuracy": 0.5364310555160046, + "num_tokens": 2818553.0, + "step": 480 + }, + { + "entropy": 2.058588495850563, + "epoch": 0.06866832498335844, + "grad_norm": 4.1875, + "learning_rate": 4.901917404129794e-06, + "loss": 2.1368818283081055, + "mean_token_accuracy": 0.5373388394713402, + "num_tokens": 2879555.0, + "step": 490 + }, + { + "entropy": 2.064988616108894, + "epoch": 0.07006971937077391, + "grad_norm": 3.78125, + "learning_rate": 4.894542772861358e-06, + "loss": 2.1231252670288088, + "mean_token_accuracy": 0.5416132740676403, + "num_tokens": 2936270.0, + "step": 500 + }, + { + "entropy": 2.0648070216178893, + "epoch": 0.0714711137581894, + "grad_norm": 3.703125, + "learning_rate": 4.8871681415929204e-06, + "loss": 2.1406299591064455, + "mean_token_accuracy": 0.5388594828546047, + "num_tokens": 2993931.0, + "step": 510 + }, + { + "entropy": 2.065402030944824, + "epoch": 0.07287250814560488, + "grad_norm": 3.609375, + "learning_rate": 4.879793510324485e-06, + "loss": 2.1110891342163085, + "mean_token_accuracy": 0.5408674567937851, + "num_tokens": 3052231.0, + "step": 520 + }, + { + "entropy": 2.008076322078705, + "epoch": 0.07427390253302035, + "grad_norm": 3.71875, + "learning_rate": 4.872418879056048e-06, + "loss": 2.0839359283447267, + "mean_token_accuracy": 0.5448177300393582, + "num_tokens": 3113651.0, + "step": 530 + }, + { + "entropy": 2.0742378741502763, + "epoch": 0.07567529692043583, + "grad_norm": 3.453125, + "learning_rate": 4.865044247787611e-06, + "loss": 2.1109527587890624, + "mean_token_accuracy": 0.5407277189195157, + "num_tokens": 3175338.0, + "step": 540 + }, + { + "entropy": 2.048898476362228, + "epoch": 0.07707669130785132, + "grad_norm": 4.1875, + "learning_rate": 4.8576696165191744e-06, + "loss": 2.106137275695801, + "mean_token_accuracy": 0.546953809261322, + "num_tokens": 3233510.0, + "step": 550 + }, + { + "entropy": 2.084382873773575, + "epoch": 0.07847808569526679, + "grad_norm": 3.4375, + "learning_rate": 4.850294985250738e-06, + "loss": 2.1437379837036135, + "mean_token_accuracy": 0.5365074597299099, + "num_tokens": 3294641.0, + "step": 560 + }, + { + "entropy": 2.0615872770547865, + "epoch": 0.07987948008268227, + "grad_norm": 4.21875, + "learning_rate": 4.842920353982301e-06, + "loss": 2.1082529067993163, + "mean_token_accuracy": 0.5400469101965427, + "num_tokens": 3354895.0, + "step": 570 + }, + { + "entropy": 2.0372446924448013, + "epoch": 0.08128087447009774, + "grad_norm": 3.875, + "learning_rate": 4.835545722713864e-06, + "loss": 2.0846746444702147, + "mean_token_accuracy": 0.542639608681202, + "num_tokens": 3415190.0, + "step": 580 + }, + { + "entropy": 2.0401179790496826, + "epoch": 0.08268226885751323, + "grad_norm": 3.65625, + "learning_rate": 4.8281710914454284e-06, + "loss": 2.0969858169555664, + "mean_token_accuracy": 0.5441658839583396, + "num_tokens": 3474415.0, + "step": 590 + }, + { + "entropy": 2.009382280707359, + "epoch": 0.0840836632449287, + "grad_norm": 3.75, + "learning_rate": 4.820796460176992e-06, + "loss": 2.0534940719604493, + "mean_token_accuracy": 0.5497397676110267, + "num_tokens": 3531386.0, + "step": 600 + }, + { + "entropy": 1.959547182917595, + "epoch": 0.08548505763234418, + "grad_norm": 4.0625, + "learning_rate": 4.813421828908555e-06, + "loss": 2.0382083892822265, + "mean_token_accuracy": 0.5582435041666031, + "num_tokens": 3587230.0, + "step": 610 + }, + { + "entropy": 2.0483379155397414, + "epoch": 0.08688645201975966, + "grad_norm": 3.359375, + "learning_rate": 4.806047197640118e-06, + "loss": 2.0792516708374023, + "mean_token_accuracy": 0.5420335404574871, + "num_tokens": 3649935.0, + "step": 620 + }, + { + "entropy": 2.048970940709114, + "epoch": 0.08828784640717514, + "grad_norm": 4.375, + "learning_rate": 4.7986725663716816e-06, + "loss": 2.0989675521850586, + "mean_token_accuracy": 0.5420025557279586, + "num_tokens": 3708420.0, + "step": 630 + }, + { + "entropy": 1.942202365398407, + "epoch": 0.08968924079459062, + "grad_norm": 3.390625, + "learning_rate": 4.791297935103245e-06, + "loss": 2.0210729598999024, + "mean_token_accuracy": 0.560210132598877, + "num_tokens": 3761875.0, + "step": 640 + }, + { + "entropy": 2.0344078302383424, + "epoch": 0.0910906351820061, + "grad_norm": 3.53125, + "learning_rate": 4.783923303834809e-06, + "loss": 2.0858205795288085, + "mean_token_accuracy": 0.5398732639849186, + "num_tokens": 3820824.0, + "step": 650 + }, + { + "entropy": 2.085204502940178, + "epoch": 0.09249202956942157, + "grad_norm": 4.15625, + "learning_rate": 4.776548672566372e-06, + "loss": 2.1287776947021486, + "mean_token_accuracy": 0.5379412017762661, + "num_tokens": 3881060.0, + "step": 660 + }, + { + "entropy": 2.029499676823616, + "epoch": 0.09389342395683706, + "grad_norm": 3.8125, + "learning_rate": 4.7691740412979356e-06, + "loss": 2.0897621154785155, + "mean_token_accuracy": 0.5459394969046116, + "num_tokens": 3937415.0, + "step": 670 + }, + { + "entropy": 2.04031979739666, + "epoch": 0.09529481834425253, + "grad_norm": 3.296875, + "learning_rate": 4.761799410029499e-06, + "loss": 2.08218936920166, + "mean_token_accuracy": 0.5483398318290711, + "num_tokens": 3996057.0, + "step": 680 + }, + { + "entropy": 2.031023436784744, + "epoch": 0.09669621273166801, + "grad_norm": 3.96875, + "learning_rate": 4.754424778761063e-06, + "loss": 2.074525260925293, + "mean_token_accuracy": 0.5459690198302269, + "num_tokens": 4055201.0, + "step": 690 + }, + { + "entropy": 2.017417848110199, + "epoch": 0.09809760711908348, + "grad_norm": 3.625, + "learning_rate": 4.747050147492625e-06, + "loss": 2.0694021224975585, + "mean_token_accuracy": 0.5547117449343204, + "num_tokens": 4111854.0, + "step": 700 + }, + { + "entropy": 2.0284538745880125, + "epoch": 0.09949900150649897, + "grad_norm": 3.578125, + "learning_rate": 4.7396755162241895e-06, + "loss": 2.127583694458008, + "mean_token_accuracy": 0.5464184492826462, + "num_tokens": 4171738.0, + "step": 710 + }, + { + "entropy": 1.997345969080925, + "epoch": 0.10090039589391445, + "grad_norm": 3.4375, + "learning_rate": 4.732300884955753e-06, + "loss": 2.038435363769531, + "mean_token_accuracy": 0.5501852035522461, + "num_tokens": 4229220.0, + "step": 720 + }, + { + "entropy": 2.0234489142894745, + "epoch": 0.10230179028132992, + "grad_norm": 3.71875, + "learning_rate": 4.724926253687316e-06, + "loss": 2.063991928100586, + "mean_token_accuracy": 0.5484826415777206, + "num_tokens": 4287264.0, + "step": 730 + }, + { + "entropy": 2.0313188999891283, + "epoch": 0.1037031846687454, + "grad_norm": 3.375, + "learning_rate": 4.717551622418879e-06, + "loss": 2.1104171752929686, + "mean_token_accuracy": 0.5436050571501255, + "num_tokens": 4347435.0, + "step": 740 + }, + { + "entropy": 2.058633345365524, + "epoch": 0.10510457905616089, + "grad_norm": 3.75, + "learning_rate": 4.710176991150443e-06, + "loss": 2.112974739074707, + "mean_token_accuracy": 0.5412231512367726, + "num_tokens": 4408019.0, + "step": 750 + }, + { + "entropy": 2.0089460521936418, + "epoch": 0.10650597344357636, + "grad_norm": 3.625, + "learning_rate": 4.702802359882006e-06, + "loss": 2.0665733337402346, + "mean_token_accuracy": 0.5501148752868176, + "num_tokens": 4465697.0, + "step": 760 + }, + { + "entropy": 1.9932250171899795, + "epoch": 0.10790736783099183, + "grad_norm": 3.515625, + "learning_rate": 4.695427728613569e-06, + "loss": 2.0313592910766602, + "mean_token_accuracy": 0.5506305918097496, + "num_tokens": 4523186.0, + "step": 770 + }, + { + "entropy": 2.016455405950546, + "epoch": 0.10930876221840731, + "grad_norm": 3.1875, + "learning_rate": 4.688053097345133e-06, + "loss": 2.0808271408081054, + "mean_token_accuracy": 0.5452144391834736, + "num_tokens": 4581461.0, + "step": 780 + }, + { + "entropy": 2.0657077699899675, + "epoch": 0.1107101566058228, + "grad_norm": 3.703125, + "learning_rate": 4.680678466076697e-06, + "loss": 2.089410591125488, + "mean_token_accuracy": 0.5414080828428268, + "num_tokens": 4643576.0, + "step": 790 + }, + { + "entropy": 1.9475072085857392, + "epoch": 0.11211155099323827, + "grad_norm": 3.796875, + "learning_rate": 4.67330383480826e-06, + "loss": 2.022770309448242, + "mean_token_accuracy": 0.5502500668168068, + "num_tokens": 4701452.0, + "step": 800 + }, + { + "entropy": 2.0099977761507035, + "epoch": 0.11351294538065375, + "grad_norm": 3.953125, + "learning_rate": 4.665929203539823e-06, + "loss": 2.044535255432129, + "mean_token_accuracy": 0.5522691115736962, + "num_tokens": 4755652.0, + "step": 810 + }, + { + "entropy": 2.032015699148178, + "epoch": 0.11491433976806922, + "grad_norm": 3.890625, + "learning_rate": 4.658554572271387e-06, + "loss": 2.090387153625488, + "mean_token_accuracy": 0.5472030349075794, + "num_tokens": 4815215.0, + "step": 820 + }, + { + "entropy": 2.002387708425522, + "epoch": 0.11631573415548471, + "grad_norm": 3.171875, + "learning_rate": 4.65117994100295e-06, + "loss": 2.041549301147461, + "mean_token_accuracy": 0.5508693747222424, + "num_tokens": 4872654.0, + "step": 830 + }, + { + "entropy": 2.057389384508133, + "epoch": 0.11771712854290019, + "grad_norm": 3.578125, + "learning_rate": 4.643805309734514e-06, + "loss": 2.098028564453125, + "mean_token_accuracy": 0.5409851305186748, + "num_tokens": 4932258.0, + "step": 840 + }, + { + "entropy": 2.0524306416511537, + "epoch": 0.11911852293031566, + "grad_norm": 3.40625, + "learning_rate": 4.636430678466077e-06, + "loss": 2.0935876846313475, + "mean_token_accuracy": 0.5410212010145188, + "num_tokens": 4992277.0, + "step": 850 + }, + { + "entropy": 2.0131456166505814, + "epoch": 0.12051991731773114, + "grad_norm": 3.84375, + "learning_rate": 4.6290560471976405e-06, + "loss": 2.0590002059936525, + "mean_token_accuracy": 0.5476037085056304, + "num_tokens": 5049601.0, + "step": 860 + }, + { + "entropy": 2.0318191319704058, + "epoch": 0.12192131170514663, + "grad_norm": 3.359375, + "learning_rate": 4.621681415929204e-06, + "loss": 2.0884126663208007, + "mean_token_accuracy": 0.5430484347045421, + "num_tokens": 5108748.0, + "step": 870 + }, + { + "entropy": 2.083487269282341, + "epoch": 0.1233227060925621, + "grad_norm": 3.171875, + "learning_rate": 4.614306784660768e-06, + "loss": 2.1179590225219727, + "mean_token_accuracy": 0.5432230710983277, + "num_tokens": 5168839.0, + "step": 880 + }, + { + "entropy": 2.0145327717065813, + "epoch": 0.12472410047997758, + "grad_norm": 3.75, + "learning_rate": 4.60693215339233e-06, + "loss": 2.07696475982666, + "mean_token_accuracy": 0.5457107946276665, + "num_tokens": 5227340.0, + "step": 890 + }, + { + "entropy": 2.035206711292267, + "epoch": 0.12612549486739305, + "grad_norm": 3.78125, + "learning_rate": 4.5995575221238945e-06, + "loss": 2.0905691146850587, + "mean_token_accuracy": 0.5446631409227848, + "num_tokens": 5285854.0, + "step": 900 + }, + { + "entropy": 2.0117908298969267, + "epoch": 0.12752688925480854, + "grad_norm": 3.703125, + "learning_rate": 4.592182890855458e-06, + "loss": 2.0560426712036133, + "mean_token_accuracy": 0.5508834436535835, + "num_tokens": 5341221.0, + "step": 910 + }, + { + "entropy": 2.0344759225845337, + "epoch": 0.128928283642224, + "grad_norm": 3.71875, + "learning_rate": 4.584808259587021e-06, + "loss": 2.079263687133789, + "mean_token_accuracy": 0.5434476636350155, + "num_tokens": 5401626.0, + "step": 920 + }, + { + "entropy": 1.9311437100172042, + "epoch": 0.1303296780296395, + "grad_norm": 4.0625, + "learning_rate": 4.577433628318584e-06, + "loss": 2.022018814086914, + "mean_token_accuracy": 0.5603329479694367, + "num_tokens": 5457954.0, + "step": 930 + }, + { + "entropy": 2.0410451918840407, + "epoch": 0.13173107241705498, + "grad_norm": 3.6875, + "learning_rate": 4.570058997050148e-06, + "loss": 2.0759771347045897, + "mean_token_accuracy": 0.5456251330673695, + "num_tokens": 5514411.0, + "step": 940 + }, + { + "entropy": 2.0314603090286254, + "epoch": 0.13313246680447044, + "grad_norm": 3.625, + "learning_rate": 4.562684365781711e-06, + "loss": 2.087483787536621, + "mean_token_accuracy": 0.5457659848034382, + "num_tokens": 5573836.0, + "step": 950 + }, + { + "entropy": 2.048544630408287, + "epoch": 0.13453386119188593, + "grad_norm": 3.265625, + "learning_rate": 4.555309734513274e-06, + "loss": 2.0968584060668944, + "mean_token_accuracy": 0.5420685298740864, + "num_tokens": 5631848.0, + "step": 960 + }, + { + "entropy": 2.0000774681568148, + "epoch": 0.13593525557930142, + "grad_norm": 3.453125, + "learning_rate": 4.547935103244838e-06, + "loss": 2.0616247177124025, + "mean_token_accuracy": 0.5471087761223317, + "num_tokens": 5690136.0, + "step": 970 + }, + { + "entropy": 2.049460005760193, + "epoch": 0.13733664996671688, + "grad_norm": 3.71875, + "learning_rate": 4.540560471976402e-06, + "loss": 2.0837566375732424, + "mean_token_accuracy": 0.5436567731201649, + "num_tokens": 5749865.0, + "step": 980 + }, + { + "entropy": 1.996617168188095, + "epoch": 0.13873804435413237, + "grad_norm": 4.03125, + "learning_rate": 4.533185840707965e-06, + "loss": 2.039174270629883, + "mean_token_accuracy": 0.5500171341001987, + "num_tokens": 5809556.0, + "step": 990 + }, + { + "entropy": 2.08602631688118, + "epoch": 0.14013943874154783, + "grad_norm": 3.59375, + "learning_rate": 4.525811209439528e-06, + "loss": 2.1357633590698244, + "mean_token_accuracy": 0.5347492642700672, + "num_tokens": 5871230.0, + "step": 1000 + }, + { + "entropy": 2.050332334637642, + "epoch": 0.14154083312896332, + "grad_norm": 3.5625, + "learning_rate": 4.518436578171092e-06, + "loss": 2.0810262680053713, + "mean_token_accuracy": 0.5481423802673817, + "num_tokens": 5927611.0, + "step": 1010 + }, + { + "entropy": 2.017560285329819, + "epoch": 0.1429422275163788, + "grad_norm": 3.484375, + "learning_rate": 4.511061946902655e-06, + "loss": 2.044953727722168, + "mean_token_accuracy": 0.5441621497273446, + "num_tokens": 5988181.0, + "step": 1020 + }, + { + "entropy": 1.978191888332367, + "epoch": 0.14434362190379427, + "grad_norm": 3.25, + "learning_rate": 4.503687315634219e-06, + "loss": 2.0562999725341795, + "mean_token_accuracy": 0.553606178611517, + "num_tokens": 6044917.0, + "step": 1030 + }, + { + "entropy": 2.0290765553712844, + "epoch": 0.14574501629120976, + "grad_norm": 3.59375, + "learning_rate": 4.496312684365782e-06, + "loss": 2.0762353897094727, + "mean_token_accuracy": 0.5505865596234798, + "num_tokens": 6103386.0, + "step": 1040 + }, + { + "entropy": 1.9776984721422195, + "epoch": 0.14714641067862524, + "grad_norm": 3.8125, + "learning_rate": 4.4889380530973455e-06, + "loss": 2.0386898040771486, + "mean_token_accuracy": 0.554041002690792, + "num_tokens": 6160068.0, + "step": 1050 + }, + { + "entropy": 2.023387759923935, + "epoch": 0.1485478050660407, + "grad_norm": 4.4375, + "learning_rate": 4.481563421828909e-06, + "loss": 2.0716257095336914, + "mean_token_accuracy": 0.5479126609861851, + "num_tokens": 6220498.0, + "step": 1060 + }, + { + "entropy": 2.0474730670452117, + "epoch": 0.1499491994534562, + "grad_norm": 3.203125, + "learning_rate": 4.474188790560473e-06, + "loss": 2.077955436706543, + "mean_token_accuracy": 0.5469909913837909, + "num_tokens": 6278819.0, + "step": 1070 + }, + { + "entropy": 2.0014224141836165, + "epoch": 0.15135059384087166, + "grad_norm": 3.59375, + "learning_rate": 4.466814159292035e-06, + "loss": 2.049655723571777, + "mean_token_accuracy": 0.5477922618389129, + "num_tokens": 6338511.0, + "step": 1080 + }, + { + "entropy": 1.9802268594503403, + "epoch": 0.15275198822828714, + "grad_norm": 3.734375, + "learning_rate": 4.4594395280235995e-06, + "loss": 2.0510026931762697, + "mean_token_accuracy": 0.549194262176752, + "num_tokens": 6398236.0, + "step": 1090 + }, + { + "entropy": 2.0562620222568513, + "epoch": 0.15415338261570263, + "grad_norm": 3.515625, + "learning_rate": 4.452064896755163e-06, + "loss": 2.075428771972656, + "mean_token_accuracy": 0.5397219233214855, + "num_tokens": 6458469.0, + "step": 1100 + }, + { + "entropy": 1.9961812257766725, + "epoch": 0.1555547770031181, + "grad_norm": 3.90625, + "learning_rate": 4.444690265486726e-06, + "loss": 2.042446327209473, + "mean_token_accuracy": 0.546827956289053, + "num_tokens": 6520285.0, + "step": 1110 + }, + { + "entropy": 1.9569410175085067, + "epoch": 0.15695617139053358, + "grad_norm": 3.640625, + "learning_rate": 4.437315634218289e-06, + "loss": 2.0179250717163084, + "mean_token_accuracy": 0.5565195336937905, + "num_tokens": 6575492.0, + "step": 1120 + }, + { + "entropy": 2.020855191349983, + "epoch": 0.15835756577794907, + "grad_norm": 3.84375, + "learning_rate": 4.429941002949853e-06, + "loss": 2.0828372955322267, + "mean_token_accuracy": 0.5492666378617287, + "num_tokens": 6632686.0, + "step": 1130 + }, + { + "entropy": 2.033621999621391, + "epoch": 0.15975896016536453, + "grad_norm": 3.265625, + "learning_rate": 4.422566371681417e-06, + "loss": 2.08284969329834, + "mean_token_accuracy": 0.5422478929162026, + "num_tokens": 6693725.0, + "step": 1140 + }, + { + "entropy": 2.0249646574258806, + "epoch": 0.16116035455278002, + "grad_norm": 3.3125, + "learning_rate": 4.41519174041298e-06, + "loss": 2.0838279724121094, + "mean_token_accuracy": 0.5503048494458198, + "num_tokens": 6748881.0, + "step": 1150 + }, + { + "entropy": 2.014036813378334, + "epoch": 0.16256174894019548, + "grad_norm": 4.09375, + "learning_rate": 4.407817109144543e-06, + "loss": 2.0788618087768556, + "mean_token_accuracy": 0.5560389801859855, + "num_tokens": 6803170.0, + "step": 1160 + }, + { + "entropy": 2.036391496658325, + "epoch": 0.16396314332761097, + "grad_norm": 3.296875, + "learning_rate": 4.400442477876107e-06, + "loss": 2.08026008605957, + "mean_token_accuracy": 0.5430544361472129, + "num_tokens": 6862145.0, + "step": 1170 + }, + { + "entropy": 2.048283538222313, + "epoch": 0.16536453771502646, + "grad_norm": 3.671875, + "learning_rate": 4.39306784660767e-06, + "loss": 2.0746267318725584, + "mean_token_accuracy": 0.5450486116111278, + "num_tokens": 6922081.0, + "step": 1180 + }, + { + "entropy": 2.039934918284416, + "epoch": 0.16676593210244192, + "grad_norm": 3.6875, + "learning_rate": 4.385693215339233e-06, + "loss": 2.0975273132324217, + "mean_token_accuracy": 0.5371013961732387, + "num_tokens": 6987071.0, + "step": 1190 + }, + { + "entropy": 2.0690238893032076, + "epoch": 0.1681673264898574, + "grad_norm": 3.65625, + "learning_rate": 4.378318584070797e-06, + "loss": 2.093341064453125, + "mean_token_accuracy": 0.5354558274149894, + "num_tokens": 7050863.0, + "step": 1200 + }, + { + "entropy": 2.0879353821277618, + "epoch": 0.1695687208772729, + "grad_norm": 3.265625, + "learning_rate": 4.37094395280236e-06, + "loss": 2.138813018798828, + "mean_token_accuracy": 0.5322487376630306, + "num_tokens": 7114408.0, + "step": 1210 + }, + { + "entropy": 2.0216532915830614, + "epoch": 0.17097011526468836, + "grad_norm": 3.921875, + "learning_rate": 4.363569321533924e-06, + "loss": 2.0587230682373048, + "mean_token_accuracy": 0.5481282226741314, + "num_tokens": 7171138.0, + "step": 1220 + }, + { + "entropy": 2.0256995797157287, + "epoch": 0.17237150965210385, + "grad_norm": 4.0, + "learning_rate": 4.356194690265487e-06, + "loss": 2.0920654296875, + "mean_token_accuracy": 0.5509567841887474, + "num_tokens": 7227571.0, + "step": 1230 + }, + { + "entropy": 1.96624498963356, + "epoch": 0.1737729040395193, + "grad_norm": 4.6875, + "learning_rate": 4.34882005899705e-06, + "loss": 2.0297975540161133, + "mean_token_accuracy": 0.5592314623296261, + "num_tokens": 7283174.0, + "step": 1240 + }, + { + "entropy": 1.9834202975034714, + "epoch": 0.1751742984269348, + "grad_norm": 3.171875, + "learning_rate": 4.341445427728614e-06, + "loss": 2.032474899291992, + "mean_token_accuracy": 0.5518560096621513, + "num_tokens": 7341620.0, + "step": 1250 + }, + { + "entropy": 2.0227621525526045, + "epoch": 0.1765756928143503, + "grad_norm": 3.59375, + "learning_rate": 4.334070796460178e-06, + "loss": 2.0725967407226564, + "mean_token_accuracy": 0.550294154882431, + "num_tokens": 7401468.0, + "step": 1260 + }, + { + "entropy": 2.0101997554302216, + "epoch": 0.17797708720176575, + "grad_norm": 3.53125, + "learning_rate": 4.32669616519174e-06, + "loss": 2.0583026885986326, + "mean_token_accuracy": 0.5472072966396808, + "num_tokens": 7458282.0, + "step": 1270 + }, + { + "entropy": 1.99407716691494, + "epoch": 0.17937848158918124, + "grad_norm": 3.84375, + "learning_rate": 4.319321533923304e-06, + "loss": 2.032002639770508, + "mean_token_accuracy": 0.5496652647852898, + "num_tokens": 7517901.0, + "step": 1280 + }, + { + "entropy": 2.0280429512262343, + "epoch": 0.18077987597659673, + "grad_norm": 3.953125, + "learning_rate": 4.311946902654868e-06, + "loss": 2.0772026062011717, + "mean_token_accuracy": 0.5498968653380871, + "num_tokens": 7576444.0, + "step": 1290 + }, + { + "entropy": 2.017439436912537, + "epoch": 0.1821812703640122, + "grad_norm": 3.4375, + "learning_rate": 4.304572271386431e-06, + "loss": 2.03843994140625, + "mean_token_accuracy": 0.5490963935852051, + "num_tokens": 7631549.0, + "step": 1300 + }, + { + "entropy": 2.017805191874504, + "epoch": 0.18358266475142768, + "grad_norm": 3.390625, + "learning_rate": 4.297197640117994e-06, + "loss": 2.061754035949707, + "mean_token_accuracy": 0.5464312724769116, + "num_tokens": 7690773.0, + "step": 1310 + }, + { + "entropy": 2.03846270442009, + "epoch": 0.18498405913884314, + "grad_norm": 3.765625, + "learning_rate": 4.2898230088495575e-06, + "loss": 2.0858694076538087, + "mean_token_accuracy": 0.5471534766256809, + "num_tokens": 7746556.0, + "step": 1320 + }, + { + "entropy": 1.9878355622291566, + "epoch": 0.18638545352625863, + "grad_norm": 4.09375, + "learning_rate": 4.282448377581122e-06, + "loss": 2.0129779815673827, + "mean_token_accuracy": 0.555971772223711, + "num_tokens": 7802469.0, + "step": 1330 + }, + { + "entropy": 2.050147184729576, + "epoch": 0.18778684791367412, + "grad_norm": 3.234375, + "learning_rate": 4.275073746312685e-06, + "loss": 2.0990713119506834, + "mean_token_accuracy": 0.5421249151229859, + "num_tokens": 7862639.0, + "step": 1340 + }, + { + "entropy": 2.042793941497803, + "epoch": 0.18918824230108958, + "grad_norm": 3.8125, + "learning_rate": 4.267699115044248e-06, + "loss": 2.09356632232666, + "mean_token_accuracy": 0.5455923363566398, + "num_tokens": 7924419.0, + "step": 1350 + }, + { + "entropy": 1.9938032358884812, + "epoch": 0.19058963668850507, + "grad_norm": 3.578125, + "learning_rate": 4.2603244837758115e-06, + "loss": 2.031763458251953, + "mean_token_accuracy": 0.5496641293168067, + "num_tokens": 7983402.0, + "step": 1360 + }, + { + "entropy": 2.0284968227148057, + "epoch": 0.19199103107592055, + "grad_norm": 3.46875, + "learning_rate": 4.252949852507375e-06, + "loss": 2.0689836502075196, + "mean_token_accuracy": 0.543882504850626, + "num_tokens": 8043153.0, + "step": 1370 + }, + { + "entropy": 2.094452905654907, + "epoch": 0.19339242546333602, + "grad_norm": 3.484375, + "learning_rate": 4.245575221238938e-06, + "loss": 2.1328027725219725, + "mean_token_accuracy": 0.5360393956303596, + "num_tokens": 8105121.0, + "step": 1380 + }, + { + "entropy": 1.9476406127214432, + "epoch": 0.1947938198507515, + "grad_norm": 4.28125, + "learning_rate": 4.238200589970502e-06, + "loss": 1.9976503372192382, + "mean_token_accuracy": 0.5624695174396038, + "num_tokens": 8158071.0, + "step": 1390 + }, + { + "entropy": 2.04414941072464, + "epoch": 0.19619521423816697, + "grad_norm": 3.546875, + "learning_rate": 4.230825958702065e-06, + "loss": 2.095863914489746, + "mean_token_accuracy": 0.5353860557079315, + "num_tokens": 8218916.0, + "step": 1400 + }, + { + "entropy": 2.0396353930234907, + "epoch": 0.19759660862558245, + "grad_norm": 3.46875, + "learning_rate": 4.223451327433629e-06, + "loss": 2.056717872619629, + "mean_token_accuracy": 0.5479662336409092, + "num_tokens": 8278908.0, + "step": 1410 + }, + { + "entropy": 2.02007874250412, + "epoch": 0.19899800301299794, + "grad_norm": 3.546875, + "learning_rate": 4.216076696165192e-06, + "loss": 2.093400001525879, + "mean_token_accuracy": 0.5425383374094963, + "num_tokens": 8340788.0, + "step": 1420 + }, + { + "entropy": 2.080344945192337, + "epoch": 0.2003993974004134, + "grad_norm": 3.4375, + "learning_rate": 4.208702064896755e-06, + "loss": 2.1184001922607423, + "mean_token_accuracy": 0.5389995083212853, + "num_tokens": 8401654.0, + "step": 1430 + }, + { + "entropy": 2.0462054193019865, + "epoch": 0.2018007917878289, + "grad_norm": 4.3125, + "learning_rate": 4.201327433628319e-06, + "loss": 2.095401382446289, + "mean_token_accuracy": 0.5458614088594913, + "num_tokens": 8461086.0, + "step": 1440 + }, + { + "entropy": 2.0348505645990373, + "epoch": 0.20320218617524438, + "grad_norm": 3.703125, + "learning_rate": 4.193952802359883e-06, + "loss": 2.0764629364013674, + "mean_token_accuracy": 0.5477914564311505, + "num_tokens": 8519544.0, + "step": 1450 + }, + { + "entropy": 2.0418587952852247, + "epoch": 0.20460358056265984, + "grad_norm": 3.78125, + "learning_rate": 4.186578171091446e-06, + "loss": 2.0719451904296875, + "mean_token_accuracy": 0.5453279495239258, + "num_tokens": 8575632.0, + "step": 1460 + }, + { + "entropy": 1.9750055521726608, + "epoch": 0.20600497495007533, + "grad_norm": 3.859375, + "learning_rate": 4.179203539823009e-06, + "loss": 2.044822883605957, + "mean_token_accuracy": 0.5503912933170796, + "num_tokens": 8632807.0, + "step": 1470 + }, + { + "entropy": 2.067868760228157, + "epoch": 0.2074063693374908, + "grad_norm": 3.484375, + "learning_rate": 4.171828908554573e-06, + "loss": 2.1004459381103517, + "mean_token_accuracy": 0.5417281433939933, + "num_tokens": 8693834.0, + "step": 1480 + }, + { + "entropy": 2.0344304889440536, + "epoch": 0.20880776372490628, + "grad_norm": 3.890625, + "learning_rate": 4.164454277286136e-06, + "loss": 2.0767101287841796, + "mean_token_accuracy": 0.5481426410377026, + "num_tokens": 8753734.0, + "step": 1490 + }, + { + "entropy": 2.0047311425209045, + "epoch": 0.21020915811232177, + "grad_norm": 4.0, + "learning_rate": 4.157079646017699e-06, + "loss": 2.075409698486328, + "mean_token_accuracy": 0.5513108044862747, + "num_tokens": 8809137.0, + "step": 1500 + }, + { + "entropy": 2.0314046144485474, + "epoch": 0.21161055249973723, + "grad_norm": 3.5625, + "learning_rate": 4.1497050147492625e-06, + "loss": 2.0660478591918947, + "mean_token_accuracy": 0.5465934894979, + "num_tokens": 8867546.0, + "step": 1510 + }, + { + "entropy": 1.9994929015636445, + "epoch": 0.21301194688715272, + "grad_norm": 4.0, + "learning_rate": 4.142330383480827e-06, + "loss": 2.0473262786865236, + "mean_token_accuracy": 0.5522368937730789, + "num_tokens": 8924366.0, + "step": 1520 + }, + { + "entropy": 2.095882478356361, + "epoch": 0.2144133412745682, + "grad_norm": 3.453125, + "learning_rate": 4.13495575221239e-06, + "loss": 2.1347635269165037, + "mean_token_accuracy": 0.5365191757678985, + "num_tokens": 8986796.0, + "step": 1530 + }, + { + "entropy": 2.0848882615566255, + "epoch": 0.21581473566198367, + "grad_norm": 3.765625, + "learning_rate": 4.127581120943953e-06, + "loss": 2.142112159729004, + "mean_token_accuracy": 0.5401039518415928, + "num_tokens": 9047723.0, + "step": 1540 + }, + { + "entropy": 2.0115451723337174, + "epoch": 0.21721613004939916, + "grad_norm": 3.8125, + "learning_rate": 4.1202064896755165e-06, + "loss": 2.0483930587768553, + "mean_token_accuracy": 0.5501567840576171, + "num_tokens": 9105059.0, + "step": 1550 + }, + { + "entropy": 2.043051043152809, + "epoch": 0.21861752443681462, + "grad_norm": 3.609375, + "learning_rate": 4.11283185840708e-06, + "loss": 2.1072006225585938, + "mean_token_accuracy": 0.5446874745190143, + "num_tokens": 9165074.0, + "step": 1560 + }, + { + "entropy": 1.990285524725914, + "epoch": 0.2200189188242301, + "grad_norm": 3.765625, + "learning_rate": 4.105457227138643e-06, + "loss": 2.042169189453125, + "mean_token_accuracy": 0.5572816863656044, + "num_tokens": 9221198.0, + "step": 1570 + }, + { + "entropy": 1.9795201420783997, + "epoch": 0.2214203132116456, + "grad_norm": 4.1875, + "learning_rate": 4.098082595870207e-06, + "loss": 2.045511245727539, + "mean_token_accuracy": 0.5509970605373382, + "num_tokens": 9279938.0, + "step": 1580 + }, + { + "entropy": 2.0109116673469543, + "epoch": 0.22282170759906106, + "grad_norm": 3.8125, + "learning_rate": 4.09070796460177e-06, + "loss": 2.0378480911254884, + "mean_token_accuracy": 0.5496018096804619, + "num_tokens": 9335237.0, + "step": 1590 + }, + { + "entropy": 1.9677739530801772, + "epoch": 0.22422310198647655, + "grad_norm": 4.0, + "learning_rate": 4.083333333333334e-06, + "loss": 2.008736419677734, + "mean_token_accuracy": 0.5573440827429295, + "num_tokens": 9389997.0, + "step": 1600 + }, + { + "entropy": 2.0067166537046432, + "epoch": 0.22562449637389204, + "grad_norm": 3.875, + "learning_rate": 4.075958702064897e-06, + "loss": 2.0409936904907227, + "mean_token_accuracy": 0.5538617163896561, + "num_tokens": 9447727.0, + "step": 1610 + }, + { + "entropy": 2.0425065875053408, + "epoch": 0.2270258907613075, + "grad_norm": 3.4375, + "learning_rate": 4.068584070796461e-06, + "loss": 2.0920581817626953, + "mean_token_accuracy": 0.5449938684701919, + "num_tokens": 9508085.0, + "step": 1620 + }, + { + "entropy": 2.008884146809578, + "epoch": 0.22842728514872299, + "grad_norm": 3.546875, + "learning_rate": 4.061209439528024e-06, + "loss": 2.077191925048828, + "mean_token_accuracy": 0.5473880305886268, + "num_tokens": 9564075.0, + "step": 1630 + }, + { + "entropy": 2.0422200560569763, + "epoch": 0.22982867953613845, + "grad_norm": 3.859375, + "learning_rate": 4.053834808259588e-06, + "loss": 2.056693458557129, + "mean_token_accuracy": 0.5427386954426765, + "num_tokens": 9625251.0, + "step": 1640 + }, + { + "entropy": 1.9331750929355622, + "epoch": 0.23123007392355394, + "grad_norm": 3.609375, + "learning_rate": 4.046460176991151e-06, + "loss": 1.997640037536621, + "mean_token_accuracy": 0.559542316198349, + "num_tokens": 9679299.0, + "step": 1650 + }, + { + "entropy": 2.0358143985271453, + "epoch": 0.23263146831096942, + "grad_norm": 3.921875, + "learning_rate": 4.039085545722714e-06, + "loss": 2.0736413955688477, + "mean_token_accuracy": 0.5433515466749668, + "num_tokens": 9738907.0, + "step": 1660 + }, + { + "entropy": 2.0287980914115904, + "epoch": 0.23403286269838489, + "grad_norm": 3.421875, + "learning_rate": 4.031710914454278e-06, + "loss": 2.050998878479004, + "mean_token_accuracy": 0.5446001760661602, + "num_tokens": 9800456.0, + "step": 1670 + }, + { + "entropy": 2.030411049723625, + "epoch": 0.23543425708580037, + "grad_norm": 3.28125, + "learning_rate": 4.024336283185841e-06, + "loss": 2.0855588912963867, + "mean_token_accuracy": 0.542332086712122, + "num_tokens": 9860929.0, + "step": 1680 + }, + { + "entropy": 1.9989683747291564, + "epoch": 0.23683565147321586, + "grad_norm": 3.640625, + "learning_rate": 4.016961651917404e-06, + "loss": 2.044294738769531, + "mean_token_accuracy": 0.5456494316458702, + "num_tokens": 9921403.0, + "step": 1690 + }, + { + "entropy": 2.0276321530342103, + "epoch": 0.23823704586063132, + "grad_norm": 4.25, + "learning_rate": 4.0095870206489675e-06, + "loss": 2.058464241027832, + "mean_token_accuracy": 0.5500309750437736, + "num_tokens": 9978695.0, + "step": 1700 + }, + { + "entropy": 1.9861027270555496, + "epoch": 0.2396384402480468, + "grad_norm": 3.5625, + "learning_rate": 4.002212389380532e-06, + "loss": 2.0480157852172853, + "mean_token_accuracy": 0.5492375880479813, + "num_tokens": 10035266.0, + "step": 1710 + }, + { + "entropy": 2.0590964436531065, + "epoch": 0.24103983463546227, + "grad_norm": 3.46875, + "learning_rate": 3.994837758112095e-06, + "loss": 2.0842300415039063, + "mean_token_accuracy": 0.5476659752428532, + "num_tokens": 10095628.0, + "step": 1720 + }, + { + "entropy": 2.0491928100585937, + "epoch": 0.24244122902287776, + "grad_norm": 3.4375, + "learning_rate": 3.987463126843658e-06, + "loss": 2.097138786315918, + "mean_token_accuracy": 0.5445733956992627, + "num_tokens": 10152840.0, + "step": 1730 + }, + { + "entropy": 2.0307011008262634, + "epoch": 0.24384262341029325, + "grad_norm": 3.46875, + "learning_rate": 3.9800884955752215e-06, + "loss": 2.065476417541504, + "mean_token_accuracy": 0.5439828269183635, + "num_tokens": 10212647.0, + "step": 1740 + }, + { + "entropy": 2.031652170419693, + "epoch": 0.2452440177977087, + "grad_norm": 3.8125, + "learning_rate": 3.972713864306785e-06, + "loss": 2.0586521148681642, + "mean_token_accuracy": 0.5489093273878097, + "num_tokens": 10270971.0, + "step": 1750 + }, + { + "entropy": 2.0088635206222536, + "epoch": 0.2466454121851242, + "grad_norm": 4.0, + "learning_rate": 3.965339233038348e-06, + "loss": 2.0783411026000977, + "mean_token_accuracy": 0.54852824062109, + "num_tokens": 10330561.0, + "step": 1760 + }, + { + "entropy": 2.0077249109745026, + "epoch": 0.24804680657253966, + "grad_norm": 3.4375, + "learning_rate": 3.957964601769912e-06, + "loss": 2.0558242797851562, + "mean_token_accuracy": 0.5530465073883534, + "num_tokens": 10387513.0, + "step": 1770 + }, + { + "entropy": 1.9883006066083908, + "epoch": 0.24944820095995515, + "grad_norm": 3.515625, + "learning_rate": 3.9505899705014754e-06, + "loss": 2.0360387802124023, + "mean_token_accuracy": 0.5527264244854451, + "num_tokens": 10445558.0, + "step": 1780 + }, + { + "entropy": 2.048704609274864, + "epoch": 0.2508495953473706, + "grad_norm": 3.28125, + "learning_rate": 3.943215339233039e-06, + "loss": 2.097726249694824, + "mean_token_accuracy": 0.5423043698072434, + "num_tokens": 10507169.0, + "step": 1790 + }, + { + "entropy": 2.040918904542923, + "epoch": 0.2522509897347861, + "grad_norm": 3.625, + "learning_rate": 3.935840707964602e-06, + "loss": 2.080495262145996, + "mean_token_accuracy": 0.5438195087015629, + "num_tokens": 10566307.0, + "step": 1800 + }, + { + "entropy": 2.0052473068237306, + "epoch": 0.2536523841222016, + "grad_norm": 2.9375, + "learning_rate": 3.928466076696166e-06, + "loss": 2.0398183822631837, + "mean_token_accuracy": 0.546070022881031, + "num_tokens": 10624818.0, + "step": 1810 + }, + { + "entropy": 2.083695039153099, + "epoch": 0.2550537785096171, + "grad_norm": 4.3125, + "learning_rate": 3.921091445427729e-06, + "loss": 2.117705726623535, + "mean_token_accuracy": 0.5372008338570595, + "num_tokens": 10687638.0, + "step": 1820 + }, + { + "entropy": 2.0451648265123366, + "epoch": 0.25645517289703257, + "grad_norm": 3.328125, + "learning_rate": 3.913716814159293e-06, + "loss": 2.101591873168945, + "mean_token_accuracy": 0.5431081369519234, + "num_tokens": 10748647.0, + "step": 1830 + }, + { + "entropy": 2.028788596391678, + "epoch": 0.257856567284448, + "grad_norm": 3.890625, + "learning_rate": 3.906342182890856e-06, + "loss": 2.0835641860961913, + "mean_token_accuracy": 0.5432848446071148, + "num_tokens": 10810906.0, + "step": 1840 + }, + { + "entropy": 1.9781817615032196, + "epoch": 0.2592579616718635, + "grad_norm": 3.640625, + "learning_rate": 3.898967551622419e-06, + "loss": 2.0251651763916017, + "mean_token_accuracy": 0.5534879423677921, + "num_tokens": 10867014.0, + "step": 1850 + }, + { + "entropy": 1.9987586200237275, + "epoch": 0.260659356059279, + "grad_norm": 3.65625, + "learning_rate": 3.8915929203539826e-06, + "loss": 2.0439634323120117, + "mean_token_accuracy": 0.5515172652900219, + "num_tokens": 10923966.0, + "step": 1860 + }, + { + "entropy": 1.9946261763572692, + "epoch": 0.26206075044669447, + "grad_norm": 4.34375, + "learning_rate": 3.884218289085546e-06, + "loss": 2.0570749282836913, + "mean_token_accuracy": 0.5502081900835037, + "num_tokens": 10982331.0, + "step": 1870 + }, + { + "entropy": 2.082772919535637, + "epoch": 0.26346214483410996, + "grad_norm": 3.1875, + "learning_rate": 3.876843657817109e-06, + "loss": 2.124038505554199, + "mean_token_accuracy": 0.5378222659230232, + "num_tokens": 11042321.0, + "step": 1880 + }, + { + "entropy": 2.0446047335863113, + "epoch": 0.2648635392215254, + "grad_norm": 3.25, + "learning_rate": 3.869469026548673e-06, + "loss": 2.081063461303711, + "mean_token_accuracy": 0.5414594948291779, + "num_tokens": 11103961.0, + "step": 1890 + }, + { + "entropy": 2.003836825489998, + "epoch": 0.2662649336089409, + "grad_norm": 3.6875, + "learning_rate": 3.8620943952802366e-06, + "loss": 2.0427907943725585, + "mean_token_accuracy": 0.5516212925314903, + "num_tokens": 11161256.0, + "step": 1900 + }, + { + "entropy": 2.066277724504471, + "epoch": 0.26766632799635637, + "grad_norm": 3.484375, + "learning_rate": 3.8547197640118e-06, + "loss": 2.1015077590942384, + "mean_token_accuracy": 0.5404363766312599, + "num_tokens": 11220888.0, + "step": 1910 + }, + { + "entropy": 2.0015517294406893, + "epoch": 0.26906772238377186, + "grad_norm": 3.765625, + "learning_rate": 3.847345132743363e-06, + "loss": 2.0402530670166015, + "mean_token_accuracy": 0.5519128695130349, + "num_tokens": 11278153.0, + "step": 1920 + }, + { + "entropy": 1.9748200744390487, + "epoch": 0.27046911677118735, + "grad_norm": 3.359375, + "learning_rate": 3.839970501474926e-06, + "loss": 1.996561050415039, + "mean_token_accuracy": 0.5551008068025112, + "num_tokens": 11333108.0, + "step": 1930 + }, + { + "entropy": 2.024563890695572, + "epoch": 0.27187051115860283, + "grad_norm": 4.125, + "learning_rate": 3.8325958702064905e-06, + "loss": 2.0915248870849608, + "mean_token_accuracy": 0.5441000409424305, + "num_tokens": 11392826.0, + "step": 1940 + }, + { + "entropy": 2.048647916316986, + "epoch": 0.27327190554601827, + "grad_norm": 3.703125, + "learning_rate": 3.825221238938053e-06, + "loss": 2.1001590728759765, + "mean_token_accuracy": 0.5445165455341339, + "num_tokens": 11451500.0, + "step": 1950 + }, + { + "entropy": 2.009050303697586, + "epoch": 0.27467329993343376, + "grad_norm": 3.71875, + "learning_rate": 3.817846607669617e-06, + "loss": 2.037006950378418, + "mean_token_accuracy": 0.5515233352780342, + "num_tokens": 11508406.0, + "step": 1960 + }, + { + "entropy": 1.983980432152748, + "epoch": 0.27607469432084925, + "grad_norm": 3.578125, + "learning_rate": 3.8104719764011804e-06, + "loss": 2.030117416381836, + "mean_token_accuracy": 0.5510232672095299, + "num_tokens": 11565640.0, + "step": 1970 + }, + { + "entropy": 2.0504226833581924, + "epoch": 0.27747608870826473, + "grad_norm": 3.734375, + "learning_rate": 3.8030973451327437e-06, + "loss": 2.074881744384766, + "mean_token_accuracy": 0.5433954305946826, + "num_tokens": 11626552.0, + "step": 1980 + }, + { + "entropy": 2.04710875749588, + "epoch": 0.2788774830956802, + "grad_norm": 3.484375, + "learning_rate": 3.795722713864307e-06, + "loss": 2.0955013275146483, + "mean_token_accuracy": 0.5433564156293869, + "num_tokens": 11686265.0, + "step": 1990 + }, + { + "entropy": 2.0252153426408768, + "epoch": 0.28027887748309566, + "grad_norm": 3.515625, + "learning_rate": 3.7883480825958707e-06, + "loss": 2.0773523330688475, + "mean_token_accuracy": 0.5479929871857167, + "num_tokens": 11744420.0, + "step": 2000 + }, + { + "entropy": 2.0120167046785356, + "epoch": 0.28168027187051115, + "grad_norm": 3.453125, + "learning_rate": 3.780973451327434e-06, + "loss": 2.06146240234375, + "mean_token_accuracy": 0.5448708184063434, + "num_tokens": 11805879.0, + "step": 2010 + }, + { + "entropy": 2.0325658559799193, + "epoch": 0.28308166625792663, + "grad_norm": 3.390625, + "learning_rate": 3.7735988200589972e-06, + "loss": 2.07748966217041, + "mean_token_accuracy": 0.5456863172352314, + "num_tokens": 11865695.0, + "step": 2020 + }, + { + "entropy": 1.9818778395652772, + "epoch": 0.2844830606453421, + "grad_norm": 3.390625, + "learning_rate": 3.766224188790561e-06, + "loss": 2.02254638671875, + "mean_token_accuracy": 0.5516637116670609, + "num_tokens": 11924218.0, + "step": 2030 + }, + { + "entropy": 2.0609579384326935, + "epoch": 0.2858844550327576, + "grad_norm": 3.375, + "learning_rate": 3.758849557522124e-06, + "loss": 2.1082080841064452, + "mean_token_accuracy": 0.5414181008934975, + "num_tokens": 11985925.0, + "step": 2040 + }, + { + "entropy": 2.056264355778694, + "epoch": 0.28728584942017305, + "grad_norm": 3.578125, + "learning_rate": 3.7514749262536875e-06, + "loss": 2.0749874114990234, + "mean_token_accuracy": 0.542043574154377, + "num_tokens": 12043546.0, + "step": 2050 + }, + { + "entropy": 2.0399533569812776, + "epoch": 0.28868724380758853, + "grad_norm": 3.625, + "learning_rate": 3.7441002949852512e-06, + "loss": 2.115126037597656, + "mean_token_accuracy": 0.542166319489479, + "num_tokens": 12101450.0, + "step": 2060 + }, + { + "entropy": 2.005798429250717, + "epoch": 0.290088638195004, + "grad_norm": 3.59375, + "learning_rate": 3.736725663716814e-06, + "loss": 2.0584402084350586, + "mean_token_accuracy": 0.5555612161755562, + "num_tokens": 12158295.0, + "step": 2070 + }, + { + "entropy": 2.027473473548889, + "epoch": 0.2914900325824195, + "grad_norm": 4.1875, + "learning_rate": 3.729351032448378e-06, + "loss": 2.0642648696899415, + "mean_token_accuracy": 0.5496302887797355, + "num_tokens": 12217180.0, + "step": 2080 + }, + { + "entropy": 2.068603280186653, + "epoch": 0.292891426969835, + "grad_norm": 3.875, + "learning_rate": 3.7219764011799415e-06, + "loss": 2.1073820114135744, + "mean_token_accuracy": 0.5398187652230263, + "num_tokens": 12279165.0, + "step": 2090 + }, + { + "entropy": 1.9664050936698914, + "epoch": 0.2942928213572505, + "grad_norm": 3.40625, + "learning_rate": 3.7146017699115044e-06, + "loss": 2.031716537475586, + "mean_token_accuracy": 0.5617817871272563, + "num_tokens": 12332864.0, + "step": 2100 + }, + { + "entropy": 2.031452310085297, + "epoch": 0.2956942157446659, + "grad_norm": 3.78125, + "learning_rate": 3.707227138643068e-06, + "loss": 2.058265495300293, + "mean_token_accuracy": 0.550971408188343, + "num_tokens": 12391694.0, + "step": 2110 + }, + { + "entropy": 1.9575803756713868, + "epoch": 0.2970956101320814, + "grad_norm": 3.484375, + "learning_rate": 3.699852507374632e-06, + "loss": 2.0243839263916015, + "mean_token_accuracy": 0.5532980509102344, + "num_tokens": 12452312.0, + "step": 2120 + }, + { + "entropy": 2.0010867685079576, + "epoch": 0.2984970045194969, + "grad_norm": 3.640625, + "learning_rate": 3.692477876106195e-06, + "loss": 2.0592809677124024, + "mean_token_accuracy": 0.5490673378109932, + "num_tokens": 12507943.0, + "step": 2130 + }, + { + "entropy": 2.029738873243332, + "epoch": 0.2998983989069124, + "grad_norm": 3.9375, + "learning_rate": 3.6851032448377584e-06, + "loss": 2.07022705078125, + "mean_token_accuracy": 0.5449296228587628, + "num_tokens": 12567374.0, + "step": 2140 + }, + { + "entropy": 2.0201781749725343, + "epoch": 0.3012997932943279, + "grad_norm": 3.96875, + "learning_rate": 3.677728613569322e-06, + "loss": 2.0301849365234377, + "mean_token_accuracy": 0.5546999678015709, + "num_tokens": 12622888.0, + "step": 2150 + }, + { + "entropy": 1.9730218589305877, + "epoch": 0.3027011876817433, + "grad_norm": 3.796875, + "learning_rate": 3.6703539823008854e-06, + "loss": 2.0661874771118165, + "mean_token_accuracy": 0.5545330084860325, + "num_tokens": 12678493.0, + "step": 2160 + }, + { + "entropy": 2.0124511659145354, + "epoch": 0.3041025820691588, + "grad_norm": 4.25, + "learning_rate": 3.6629793510324486e-06, + "loss": 2.0427947998046876, + "mean_token_accuracy": 0.554379727691412, + "num_tokens": 12735832.0, + "step": 2170 + }, + { + "entropy": 2.0333964198827745, + "epoch": 0.3055039764565743, + "grad_norm": 3.625, + "learning_rate": 3.655604719764012e-06, + "loss": 2.0744590759277344, + "mean_token_accuracy": 0.5443440832197666, + "num_tokens": 12795414.0, + "step": 2180 + }, + { + "entropy": 1.9904500722885132, + "epoch": 0.3069053708439898, + "grad_norm": 3.546875, + "learning_rate": 3.6482300884955756e-06, + "loss": 2.0214725494384767, + "mean_token_accuracy": 0.553394029289484, + "num_tokens": 12850628.0, + "step": 2190 + }, + { + "entropy": 2.012459713220596, + "epoch": 0.30830676523140527, + "grad_norm": 4.0, + "learning_rate": 3.640855457227139e-06, + "loss": 2.05026798248291, + "mean_token_accuracy": 0.5478941559791565, + "num_tokens": 12910543.0, + "step": 2200 + }, + { + "entropy": 2.041311630606651, + "epoch": 0.3097081596188207, + "grad_norm": 3.65625, + "learning_rate": 3.633480825958702e-06, + "loss": 2.080320930480957, + "mean_token_accuracy": 0.5436180986464023, + "num_tokens": 12969175.0, + "step": 2210 + }, + { + "entropy": 2.0246637046337126, + "epoch": 0.3111095540062362, + "grad_norm": 3.40625, + "learning_rate": 3.626106194690266e-06, + "loss": 2.075758934020996, + "mean_token_accuracy": 0.5477502450346947, + "num_tokens": 13029048.0, + "step": 2220 + }, + { + "entropy": 2.016602024435997, + "epoch": 0.3125109483936517, + "grad_norm": 3.296875, + "learning_rate": 3.6187315634218288e-06, + "loss": 2.0503862380981444, + "mean_token_accuracy": 0.5497252814471721, + "num_tokens": 13086708.0, + "step": 2230 + }, + { + "entropy": 1.9739759057760238, + "epoch": 0.31391234278106717, + "grad_norm": 3.625, + "learning_rate": 3.6113569321533925e-06, + "loss": 2.0244831085205077, + "mean_token_accuracy": 0.5533089518547059, + "num_tokens": 13145789.0, + "step": 2240 + }, + { + "entropy": 2.066014051437378, + "epoch": 0.31531373716848266, + "grad_norm": 3.453125, + "learning_rate": 3.603982300884956e-06, + "loss": 2.1090654373168944, + "mean_token_accuracy": 0.5381431728601456, + "num_tokens": 13204643.0, + "step": 2250 + }, + { + "entropy": 2.032071939110756, + "epoch": 0.31671513155589814, + "grad_norm": 3.859375, + "learning_rate": 3.596607669616519e-06, + "loss": 2.0973888397216798, + "mean_token_accuracy": 0.5404280602931977, + "num_tokens": 13264537.0, + "step": 2260 + }, + { + "entropy": 2.030746152997017, + "epoch": 0.3181165259433136, + "grad_norm": 3.546875, + "learning_rate": 3.5892330383480828e-06, + "loss": 2.066508483886719, + "mean_token_accuracy": 0.5464574486017227, + "num_tokens": 13326486.0, + "step": 2270 + }, + { + "entropy": 2.0341346323490144, + "epoch": 0.31951792033072907, + "grad_norm": 3.421875, + "learning_rate": 3.5818584070796465e-06, + "loss": 2.0528663635253905, + "mean_token_accuracy": 0.5416016794741154, + "num_tokens": 13386625.0, + "step": 2280 + }, + { + "entropy": 2.033405900001526, + "epoch": 0.32091931471814455, + "grad_norm": 3.734375, + "learning_rate": 3.57448377581121e-06, + "loss": 2.077082061767578, + "mean_token_accuracy": 0.5406279399991035, + "num_tokens": 13446811.0, + "step": 2290 + }, + { + "entropy": 2.043126568198204, + "epoch": 0.32232070910556004, + "grad_norm": 3.640625, + "learning_rate": 3.567109144542773e-06, + "loss": 2.0811180114746093, + "mean_token_accuracy": 0.542109789699316, + "num_tokens": 13505758.0, + "step": 2300 + }, + { + "entropy": 2.0713710308074953, + "epoch": 0.32372210349297553, + "grad_norm": 3.625, + "learning_rate": 3.5597345132743367e-06, + "loss": 2.0945466995239257, + "mean_token_accuracy": 0.539750412106514, + "num_tokens": 13564426.0, + "step": 2310 + }, + { + "entropy": 2.009327819943428, + "epoch": 0.32512349788039097, + "grad_norm": 3.828125, + "learning_rate": 3.5523598820059e-06, + "loss": 2.0479202270507812, + "mean_token_accuracy": 0.5478827625513076, + "num_tokens": 13624506.0, + "step": 2320 + }, + { + "entropy": 2.0233408570289613, + "epoch": 0.32652489226780645, + "grad_norm": 3.625, + "learning_rate": 3.5449852507374633e-06, + "loss": 2.072812080383301, + "mean_token_accuracy": 0.5496255904436111, + "num_tokens": 13681895.0, + "step": 2330 + }, + { + "entropy": 2.0150050669908524, + "epoch": 0.32792628665522194, + "grad_norm": 3.640625, + "learning_rate": 3.537610619469027e-06, + "loss": 2.0481910705566406, + "mean_token_accuracy": 0.5474696174263954, + "num_tokens": 13742408.0, + "step": 2340 + }, + { + "entropy": 1.996006327867508, + "epoch": 0.32932768104263743, + "grad_norm": 4.21875, + "learning_rate": 3.5302359882005903e-06, + "loss": 2.039927673339844, + "mean_token_accuracy": 0.5505919076502324, + "num_tokens": 13801936.0, + "step": 2350 + }, + { + "entropy": 2.0160360783338547, + "epoch": 0.3307290754300529, + "grad_norm": 4.25, + "learning_rate": 3.5228613569321536e-06, + "loss": 2.0754911422729494, + "mean_token_accuracy": 0.5470316275954247, + "num_tokens": 13860452.0, + "step": 2360 + }, + { + "entropy": 2.033596268296242, + "epoch": 0.33213046981746835, + "grad_norm": 3.640625, + "learning_rate": 3.515486725663717e-06, + "loss": 2.0809364318847656, + "mean_token_accuracy": 0.5446468167006969, + "num_tokens": 13917770.0, + "step": 2370 + }, + { + "entropy": 1.9926137715578078, + "epoch": 0.33353186420488384, + "grad_norm": 3.40625, + "learning_rate": 3.5081120943952806e-06, + "loss": 2.028292655944824, + "mean_token_accuracy": 0.5545202195644379, + "num_tokens": 13973511.0, + "step": 2380 + }, + { + "entropy": 1.956346869468689, + "epoch": 0.33493325859229933, + "grad_norm": 4.46875, + "learning_rate": 3.500737463126844e-06, + "loss": 2.014778709411621, + "mean_token_accuracy": 0.5580305725336074, + "num_tokens": 14029619.0, + "step": 2390 + }, + { + "entropy": 2.021263125538826, + "epoch": 0.3363346529797148, + "grad_norm": 4.1875, + "learning_rate": 3.493362831858407e-06, + "loss": 2.0767066955566404, + "mean_token_accuracy": 0.5438144609332085, + "num_tokens": 14090701.0, + "step": 2400 + }, + { + "entropy": 2.0122366905212403, + "epoch": 0.3377360473671303, + "grad_norm": 3.484375, + "learning_rate": 3.485988200589971e-06, + "loss": 2.0509740829467775, + "mean_token_accuracy": 0.5462699607014656, + "num_tokens": 14152154.0, + "step": 2410 + }, + { + "entropy": 1.9940638601779939, + "epoch": 0.3391374417545458, + "grad_norm": 4.09375, + "learning_rate": 3.4786135693215337e-06, + "loss": 2.0240846633911134, + "mean_token_accuracy": 0.5554023273289204, + "num_tokens": 14206928.0, + "step": 2420 + }, + { + "entropy": 2.0023196965456007, + "epoch": 0.34053883614196123, + "grad_norm": 3.75, + "learning_rate": 3.4712389380530974e-06, + "loss": 2.038427543640137, + "mean_token_accuracy": 0.5528297238051891, + "num_tokens": 14265500.0, + "step": 2430 + }, + { + "entropy": 2.02716104388237, + "epoch": 0.3419402305293767, + "grad_norm": 3.9375, + "learning_rate": 3.463864306784661e-06, + "loss": 2.0936708450317383, + "mean_token_accuracy": 0.5434965647757053, + "num_tokens": 14323980.0, + "step": 2440 + }, + { + "entropy": 2.006233334541321, + "epoch": 0.3433416249167922, + "grad_norm": 3.625, + "learning_rate": 3.456489675516225e-06, + "loss": 2.010869789123535, + "mean_token_accuracy": 0.5536993630230427, + "num_tokens": 14381484.0, + "step": 2450 + }, + { + "entropy": 1.956309551000595, + "epoch": 0.3447430193042077, + "grad_norm": 4.125, + "learning_rate": 3.4491150442477877e-06, + "loss": 2.0028940200805665, + "mean_token_accuracy": 0.5577565245330334, + "num_tokens": 14437013.0, + "step": 2460 + }, + { + "entropy": 1.986230832338333, + "epoch": 0.3461444136916232, + "grad_norm": 4.03125, + "learning_rate": 3.4417404129793514e-06, + "loss": 2.0378223419189454, + "mean_token_accuracy": 0.5529210731387139, + "num_tokens": 14494172.0, + "step": 2470 + }, + { + "entropy": 2.020972040295601, + "epoch": 0.3475458080790386, + "grad_norm": 3.34375, + "learning_rate": 3.434365781710915e-06, + "loss": 2.057634162902832, + "mean_token_accuracy": 0.5544227443635463, + "num_tokens": 14549695.0, + "step": 2480 + }, + { + "entropy": 2.0772712618112563, + "epoch": 0.3489472024664541, + "grad_norm": 3.84375, + "learning_rate": 3.426991150442478e-06, + "loss": 2.1363456726074217, + "mean_token_accuracy": 0.5460018336772918, + "num_tokens": 14608869.0, + "step": 2490 + }, + { + "entropy": 1.991566962003708, + "epoch": 0.3503485968538696, + "grad_norm": 3.4375, + "learning_rate": 3.4196165191740417e-06, + "loss": 2.0198110580444335, + "mean_token_accuracy": 0.5531622432172298, + "num_tokens": 14665103.0, + "step": 2500 + }, + { + "entropy": 2.051846295595169, + "epoch": 0.3517499912412851, + "grad_norm": 3.53125, + "learning_rate": 3.412241887905605e-06, + "loss": 2.115659713745117, + "mean_token_accuracy": 0.5371693730354309, + "num_tokens": 14725608.0, + "step": 2510 + }, + { + "entropy": 2.0814440310001374, + "epoch": 0.3531513856287006, + "grad_norm": 3.828125, + "learning_rate": 3.4048672566371683e-06, + "loss": 2.1354246139526367, + "mean_token_accuracy": 0.534952775388956, + "num_tokens": 14785719.0, + "step": 2520 + }, + { + "entropy": 1.9824702113866806, + "epoch": 0.354552780016116, + "grad_norm": 3.625, + "learning_rate": 3.397492625368732e-06, + "loss": 2.029611015319824, + "mean_token_accuracy": 0.5569346085190773, + "num_tokens": 14843220.0, + "step": 2530 + }, + { + "entropy": 1.9848757982254028, + "epoch": 0.3559541744035315, + "grad_norm": 3.90625, + "learning_rate": 3.3901179941002953e-06, + "loss": 2.036295700073242, + "mean_token_accuracy": 0.5557854510843754, + "num_tokens": 14901805.0, + "step": 2540 + }, + { + "entropy": 2.012274181842804, + "epoch": 0.357355568790947, + "grad_norm": 3.71875, + "learning_rate": 3.3827433628318586e-06, + "loss": 2.0520450592041017, + "mean_token_accuracy": 0.5437561951577663, + "num_tokens": 14961782.0, + "step": 2550 + }, + { + "entropy": 2.0370524376630783, + "epoch": 0.3587569631783625, + "grad_norm": 3.8125, + "learning_rate": 3.375368731563422e-06, + "loss": 2.1124666213989256, + "mean_token_accuracy": 0.5437058597803116, + "num_tokens": 15022925.0, + "step": 2560 + }, + { + "entropy": 2.0400708824396134, + "epoch": 0.36015835756577796, + "grad_norm": 4.09375, + "learning_rate": 3.3679941002949855e-06, + "loss": 2.046133613586426, + "mean_token_accuracy": 0.553859393298626, + "num_tokens": 15082687.0, + "step": 2570 + }, + { + "entropy": 2.0404021978378295, + "epoch": 0.36155975195319345, + "grad_norm": 3.6875, + "learning_rate": 3.360619469026549e-06, + "loss": 2.08302001953125, + "mean_token_accuracy": 0.5444831892848014, + "num_tokens": 15143071.0, + "step": 2580 + }, + { + "entropy": 1.9733462482690811, + "epoch": 0.3629611463406089, + "grad_norm": 3.5, + "learning_rate": 3.353244837758112e-06, + "loss": 2.0270782470703126, + "mean_token_accuracy": 0.5567068234086037, + "num_tokens": 15201082.0, + "step": 2590 + }, + { + "entropy": 2.010732626914978, + "epoch": 0.3643625407280244, + "grad_norm": 3.328125, + "learning_rate": 3.345870206489676e-06, + "loss": 2.0494049072265623, + "mean_token_accuracy": 0.5488976605236531, + "num_tokens": 15260037.0, + "step": 2600 + }, + { + "entropy": 2.0001643508672715, + "epoch": 0.36576393511543986, + "grad_norm": 4.03125, + "learning_rate": 3.3384955752212395e-06, + "loss": 2.0865619659423826, + "mean_token_accuracy": 0.5552799321711064, + "num_tokens": 15317347.0, + "step": 2610 + }, + { + "entropy": 2.0003055959939955, + "epoch": 0.36716532950285535, + "grad_norm": 3.421875, + "learning_rate": 3.3311209439528024e-06, + "loss": 2.032404327392578, + "mean_token_accuracy": 0.5533877588808537, + "num_tokens": 15376104.0, + "step": 2620 + }, + { + "entropy": 2.0134785562753676, + "epoch": 0.36856672389027084, + "grad_norm": 3.796875, + "learning_rate": 3.323746312684366e-06, + "loss": 2.075766181945801, + "mean_token_accuracy": 0.5478799432516098, + "num_tokens": 15433006.0, + "step": 2630 + }, + { + "entropy": 2.0133273512125016, + "epoch": 0.3699681182776863, + "grad_norm": 3.671875, + "learning_rate": 3.31637168141593e-06, + "loss": 2.0486371994018553, + "mean_token_accuracy": 0.5523924797773361, + "num_tokens": 15488723.0, + "step": 2640 + }, + { + "entropy": 2.02708223760128, + "epoch": 0.37136951266510176, + "grad_norm": 3.09375, + "learning_rate": 3.3089970501474927e-06, + "loss": 2.1050783157348634, + "mean_token_accuracy": 0.543995326012373, + "num_tokens": 15549170.0, + "step": 2650 + }, + { + "entropy": 2.059222882986069, + "epoch": 0.37277090705251725, + "grad_norm": 3.625, + "learning_rate": 3.3016224188790564e-06, + "loss": 2.0997392654418947, + "mean_token_accuracy": 0.5429941609501838, + "num_tokens": 15612383.0, + "step": 2660 + }, + { + "entropy": 1.996263289451599, + "epoch": 0.37417230143993274, + "grad_norm": 3.421875, + "learning_rate": 3.29424778761062e-06, + "loss": 2.0333202362060545, + "mean_token_accuracy": 0.5534181974828243, + "num_tokens": 15668533.0, + "step": 2670 + }, + { + "entropy": 2.0458276212215423, + "epoch": 0.37557369582734823, + "grad_norm": 3.6875, + "learning_rate": 3.286873156342183e-06, + "loss": 2.11057186126709, + "mean_token_accuracy": 0.5400335542857647, + "num_tokens": 15728891.0, + "step": 2680 + }, + { + "entropy": 2.012293756008148, + "epoch": 0.37697509021476366, + "grad_norm": 3.390625, + "learning_rate": 3.2794985250737467e-06, + "loss": 2.0523448944091798, + "mean_token_accuracy": 0.5516530051827431, + "num_tokens": 15786671.0, + "step": 2690 + }, + { + "entropy": 2.03779356777668, + "epoch": 0.37837648460217915, + "grad_norm": 3.953125, + "learning_rate": 3.27212389380531e-06, + "loss": 2.0828990936279297, + "mean_token_accuracy": 0.5453480415046215, + "num_tokens": 15847796.0, + "step": 2700 + }, + { + "entropy": 2.0558393210172654, + "epoch": 0.37977787898959464, + "grad_norm": 3.8125, + "learning_rate": 3.2647492625368732e-06, + "loss": 2.076746368408203, + "mean_token_accuracy": 0.5400703974068165, + "num_tokens": 15907761.0, + "step": 2710 + }, + { + "entropy": 1.979285529255867, + "epoch": 0.38117927337701013, + "grad_norm": 3.390625, + "learning_rate": 3.257374631268437e-06, + "loss": 2.04888916015625, + "mean_token_accuracy": 0.557814684510231, + "num_tokens": 15963089.0, + "step": 2720 + }, + { + "entropy": 2.062352991104126, + "epoch": 0.3825806677644256, + "grad_norm": 3.859375, + "learning_rate": 3.2500000000000002e-06, + "loss": 2.0771505355834963, + "mean_token_accuracy": 0.5436685405671596, + "num_tokens": 16021866.0, + "step": 2730 + }, + { + "entropy": 2.033045384287834, + "epoch": 0.3839820621518411, + "grad_norm": 3.5625, + "learning_rate": 3.2426253687315635e-06, + "loss": 2.094202995300293, + "mean_token_accuracy": 0.5457345262169838, + "num_tokens": 16081435.0, + "step": 2740 + }, + { + "entropy": 2.038944327831268, + "epoch": 0.38538345653925654, + "grad_norm": 3.578125, + "learning_rate": 3.2352507374631272e-06, + "loss": 2.0836963653564453, + "mean_token_accuracy": 0.5412560358643532, + "num_tokens": 16140741.0, + "step": 2750 + }, + { + "entropy": 1.9726036459207534, + "epoch": 0.38678485092667203, + "grad_norm": 3.625, + "learning_rate": 3.2278761061946905e-06, + "loss": 2.0361520767211916, + "mean_token_accuracy": 0.5550483211874961, + "num_tokens": 16198658.0, + "step": 2760 + }, + { + "entropy": 2.0610845535993576, + "epoch": 0.3881862453140875, + "grad_norm": 3.5, + "learning_rate": 3.220501474926254e-06, + "loss": 2.091715431213379, + "mean_token_accuracy": 0.5454363986849785, + "num_tokens": 16262732.0, + "step": 2770 + }, + { + "entropy": 2.027015134692192, + "epoch": 0.389587639701503, + "grad_norm": 3.96875, + "learning_rate": 3.213126843657817e-06, + "loss": 2.0607582092285157, + "mean_token_accuracy": 0.5436989083886147, + "num_tokens": 16323072.0, + "step": 2780 + }, + { + "entropy": 2.0341162979602814, + "epoch": 0.3909890340889185, + "grad_norm": 4.03125, + "learning_rate": 3.2057522123893808e-06, + "loss": 2.080152893066406, + "mean_token_accuracy": 0.5442491464316845, + "num_tokens": 16383825.0, + "step": 2790 + }, + { + "entropy": 1.9967200189828873, + "epoch": 0.39239042847633393, + "grad_norm": 3.21875, + "learning_rate": 3.1983775811209445e-06, + "loss": 2.0428693771362303, + "mean_token_accuracy": 0.5509172320365906, + "num_tokens": 16442919.0, + "step": 2800 + }, + { + "entropy": 2.0158123135566712, + "epoch": 0.3937918228637494, + "grad_norm": 4.03125, + "learning_rate": 3.1910029498525074e-06, + "loss": 2.0505596160888673, + "mean_token_accuracy": 0.5477202244102954, + "num_tokens": 16503445.0, + "step": 2810 + }, + { + "entropy": 2.0057578057050707, + "epoch": 0.3951932172511649, + "grad_norm": 3.71875, + "learning_rate": 3.183628318584071e-06, + "loss": 2.052252197265625, + "mean_token_accuracy": 0.5474997572600842, + "num_tokens": 16564816.0, + "step": 2820 + }, + { + "entropy": 1.9953262776136398, + "epoch": 0.3965946116385804, + "grad_norm": 3.359375, + "learning_rate": 3.1762536873156348e-06, + "loss": 2.0280698776245116, + "mean_token_accuracy": 0.5522720851004124, + "num_tokens": 16620817.0, + "step": 2830 + }, + { + "entropy": 1.9913746654987334, + "epoch": 0.3979960060259959, + "grad_norm": 3.703125, + "learning_rate": 3.1688790560471976e-06, + "loss": 2.044495391845703, + "mean_token_accuracy": 0.5515089079737663, + "num_tokens": 16680154.0, + "step": 2840 + }, + { + "entropy": 2.0292646408081056, + "epoch": 0.3993974004134113, + "grad_norm": 3.15625, + "learning_rate": 3.1615044247787613e-06, + "loss": 2.0748342514038085, + "mean_token_accuracy": 0.5438324570655823, + "num_tokens": 16741428.0, + "step": 2850 + }, + { + "entropy": 2.059218314290047, + "epoch": 0.4007987948008268, + "grad_norm": 3.171875, + "learning_rate": 3.154129793510325e-06, + "loss": 2.087131309509277, + "mean_token_accuracy": 0.5398609600961208, + "num_tokens": 16803820.0, + "step": 2860 + }, + { + "entropy": 1.988922455906868, + "epoch": 0.4022001891882423, + "grad_norm": 3.40625, + "learning_rate": 3.146755162241888e-06, + "loss": 2.014337348937988, + "mean_token_accuracy": 0.5536053828895092, + "num_tokens": 16862017.0, + "step": 2870 + }, + { + "entropy": 2.009887772798538, + "epoch": 0.4036015835756578, + "grad_norm": 3.484375, + "learning_rate": 3.1393805309734516e-06, + "loss": 2.0723125457763674, + "mean_token_accuracy": 0.5473446324467659, + "num_tokens": 16920887.0, + "step": 2880 + }, + { + "entropy": 2.06411289870739, + "epoch": 0.4050029779630733, + "grad_norm": 3.34375, + "learning_rate": 3.1320058997050153e-06, + "loss": 2.109910774230957, + "mean_token_accuracy": 0.5424987003207207, + "num_tokens": 16981303.0, + "step": 2890 + }, + { + "entropy": 1.9845988363027574, + "epoch": 0.40640437235048876, + "grad_norm": 3.53125, + "learning_rate": 3.124631268436578e-06, + "loss": 2.025481605529785, + "mean_token_accuracy": 0.5532370284199715, + "num_tokens": 17040169.0, + "step": 2900 + }, + { + "entropy": 2.049331721663475, + "epoch": 0.4078057667379042, + "grad_norm": 3.6875, + "learning_rate": 3.117256637168142e-06, + "loss": 2.112797164916992, + "mean_token_accuracy": 0.537342993915081, + "num_tokens": 17102359.0, + "step": 2910 + }, + { + "entropy": 1.972009301185608, + "epoch": 0.4092071611253197, + "grad_norm": 3.53125, + "learning_rate": 3.109882005899705e-06, + "loss": 2.0128068923950195, + "mean_token_accuracy": 0.5560865074396133, + "num_tokens": 17158577.0, + "step": 2920 + }, + { + "entropy": 2.049245524406433, + "epoch": 0.4106085555127352, + "grad_norm": 3.03125, + "learning_rate": 3.102507374631269e-06, + "loss": 2.0594350814819338, + "mean_token_accuracy": 0.5510149970650673, + "num_tokens": 17219246.0, + "step": 2930 + }, + { + "entropy": 1.9770815700292588, + "epoch": 0.41200994990015066, + "grad_norm": 3.390625, + "learning_rate": 3.095132743362832e-06, + "loss": 2.0263744354248048, + "mean_token_accuracy": 0.550960586220026, + "num_tokens": 17276673.0, + "step": 2940 + }, + { + "entropy": 2.054537570476532, + "epoch": 0.41341134428756615, + "grad_norm": 3.265625, + "learning_rate": 3.0877581120943955e-06, + "loss": 2.0924644470214844, + "mean_token_accuracy": 0.5403410479426384, + "num_tokens": 17337580.0, + "step": 2950 + }, + { + "entropy": 1.9809619694948197, + "epoch": 0.4148127386749816, + "grad_norm": 3.609375, + "learning_rate": 3.080383480825959e-06, + "loss": 2.0583356857299804, + "mean_token_accuracy": 0.555622187256813, + "num_tokens": 17395739.0, + "step": 2960 + }, + { + "entropy": 1.970469206571579, + "epoch": 0.4162141330623971, + "grad_norm": 4.21875, + "learning_rate": 3.073008849557522e-06, + "loss": 2.010905647277832, + "mean_token_accuracy": 0.5615350589156151, + "num_tokens": 17452172.0, + "step": 2970 + }, + { + "entropy": 1.9751002043485641, + "epoch": 0.41761552744981256, + "grad_norm": 3.4375, + "learning_rate": 3.0656342182890857e-06, + "loss": 2.0333751678466796, + "mean_token_accuracy": 0.5564141780138016, + "num_tokens": 17507476.0, + "step": 2980 + }, + { + "entropy": 2.0776767164468763, + "epoch": 0.41901692183722805, + "grad_norm": 3.546875, + "learning_rate": 3.0582595870206494e-06, + "loss": 2.1196556091308594, + "mean_token_accuracy": 0.5379776880145073, + "num_tokens": 17569888.0, + "step": 2990 + }, + { + "entropy": 2.0239798009395598, + "epoch": 0.42041831622464354, + "grad_norm": 3.25, + "learning_rate": 3.0508849557522123e-06, + "loss": 2.067799758911133, + "mean_token_accuracy": 0.5501157879829407, + "num_tokens": 17629147.0, + "step": 3000 + }, + { + "entropy": 2.0476420164108275, + "epoch": 0.421819710612059, + "grad_norm": 3.453125, + "learning_rate": 3.043510324483776e-06, + "loss": 2.100557327270508, + "mean_token_accuracy": 0.5459968023002147, + "num_tokens": 17683878.0, + "step": 3010 + }, + { + "entropy": 1.994628182053566, + "epoch": 0.42322110499947446, + "grad_norm": 3.578125, + "learning_rate": 3.0361356932153397e-06, + "loss": 2.0465036392211915, + "mean_token_accuracy": 0.5495273642241955, + "num_tokens": 17743179.0, + "step": 3020 + }, + { + "entropy": 2.00454503595829, + "epoch": 0.42462249938688995, + "grad_norm": 3.75, + "learning_rate": 3.0287610619469026e-06, + "loss": 2.0378849029541017, + "mean_token_accuracy": 0.5559940241277218, + "num_tokens": 17799589.0, + "step": 3030 + }, + { + "entropy": 2.0207887947559358, + "epoch": 0.42602389377430544, + "grad_norm": 3.203125, + "learning_rate": 3.0213864306784663e-06, + "loss": 2.0674230575561525, + "mean_token_accuracy": 0.5485209092497826, + "num_tokens": 17857747.0, + "step": 3040 + }, + { + "entropy": 1.9799177139997481, + "epoch": 0.42742528816172093, + "grad_norm": 4.21875, + "learning_rate": 3.01401179941003e-06, + "loss": 2.004818344116211, + "mean_token_accuracy": 0.5566212393343448, + "num_tokens": 17912862.0, + "step": 3050 + }, + { + "entropy": 1.9984881341457368, + "epoch": 0.4288266825491364, + "grad_norm": 3.34375, + "learning_rate": 3.006637168141593e-06, + "loss": 2.059418869018555, + "mean_token_accuracy": 0.550378105789423, + "num_tokens": 17972825.0, + "step": 3060 + }, + { + "entropy": 2.02097764313221, + "epoch": 0.43022807693655185, + "grad_norm": 3.046875, + "learning_rate": 2.9992625368731566e-06, + "loss": 2.080335807800293, + "mean_token_accuracy": 0.5408771336078644, + "num_tokens": 18031427.0, + "step": 3070 + }, + { + "entropy": 2.032861089706421, + "epoch": 0.43162947132396734, + "grad_norm": 3.765625, + "learning_rate": 2.9918879056047203e-06, + "loss": 2.0570114135742186, + "mean_token_accuracy": 0.5447784259915351, + "num_tokens": 18089201.0, + "step": 3080 + }, + { + "entropy": 2.0530674159526825, + "epoch": 0.43303086571138283, + "grad_norm": 3.4375, + "learning_rate": 2.9845132743362836e-06, + "loss": 2.1033819198608397, + "mean_token_accuracy": 0.5419682547450065, + "num_tokens": 18151114.0, + "step": 3090 + }, + { + "entropy": 1.988242071866989, + "epoch": 0.4344322600987983, + "grad_norm": 4.25, + "learning_rate": 2.977138643067847e-06, + "loss": 2.0315269470214843, + "mean_token_accuracy": 0.552942118793726, + "num_tokens": 18209516.0, + "step": 3100 + }, + { + "entropy": 2.0201914995908736, + "epoch": 0.4358336544862138, + "grad_norm": 4.25, + "learning_rate": 2.96976401179941e-06, + "loss": 2.067410469055176, + "mean_token_accuracy": 0.5459202371537686, + "num_tokens": 18268921.0, + "step": 3110 + }, + { + "entropy": 1.954800271987915, + "epoch": 0.43723504887362924, + "grad_norm": 3.515625, + "learning_rate": 2.962389380530974e-06, + "loss": 1.9979192733764648, + "mean_token_accuracy": 0.5595223732292652, + "num_tokens": 18325142.0, + "step": 3120 + }, + { + "entropy": 2.017845964431763, + "epoch": 0.43863644326104473, + "grad_norm": 3.5625, + "learning_rate": 2.955014749262537e-06, + "loss": 2.077107810974121, + "mean_token_accuracy": 0.5507882386445999, + "num_tokens": 18384427.0, + "step": 3130 + }, + { + "entropy": 2.0217061281204223, + "epoch": 0.4400378376484602, + "grad_norm": 3.890625, + "learning_rate": 2.9476401179941004e-06, + "loss": 2.064482307434082, + "mean_token_accuracy": 0.5540095090866088, + "num_tokens": 18442826.0, + "step": 3140 + }, + { + "entropy": 2.007353922724724, + "epoch": 0.4414392320358757, + "grad_norm": 3.96875, + "learning_rate": 2.940265486725664e-06, + "loss": 2.0431005477905275, + "mean_token_accuracy": 0.5479564107954502, + "num_tokens": 18502794.0, + "step": 3150 + }, + { + "entropy": 1.9794378250837326, + "epoch": 0.4428406264232912, + "grad_norm": 3.8125, + "learning_rate": 2.932890855457227e-06, + "loss": 2.052911376953125, + "mean_token_accuracy": 0.5498085469007492, + "num_tokens": 18560966.0, + "step": 3160 + }, + { + "entropy": 1.9761908054351807, + "epoch": 0.44424202081070663, + "grad_norm": 3.90625, + "learning_rate": 2.9255162241887907e-06, + "loss": 2.0283836364746093, + "mean_token_accuracy": 0.554186000674963, + "num_tokens": 18616685.0, + "step": 3170 + }, + { + "entropy": 2.015652891993523, + "epoch": 0.4456434151981221, + "grad_norm": 3.609375, + "learning_rate": 2.9181415929203544e-06, + "loss": 2.037789535522461, + "mean_token_accuracy": 0.5506032936275005, + "num_tokens": 18672575.0, + "step": 3180 + }, + { + "entropy": 2.0318040758371354, + "epoch": 0.4470448095855376, + "grad_norm": 3.765625, + "learning_rate": 2.9107669616519173e-06, + "loss": 2.0607019424438477, + "mean_token_accuracy": 0.5467961005866527, + "num_tokens": 18731995.0, + "step": 3190 + }, + { + "entropy": 2.0122063130140306, + "epoch": 0.4484462039729531, + "grad_norm": 3.5, + "learning_rate": 2.903392330383481e-06, + "loss": 2.051531219482422, + "mean_token_accuracy": 0.551438144594431, + "num_tokens": 18789636.0, + "step": 3200 + }, + { + "entropy": 1.9415486752986908, + "epoch": 0.4498475983603686, + "grad_norm": 3.265625, + "learning_rate": 2.8960176991150447e-06, + "loss": 1.9925420761108399, + "mean_token_accuracy": 0.5665767565369606, + "num_tokens": 18845733.0, + "step": 3210 + }, + { + "entropy": 2.091024360060692, + "epoch": 0.4512489927477841, + "grad_norm": 3.75, + "learning_rate": 2.8886430678466075e-06, + "loss": 2.150341796875, + "mean_token_accuracy": 0.5364172495901585, + "num_tokens": 18906167.0, + "step": 3220 + }, + { + "entropy": 2.0123080760240555, + "epoch": 0.4526503871351995, + "grad_norm": 3.671875, + "learning_rate": 2.8812684365781713e-06, + "loss": 2.054686737060547, + "mean_token_accuracy": 0.5530279859900474, + "num_tokens": 18965813.0, + "step": 3230 + }, + { + "entropy": 2.0785245269536974, + "epoch": 0.454051781522615, + "grad_norm": 3.53125, + "learning_rate": 2.873893805309735e-06, + "loss": 2.1151668548583986, + "mean_token_accuracy": 0.5389002986252308, + "num_tokens": 19026553.0, + "step": 3240 + }, + { + "entropy": 2.09374221265316, + "epoch": 0.4554531759100305, + "grad_norm": 3.703125, + "learning_rate": 2.8665191740412982e-06, + "loss": 2.1182989120483398, + "mean_token_accuracy": 0.5350490301847458, + "num_tokens": 19090598.0, + "step": 3250 + }, + { + "entropy": 2.040800130367279, + "epoch": 0.45685457029744597, + "grad_norm": 3.296875, + "learning_rate": 2.8591445427728615e-06, + "loss": 2.0858171463012694, + "mean_token_accuracy": 0.5454586446285248, + "num_tokens": 19149415.0, + "step": 3260 + }, + { + "entropy": 2.0399263858795167, + "epoch": 0.45825596468486146, + "grad_norm": 4.125, + "learning_rate": 2.8517699115044252e-06, + "loss": 2.071771812438965, + "mean_token_accuracy": 0.5436090737581253, + "num_tokens": 19209559.0, + "step": 3270 + }, + { + "entropy": 2.0385482013225555, + "epoch": 0.4596573590722769, + "grad_norm": 3.53125, + "learning_rate": 2.8443952802359885e-06, + "loss": 2.076809310913086, + "mean_token_accuracy": 0.5420018076896668, + "num_tokens": 19270353.0, + "step": 3280 + }, + { + "entropy": 2.0251143515110015, + "epoch": 0.4610587534596924, + "grad_norm": 3.5625, + "learning_rate": 2.837020648967552e-06, + "loss": 2.059922218322754, + "mean_token_accuracy": 0.551348476856947, + "num_tokens": 19328176.0, + "step": 3290 + }, + { + "entropy": 2.0049924582242964, + "epoch": 0.46246014784710787, + "grad_norm": 3.59375, + "learning_rate": 2.829646017699115e-06, + "loss": 2.062200927734375, + "mean_token_accuracy": 0.5466015346348285, + "num_tokens": 19386709.0, + "step": 3300 + }, + { + "entropy": 1.991297048330307, + "epoch": 0.46386154223452336, + "grad_norm": 3.890625, + "learning_rate": 2.822271386430679e-06, + "loss": 2.0469194412231446, + "mean_token_accuracy": 0.5508559308946133, + "num_tokens": 19444741.0, + "step": 3310 + }, + { + "entropy": 2.0167817562818526, + "epoch": 0.46526293662193885, + "grad_norm": 3.625, + "learning_rate": 2.814896755162242e-06, + "loss": 2.0712039947509764, + "mean_token_accuracy": 0.5523478977382184, + "num_tokens": 19503028.0, + "step": 3320 + }, + { + "entropy": 2.0049867272377013, + "epoch": 0.4666643310093543, + "grad_norm": 4.09375, + "learning_rate": 2.8075221238938054e-06, + "loss": 2.043589401245117, + "mean_token_accuracy": 0.5559856280684471, + "num_tokens": 19558987.0, + "step": 3330 + }, + { + "entropy": 2.005906584858894, + "epoch": 0.46806572539676977, + "grad_norm": 3.84375, + "learning_rate": 2.800147492625369e-06, + "loss": 2.0505804061889648, + "mean_token_accuracy": 0.5503264181315899, + "num_tokens": 19618160.0, + "step": 3340 + }, + { + "entropy": 2.000386303663254, + "epoch": 0.46946711978418526, + "grad_norm": 3.296875, + "learning_rate": 2.792772861356932e-06, + "loss": 2.054909324645996, + "mean_token_accuracy": 0.5460795871913433, + "num_tokens": 19677802.0, + "step": 3350 + }, + { + "entropy": 2.0026027977466585, + "epoch": 0.47086851417160075, + "grad_norm": 3.453125, + "learning_rate": 2.7853982300884957e-06, + "loss": 2.049390411376953, + "mean_token_accuracy": 0.551422468572855, + "num_tokens": 19735520.0, + "step": 3360 + }, + { + "entropy": 2.053162467479706, + "epoch": 0.47226990855901624, + "grad_norm": 3.140625, + "learning_rate": 2.7780235988200594e-06, + "loss": 2.097758102416992, + "mean_token_accuracy": 0.5442638322710991, + "num_tokens": 19795972.0, + "step": 3370 + }, + { + "entropy": 2.0171428352594374, + "epoch": 0.4736713029464317, + "grad_norm": 3.5625, + "learning_rate": 2.7706489675516222e-06, + "loss": 2.060979652404785, + "mean_token_accuracy": 0.5500943906605243, + "num_tokens": 19852770.0, + "step": 3380 + }, + { + "entropy": 2.065670907497406, + "epoch": 0.47507269733384716, + "grad_norm": 3.59375, + "learning_rate": 2.763274336283186e-06, + "loss": 2.0913238525390625, + "mean_token_accuracy": 0.5402901194989681, + "num_tokens": 19913506.0, + "step": 3390 + }, + { + "entropy": 1.9373196512460709, + "epoch": 0.47647409172126265, + "grad_norm": 3.78125, + "learning_rate": 2.7558997050147496e-06, + "loss": 2.016362762451172, + "mean_token_accuracy": 0.5609184913337231, + "num_tokens": 19969123.0, + "step": 3400 + }, + { + "entropy": 2.0038623034954073, + "epoch": 0.47787548610867814, + "grad_norm": 3.875, + "learning_rate": 2.7485250737463133e-06, + "loss": 2.0328384399414063, + "mean_token_accuracy": 0.5531811892986298, + "num_tokens": 20026637.0, + "step": 3410 + }, + { + "entropy": 2.025873976945877, + "epoch": 0.4792768804960936, + "grad_norm": 3.78125, + "learning_rate": 2.741150442477876e-06, + "loss": 2.074894142150879, + "mean_token_accuracy": 0.5467535518109798, + "num_tokens": 20084776.0, + "step": 3420 + }, + { + "entropy": 2.0325380116701126, + "epoch": 0.4806782748835091, + "grad_norm": 3.796875, + "learning_rate": 2.73377581120944e-06, + "loss": 2.0545236587524416, + "mean_token_accuracy": 0.5489490836858749, + "num_tokens": 20143024.0, + "step": 3430 + }, + { + "entropy": 2.0260173439979554, + "epoch": 0.48207966927092455, + "grad_norm": 3.5625, + "learning_rate": 2.726401179941003e-06, + "loss": 2.067676544189453, + "mean_token_accuracy": 0.5490511626005172, + "num_tokens": 20199880.0, + "step": 3440 + }, + { + "entropy": 2.042122220993042, + "epoch": 0.48348106365834004, + "grad_norm": 3.65625, + "learning_rate": 2.7190265486725665e-06, + "loss": 2.0859392166137694, + "mean_token_accuracy": 0.5460739366710186, + "num_tokens": 20257116.0, + "step": 3450 + }, + { + "entropy": 2.0438840121030806, + "epoch": 0.4848824580457555, + "grad_norm": 4.59375, + "learning_rate": 2.71165191740413e-06, + "loss": 2.067841339111328, + "mean_token_accuracy": 0.5486466385424137, + "num_tokens": 20315069.0, + "step": 3460 + }, + { + "entropy": 2.0156511545181273, + "epoch": 0.486283852433171, + "grad_norm": 3.84375, + "learning_rate": 2.7042772861356935e-06, + "loss": 2.0815851211547853, + "mean_token_accuracy": 0.5475437700748443, + "num_tokens": 20375449.0, + "step": 3470 + }, + { + "entropy": 2.069836437702179, + "epoch": 0.4876852468205865, + "grad_norm": 3.375, + "learning_rate": 2.6969026548672568e-06, + "loss": 2.150330352783203, + "mean_token_accuracy": 0.5382217861711979, + "num_tokens": 20438695.0, + "step": 3480 + }, + { + "entropy": 2.0334780693054197, + "epoch": 0.48908664120800194, + "grad_norm": 3.625, + "learning_rate": 2.68952802359882e-06, + "loss": 2.0477685928344727, + "mean_token_accuracy": 0.5468250714242459, + "num_tokens": 20498908.0, + "step": 3490 + }, + { + "entropy": 2.0661162793636323, + "epoch": 0.4904880355954174, + "grad_norm": 3.8125, + "learning_rate": 2.6821533923303838e-06, + "loss": 2.118264007568359, + "mean_token_accuracy": 0.5386695951223374, + "num_tokens": 20558837.0, + "step": 3500 + }, + { + "entropy": 2.0294057965278625, + "epoch": 0.4918894299828329, + "grad_norm": 3.40625, + "learning_rate": 2.674778761061947e-06, + "loss": 2.0714643478393553, + "mean_token_accuracy": 0.544650749117136, + "num_tokens": 20621754.0, + "step": 3510 + }, + { + "entropy": 1.9884522348642348, + "epoch": 0.4932908243702484, + "grad_norm": 3.21875, + "learning_rate": 2.6674041297935103e-06, + "loss": 2.037005805969238, + "mean_token_accuracy": 0.5551797017455101, + "num_tokens": 20678506.0, + "step": 3520 + }, + { + "entropy": 2.057207414507866, + "epoch": 0.4946922187576639, + "grad_norm": 4.28125, + "learning_rate": 2.660029498525074e-06, + "loss": 2.0946880340576173, + "mean_token_accuracy": 0.5411521509289742, + "num_tokens": 20738044.0, + "step": 3530 + }, + { + "entropy": 2.037112107872963, + "epoch": 0.4960936131450793, + "grad_norm": 3.75, + "learning_rate": 2.6526548672566373e-06, + "loss": 2.0831565856933594, + "mean_token_accuracy": 0.5441927686333656, + "num_tokens": 20795455.0, + "step": 3540 + }, + { + "entropy": 2.0589177668094636, + "epoch": 0.4974950075324948, + "grad_norm": 3.390625, + "learning_rate": 2.6452802359882006e-06, + "loss": 2.078609085083008, + "mean_token_accuracy": 0.5393376901745797, + "num_tokens": 20857923.0, + "step": 3550 + }, + { + "entropy": 1.9840614557266236, + "epoch": 0.4988964019199103, + "grad_norm": 3.8125, + "learning_rate": 2.6379056047197643e-06, + "loss": 2.0415136337280275, + "mean_token_accuracy": 0.5558890357613564, + "num_tokens": 20914435.0, + "step": 3560 + }, + { + "entropy": 2.0576535284519197, + "epoch": 0.5002977963073257, + "grad_norm": 3.5625, + "learning_rate": 2.630530973451328e-06, + "loss": 2.100432205200195, + "mean_token_accuracy": 0.5417121030390263, + "num_tokens": 20975121.0, + "step": 3570 + }, + { + "entropy": 1.977579164505005, + "epoch": 0.5016991906947412, + "grad_norm": 3.28125, + "learning_rate": 2.623156342182891e-06, + "loss": 1.9968931198120117, + "mean_token_accuracy": 0.5547671675682068, + "num_tokens": 21033320.0, + "step": 3580 + }, + { + "entropy": 1.994506189227104, + "epoch": 0.5031005850821567, + "grad_norm": 3.640625, + "learning_rate": 2.6157817109144546e-06, + "loss": 2.055440139770508, + "mean_token_accuracy": 0.5502506762742996, + "num_tokens": 21090281.0, + "step": 3590 + }, + { + "entropy": 2.013153353333473, + "epoch": 0.5045019794695722, + "grad_norm": 3.578125, + "learning_rate": 2.6084070796460183e-06, + "loss": 2.0509225845336916, + "mean_token_accuracy": 0.5509684525430203, + "num_tokens": 21147874.0, + "step": 3600 + }, + { + "entropy": 2.0328510105609894, + "epoch": 0.5059033738569877, + "grad_norm": 3.640625, + "learning_rate": 2.601032448377581e-06, + "loss": 2.063313293457031, + "mean_token_accuracy": 0.5480868622660637, + "num_tokens": 21208265.0, + "step": 3610 + }, + { + "entropy": 2.0235374003648756, + "epoch": 0.5073047682444032, + "grad_norm": 4.1875, + "learning_rate": 2.593657817109145e-06, + "loss": 2.049294090270996, + "mean_token_accuracy": 0.5469991482794285, + "num_tokens": 21267181.0, + "step": 3620 + }, + { + "entropy": 2.0558235228061674, + "epoch": 0.5087061626318187, + "grad_norm": 3.9375, + "learning_rate": 2.586283185840708e-06, + "loss": 2.0990793228149416, + "mean_token_accuracy": 0.5421556040644646, + "num_tokens": 21330102.0, + "step": 3630 + }, + { + "entropy": 2.007439586520195, + "epoch": 0.5101075570192342, + "grad_norm": 3.859375, + "learning_rate": 2.5789085545722714e-06, + "loss": 2.031885528564453, + "mean_token_accuracy": 0.5457480415701866, + "num_tokens": 21391219.0, + "step": 3640 + }, + { + "entropy": 2.0088407576084135, + "epoch": 0.5115089514066496, + "grad_norm": 3.4375, + "learning_rate": 2.571533923303835e-06, + "loss": 2.059234046936035, + "mean_token_accuracy": 0.5426969014108181, + "num_tokens": 21450040.0, + "step": 3650 + }, + { + "entropy": 2.077793797850609, + "epoch": 0.5129103457940651, + "grad_norm": 3.96875, + "learning_rate": 2.5641592920353984e-06, + "loss": 2.12182731628418, + "mean_token_accuracy": 0.536633738130331, + "num_tokens": 21513501.0, + "step": 3660 + }, + { + "entropy": 2.0497137665748597, + "epoch": 0.5143117401814806, + "grad_norm": 3.734375, + "learning_rate": 2.5567846607669617e-06, + "loss": 2.0897573471069335, + "mean_token_accuracy": 0.5466533005237579, + "num_tokens": 21571729.0, + "step": 3670 + }, + { + "entropy": 2.069832128286362, + "epoch": 0.515713134568896, + "grad_norm": 3.359375, + "learning_rate": 2.5494100294985254e-06, + "loss": 2.104374122619629, + "mean_token_accuracy": 0.5386730141937732, + "num_tokens": 21634312.0, + "step": 3680 + }, + { + "entropy": 1.9782506972551346, + "epoch": 0.5171145289563115, + "grad_norm": 3.8125, + "learning_rate": 2.5420353982300887e-06, + "loss": 2.0286142349243166, + "mean_token_accuracy": 0.5537291884422302, + "num_tokens": 21692193.0, + "step": 3690 + }, + { + "entropy": 2.032965365052223, + "epoch": 0.518515923343727, + "grad_norm": 3.1875, + "learning_rate": 2.534660766961652e-06, + "loss": 2.0770105361938476, + "mean_token_accuracy": 0.5466852344572544, + "num_tokens": 21754024.0, + "step": 3700 + }, + { + "entropy": 2.0536326289176943, + "epoch": 0.5199173177311425, + "grad_norm": 4.28125, + "learning_rate": 2.5272861356932153e-06, + "loss": 2.093625068664551, + "mean_token_accuracy": 0.5425667576491833, + "num_tokens": 21814964.0, + "step": 3710 + }, + { + "entropy": 2.0720867395401, + "epoch": 0.521318712118558, + "grad_norm": 3.859375, + "learning_rate": 2.519911504424779e-06, + "loss": 2.115810012817383, + "mean_token_accuracy": 0.5382542759180069, + "num_tokens": 21876670.0, + "step": 3720 + }, + { + "entropy": 2.0418166905641555, + "epoch": 0.5227201065059734, + "grad_norm": 3.234375, + "learning_rate": 2.5125368731563427e-06, + "loss": 2.0819705963134765, + "mean_token_accuracy": 0.5415958672761917, + "num_tokens": 21936186.0, + "step": 3730 + }, + { + "entropy": 1.9627337753772736, + "epoch": 0.5241215008933889, + "grad_norm": 3.71875, + "learning_rate": 2.5051622418879056e-06, + "loss": 2.0003637313842773, + "mean_token_accuracy": 0.5582445688545704, + "num_tokens": 21990937.0, + "step": 3740 + }, + { + "entropy": 2.041642299294472, + "epoch": 0.5255228952808044, + "grad_norm": 3.421875, + "learning_rate": 2.4977876106194693e-06, + "loss": 2.0848148345947264, + "mean_token_accuracy": 0.5404154628515243, + "num_tokens": 22049685.0, + "step": 3750 + }, + { + "entropy": 2.0510466665029528, + "epoch": 0.5269242896682199, + "grad_norm": 3.625, + "learning_rate": 2.4904129793510326e-06, + "loss": 2.072761344909668, + "mean_token_accuracy": 0.5417856775224209, + "num_tokens": 22111400.0, + "step": 3760 + }, + { + "entropy": 2.0478140890598295, + "epoch": 0.5283256840556354, + "grad_norm": 3.484375, + "learning_rate": 2.4830383480825963e-06, + "loss": 2.074302101135254, + "mean_token_accuracy": 0.5471907205879688, + "num_tokens": 22169953.0, + "step": 3770 + }, + { + "entropy": 1.9797133803367615, + "epoch": 0.5297270784430508, + "grad_norm": 3.5625, + "learning_rate": 2.4756637168141596e-06, + "loss": 2.05294246673584, + "mean_token_accuracy": 0.5519698172807693, + "num_tokens": 22225622.0, + "step": 3780 + }, + { + "entropy": 2.0385789662599563, + "epoch": 0.5311284728304663, + "grad_norm": 3.71875, + "learning_rate": 2.468289085545723e-06, + "loss": 2.064686965942383, + "mean_token_accuracy": 0.5464815199375153, + "num_tokens": 22286974.0, + "step": 3790 + }, + { + "entropy": 2.001328268647194, + "epoch": 0.5325298672178818, + "grad_norm": 3.5, + "learning_rate": 2.4609144542772865e-06, + "loss": 2.0372297286987306, + "mean_token_accuracy": 0.5501808658242225, + "num_tokens": 22344418.0, + "step": 3800 + }, + { + "entropy": 2.0195149928331375, + "epoch": 0.5339312616052972, + "grad_norm": 3.4375, + "learning_rate": 2.45353982300885e-06, + "loss": 2.0701387405395506, + "mean_token_accuracy": 0.5477361977100372, + "num_tokens": 22408335.0, + "step": 3810 + }, + { + "entropy": 2.0369257777929306, + "epoch": 0.5353326559927127, + "grad_norm": 4.03125, + "learning_rate": 2.446165191740413e-06, + "loss": 2.0915966033935547, + "mean_token_accuracy": 0.5456332109868527, + "num_tokens": 22466521.0, + "step": 3820 + }, + { + "entropy": 1.9785560071468353, + "epoch": 0.5367340503801282, + "grad_norm": 3.546875, + "learning_rate": 2.438790560471977e-06, + "loss": 2.0245759963989256, + "mean_token_accuracy": 0.5537137188017368, + "num_tokens": 22522827.0, + "step": 3830 + }, + { + "entropy": 2.009777495265007, + "epoch": 0.5381354447675437, + "grad_norm": 3.609375, + "learning_rate": 2.43141592920354e-06, + "loss": 2.04541130065918, + "mean_token_accuracy": 0.5507000245153904, + "num_tokens": 22579352.0, + "step": 3840 + }, + { + "entropy": 2.0158716648817063, + "epoch": 0.5395368391549592, + "grad_norm": 3.953125, + "learning_rate": 2.4240412979351034e-06, + "loss": 2.0709373474121096, + "mean_token_accuracy": 0.5494461841881275, + "num_tokens": 22633813.0, + "step": 3850 + }, + { + "entropy": 1.9816727459430694, + "epoch": 0.5409382335423747, + "grad_norm": 3.703125, + "learning_rate": 2.4166666666666667e-06, + "loss": 2.028717803955078, + "mean_token_accuracy": 0.5602708630263805, + "num_tokens": 22690510.0, + "step": 3860 + }, + { + "entropy": 2.0063779681921003, + "epoch": 0.5423396279297902, + "grad_norm": 3.984375, + "learning_rate": 2.4092920353982304e-06, + "loss": 2.027983856201172, + "mean_token_accuracy": 0.5518012642860413, + "num_tokens": 22751639.0, + "step": 3870 + }, + { + "entropy": 1.9799637734889983, + "epoch": 0.5437410223172057, + "grad_norm": 4.0, + "learning_rate": 2.4019174041297937e-06, + "loss": 2.0037744522094725, + "mean_token_accuracy": 0.5529186941683293, + "num_tokens": 22809956.0, + "step": 3880 + }, + { + "entropy": 1.9927177160978318, + "epoch": 0.545142416704621, + "grad_norm": 3.515625, + "learning_rate": 2.394542772861357e-06, + "loss": 2.0321245193481445, + "mean_token_accuracy": 0.5531116656959056, + "num_tokens": 22868091.0, + "step": 3890 + }, + { + "entropy": 2.026549074053764, + "epoch": 0.5465438110920365, + "grad_norm": 7.3125, + "learning_rate": 2.3871681415929202e-06, + "loss": 2.0672191619873046, + "mean_token_accuracy": 0.5436496593058109, + "num_tokens": 22929148.0, + "step": 3900 + }, + { + "entropy": 1.9840692311525345, + "epoch": 0.547945205479452, + "grad_norm": 3.375, + "learning_rate": 2.379793510324484e-06, + "loss": 2.0380395889282226, + "mean_token_accuracy": 0.5514030374586583, + "num_tokens": 22988015.0, + "step": 3910 + }, + { + "entropy": 2.035917988419533, + "epoch": 0.5493465998668675, + "grad_norm": 3.578125, + "learning_rate": 2.3724188790560472e-06, + "loss": 2.0763721466064453, + "mean_token_accuracy": 0.5406392715871334, + "num_tokens": 23049171.0, + "step": 3920 + }, + { + "entropy": 1.99947247505188, + "epoch": 0.550747994254283, + "grad_norm": 3.828125, + "learning_rate": 2.365044247787611e-06, + "loss": 2.052603530883789, + "mean_token_accuracy": 0.5530374385416508, + "num_tokens": 23105763.0, + "step": 3930 + }, + { + "entropy": 2.0505595415830613, + "epoch": 0.5521493886416985, + "grad_norm": 3.59375, + "learning_rate": 2.3576696165191742e-06, + "loss": 2.094431495666504, + "mean_token_accuracy": 0.5417370781302452, + "num_tokens": 23166182.0, + "step": 3940 + }, + { + "entropy": 2.050633665919304, + "epoch": 0.553550783029114, + "grad_norm": 4.3125, + "learning_rate": 2.3502949852507375e-06, + "loss": 2.0828453063964845, + "mean_token_accuracy": 0.5454864524304867, + "num_tokens": 23227099.0, + "step": 3950 + }, + { + "entropy": 2.0483777284622193, + "epoch": 0.5549521774165295, + "grad_norm": 3.625, + "learning_rate": 2.3429203539823012e-06, + "loss": 2.0835256576538086, + "mean_token_accuracy": 0.5411513276398182, + "num_tokens": 23288781.0, + "step": 3960 + }, + { + "entropy": 2.064347520470619, + "epoch": 0.556353571803945, + "grad_norm": 3.265625, + "learning_rate": 2.3355457227138645e-06, + "loss": 2.107004165649414, + "mean_token_accuracy": 0.5416832119226456, + "num_tokens": 23350486.0, + "step": 3970 + }, + { + "entropy": 2.0445487290620803, + "epoch": 0.5577549661913604, + "grad_norm": 3.53125, + "learning_rate": 2.328171091445428e-06, + "loss": 2.08209342956543, + "mean_token_accuracy": 0.5434218257665634, + "num_tokens": 23411926.0, + "step": 3980 + }, + { + "entropy": 2.0346221029758453, + "epoch": 0.5591563605787759, + "grad_norm": 3.234375, + "learning_rate": 2.3207964601769915e-06, + "loss": 2.0527862548828124, + "mean_token_accuracy": 0.5450842939317226, + "num_tokens": 23472131.0, + "step": 3990 + }, + { + "entropy": 2.0058074325323103, + "epoch": 0.5605577549661913, + "grad_norm": 3.78125, + "learning_rate": 2.313421828908555e-06, + "loss": 2.0455265045166016, + "mean_token_accuracy": 0.5474824257194996, + "num_tokens": 23531627.0, + "step": 4000 + }, + { + "entropy": 2.0183048009872437, + "epoch": 0.5619591493536068, + "grad_norm": 3.8125, + "learning_rate": 2.3060471976401185e-06, + "loss": 2.0639692306518556, + "mean_token_accuracy": 0.5463249370455742, + "num_tokens": 23588833.0, + "step": 4010 + }, + { + "entropy": 1.9974335372447967, + "epoch": 0.5633605437410223, + "grad_norm": 3.703125, + "learning_rate": 2.2986725663716818e-06, + "loss": 2.0383316040039063, + "mean_token_accuracy": 0.5446601718664169, + "num_tokens": 23648905.0, + "step": 4020 + }, + { + "entropy": 2.0130158990621565, + "epoch": 0.5647619381284378, + "grad_norm": 3.828125, + "learning_rate": 2.291297935103245e-06, + "loss": 2.0863428115844727, + "mean_token_accuracy": 0.5524916179478169, + "num_tokens": 23708346.0, + "step": 4030 + }, + { + "entropy": 2.0139412343502046, + "epoch": 0.5661633325158533, + "grad_norm": 3.40625, + "learning_rate": 2.2839233038348084e-06, + "loss": 2.0729434967041014, + "mean_token_accuracy": 0.5456508606672287, + "num_tokens": 23769092.0, + "step": 4040 + }, + { + "entropy": 2.006833681464195, + "epoch": 0.5675647269032688, + "grad_norm": 3.59375, + "learning_rate": 2.276548672566372e-06, + "loss": 2.030129814147949, + "mean_token_accuracy": 0.5505193144083023, + "num_tokens": 23826674.0, + "step": 4050 + }, + { + "entropy": 1.9849727123975753, + "epoch": 0.5689661212906842, + "grad_norm": 5.96875, + "learning_rate": 2.2691740412979353e-06, + "loss": 2.0195226669311523, + "mean_token_accuracy": 0.5553907476365566, + "num_tokens": 23883950.0, + "step": 4060 + }, + { + "entropy": 2.0020676136016844, + "epoch": 0.5703675156780997, + "grad_norm": 3.421875, + "learning_rate": 2.2617994100294986e-06, + "loss": 2.029514503479004, + "mean_token_accuracy": 0.546630323678255, + "num_tokens": 23943657.0, + "step": 4070 + }, + { + "entropy": 2.0378906697034838, + "epoch": 0.5717689100655152, + "grad_norm": 3.40625, + "learning_rate": 2.254424778761062e-06, + "loss": 2.090596008300781, + "mean_token_accuracy": 0.5463094800710678, + "num_tokens": 24003246.0, + "step": 4080 + }, + { + "entropy": 2.0127851188182833, + "epoch": 0.5731703044529307, + "grad_norm": 4.09375, + "learning_rate": 2.2470501474926256e-06, + "loss": 2.06030330657959, + "mean_token_accuracy": 0.5467360019683838, + "num_tokens": 24063024.0, + "step": 4090 + }, + { + "entropy": 1.999370601773262, + "epoch": 0.5745716988403461, + "grad_norm": 3.5, + "learning_rate": 2.239675516224189e-06, + "loss": 2.0564907073974608, + "mean_token_accuracy": 0.5505198113620281, + "num_tokens": 24125799.0, + "step": 4100 + }, + { + "entropy": 2.0180424094200133, + "epoch": 0.5759730932277616, + "grad_norm": 3.78125, + "learning_rate": 2.232300884955752e-06, + "loss": 2.0601451873779295, + "mean_token_accuracy": 0.5503242604434491, + "num_tokens": 24181795.0, + "step": 4110 + }, + { + "entropy": 2.0518387466669084, + "epoch": 0.5773744876151771, + "grad_norm": 3.828125, + "learning_rate": 2.224926253687316e-06, + "loss": 2.1045751571655273, + "mean_token_accuracy": 0.5413469187915325, + "num_tokens": 24241030.0, + "step": 4120 + }, + { + "entropy": 2.071943160891533, + "epoch": 0.5787758820025926, + "grad_norm": 3.390625, + "learning_rate": 2.217551622418879e-06, + "loss": 2.114454650878906, + "mean_token_accuracy": 0.5378143228590488, + "num_tokens": 24302855.0, + "step": 4130 + }, + { + "entropy": 1.9547184824943542, + "epoch": 0.580177276390008, + "grad_norm": 4.1875, + "learning_rate": 2.2101769911504425e-06, + "loss": 2.0317546844482424, + "mean_token_accuracy": 0.5589460290968418, + "num_tokens": 24357551.0, + "step": 4140 + }, + { + "entropy": 2.057077610492706, + "epoch": 0.5815786707774235, + "grad_norm": 3.75, + "learning_rate": 2.202802359882006e-06, + "loss": 2.083742141723633, + "mean_token_accuracy": 0.5418501496315002, + "num_tokens": 24419099.0, + "step": 4150 + }, + { + "entropy": 1.984746727347374, + "epoch": 0.582980065164839, + "grad_norm": 3.65625, + "learning_rate": 2.1954277286135695e-06, + "loss": 2.0364831924438476, + "mean_token_accuracy": 0.5524508558213711, + "num_tokens": 24477573.0, + "step": 4160 + }, + { + "entropy": 1.9938465535640717, + "epoch": 0.5843814595522545, + "grad_norm": 3.9375, + "learning_rate": 2.188053097345133e-06, + "loss": 2.0479278564453125, + "mean_token_accuracy": 0.5529054492712021, + "num_tokens": 24534505.0, + "step": 4170 + }, + { + "entropy": 2.044906014204025, + "epoch": 0.58578285393967, + "grad_norm": 3.6875, + "learning_rate": 2.1806784660766965e-06, + "loss": 2.0834814071655274, + "mean_token_accuracy": 0.545136384665966, + "num_tokens": 24595351.0, + "step": 4180 + }, + { + "entropy": 2.0310352951288224, + "epoch": 0.5871842483270855, + "grad_norm": 3.25, + "learning_rate": 2.1733038348082597e-06, + "loss": 2.081809425354004, + "mean_token_accuracy": 0.5477789878845215, + "num_tokens": 24653298.0, + "step": 4190 + }, + { + "entropy": 1.9692771136760712, + "epoch": 0.588585642714501, + "grad_norm": 3.828125, + "learning_rate": 2.1659292035398235e-06, + "loss": 1.99939022064209, + "mean_token_accuracy": 0.5581804670393467, + "num_tokens": 24709554.0, + "step": 4200 + }, + { + "entropy": 2.012720575928688, + "epoch": 0.5899870371019164, + "grad_norm": 3.953125, + "learning_rate": 2.1585545722713867e-06, + "loss": 2.050390625, + "mean_token_accuracy": 0.5485328942537308, + "num_tokens": 24768020.0, + "step": 4210 + }, + { + "entropy": 2.0160817176103594, + "epoch": 0.5913884314893318, + "grad_norm": 4.09375, + "learning_rate": 2.15117994100295e-06, + "loss": 2.0523845672607424, + "mean_token_accuracy": 0.5486735932528972, + "num_tokens": 24827064.0, + "step": 4220 + }, + { + "entropy": 2.046040180325508, + "epoch": 0.5927898258767473, + "grad_norm": 3.65625, + "learning_rate": 2.1438053097345133e-06, + "loss": 2.100558280944824, + "mean_token_accuracy": 0.538923604041338, + "num_tokens": 24888245.0, + "step": 4230 + }, + { + "entropy": 1.95506771504879, + "epoch": 0.5941912202641628, + "grad_norm": 4.09375, + "learning_rate": 2.136430678466077e-06, + "loss": 2.005992126464844, + "mean_token_accuracy": 0.5591956958174705, + "num_tokens": 24944271.0, + "step": 4240 + }, + { + "entropy": 2.0218535140156746, + "epoch": 0.5955926146515783, + "grad_norm": 4.09375, + "learning_rate": 2.1290560471976403e-06, + "loss": 2.1161304473876954, + "mean_token_accuracy": 0.5459480352699757, + "num_tokens": 25007787.0, + "step": 4250 + }, + { + "entropy": 1.9901989072561264, + "epoch": 0.5969940090389938, + "grad_norm": 4.03125, + "learning_rate": 2.1216814159292036e-06, + "loss": 2.0250368118286133, + "mean_token_accuracy": 0.555504946410656, + "num_tokens": 25064609.0, + "step": 4260 + }, + { + "entropy": 2.04794394671917, + "epoch": 0.5983954034264093, + "grad_norm": 3.640625, + "learning_rate": 2.114306784660767e-06, + "loss": 2.110860252380371, + "mean_token_accuracy": 0.5407655954360961, + "num_tokens": 25125069.0, + "step": 4270 + }, + { + "entropy": 1.9708813965320586, + "epoch": 0.5997967978138248, + "grad_norm": 3.765625, + "learning_rate": 2.1069321533923306e-06, + "loss": 2.01574649810791, + "mean_token_accuracy": 0.5548791468143464, + "num_tokens": 25179831.0, + "step": 4280 + }, + { + "entropy": 1.9761624753475189, + "epoch": 0.6011981922012403, + "grad_norm": 3.421875, + "learning_rate": 2.099557522123894e-06, + "loss": 2.03051700592041, + "mean_token_accuracy": 0.5576313950121403, + "num_tokens": 25237191.0, + "step": 4290 + }, + { + "entropy": 2.0774387955665587, + "epoch": 0.6025995865886558, + "grad_norm": 3.78125, + "learning_rate": 2.092182890855457e-06, + "loss": 2.092380142211914, + "mean_token_accuracy": 0.5382080920040607, + "num_tokens": 25297215.0, + "step": 4300 + }, + { + "entropy": 2.0198237359523774, + "epoch": 0.6040009809760712, + "grad_norm": 3.828125, + "learning_rate": 2.084808259587021e-06, + "loss": 2.0585187911987304, + "mean_token_accuracy": 0.5482968099415302, + "num_tokens": 25354816.0, + "step": 4310 + }, + { + "entropy": 2.045816385746002, + "epoch": 0.6054023753634866, + "grad_norm": 3.703125, + "learning_rate": 2.077433628318584e-06, + "loss": 2.0543848037719727, + "mean_token_accuracy": 0.5452761068940163, + "num_tokens": 25413901.0, + "step": 4320 + }, + { + "entropy": 1.9651571273803712, + "epoch": 0.6068037697509021, + "grad_norm": 3.5625, + "learning_rate": 2.070058997050148e-06, + "loss": 2.0480159759521483, + "mean_token_accuracy": 0.555195688456297, + "num_tokens": 25468413.0, + "step": 4330 + }, + { + "entropy": 2.0128135979175568, + "epoch": 0.6082051641383176, + "grad_norm": 4.1875, + "learning_rate": 2.062684365781711e-06, + "loss": 2.05289306640625, + "mean_token_accuracy": 0.5511495009064674, + "num_tokens": 25526075.0, + "step": 4340 + }, + { + "entropy": 1.9372184723615646, + "epoch": 0.6096065585257331, + "grad_norm": 3.71875, + "learning_rate": 2.0553097345132744e-06, + "loss": 1.9700294494628907, + "mean_token_accuracy": 0.5638393282890319, + "num_tokens": 25582379.0, + "step": 4350 + }, + { + "entropy": 2.0384511291980743, + "epoch": 0.6110079529131486, + "grad_norm": 3.4375, + "learning_rate": 2.047935103244838e-06, + "loss": 2.091180992126465, + "mean_token_accuracy": 0.5433830074965954, + "num_tokens": 25639216.0, + "step": 4360 + }, + { + "entropy": 1.9909147679805757, + "epoch": 0.6124093473005641, + "grad_norm": 3.546875, + "learning_rate": 2.0405604719764014e-06, + "loss": 2.054271697998047, + "mean_token_accuracy": 0.5466593787074089, + "num_tokens": 25700427.0, + "step": 4370 + }, + { + "entropy": 2.04966981112957, + "epoch": 0.6138107416879796, + "grad_norm": 3.15625, + "learning_rate": 2.0331858407079647e-06, + "loss": 2.1054479598999025, + "mean_token_accuracy": 0.5437165200710297, + "num_tokens": 25762410.0, + "step": 4380 + }, + { + "entropy": 2.080600243806839, + "epoch": 0.615212136075395, + "grad_norm": 3.578125, + "learning_rate": 2.0258112094395284e-06, + "loss": 2.1234355926513673, + "mean_token_accuracy": 0.5354627579450607, + "num_tokens": 25823106.0, + "step": 4390 + }, + { + "entropy": 2.010471725463867, + "epoch": 0.6166135304628105, + "grad_norm": 3.5, + "learning_rate": 2.0184365781710917e-06, + "loss": 2.0771133422851564, + "mean_token_accuracy": 0.550500302016735, + "num_tokens": 25882208.0, + "step": 4400 + }, + { + "entropy": 2.046468684077263, + "epoch": 0.618014924850226, + "grad_norm": 3.328125, + "learning_rate": 2.011061946902655e-06, + "loss": 2.0935302734375, + "mean_token_accuracy": 0.5437944456934929, + "num_tokens": 25942159.0, + "step": 4410 + }, + { + "entropy": 1.9386399030685424, + "epoch": 0.6194163192376414, + "grad_norm": 3.59375, + "learning_rate": 2.0036873156342183e-06, + "loss": 2.0005924224853517, + "mean_token_accuracy": 0.5630344584584236, + "num_tokens": 25999031.0, + "step": 4420 + }, + { + "entropy": 1.9665241211652755, + "epoch": 0.6208177136250569, + "grad_norm": 4.15625, + "learning_rate": 1.996312684365782e-06, + "loss": 2.0306076049804687, + "mean_token_accuracy": 0.5548419393599033, + "num_tokens": 26057864.0, + "step": 4430 + }, + { + "entropy": 2.073987701535225, + "epoch": 0.6222191080124724, + "grad_norm": 3.34375, + "learning_rate": 1.9889380530973453e-06, + "loss": 2.129755210876465, + "mean_token_accuracy": 0.5381237179040909, + "num_tokens": 26116078.0, + "step": 4440 + }, + { + "entropy": 2.070046308636665, + "epoch": 0.6236205023998879, + "grad_norm": 3.796875, + "learning_rate": 1.9815634218289085e-06, + "loss": 2.126053237915039, + "mean_token_accuracy": 0.5377902492880822, + "num_tokens": 26175970.0, + "step": 4450 + }, + { + "entropy": 2.0707394987344743, + "epoch": 0.6250218967873034, + "grad_norm": 3.203125, + "learning_rate": 1.974188790560472e-06, + "loss": 2.0814849853515627, + "mean_token_accuracy": 0.5414546556770802, + "num_tokens": 26239820.0, + "step": 4460 + }, + { + "entropy": 1.9759136408567428, + "epoch": 0.6264232911747188, + "grad_norm": 3.265625, + "learning_rate": 1.9668141592920355e-06, + "loss": 2.024312973022461, + "mean_token_accuracy": 0.5533434115350246, + "num_tokens": 26297034.0, + "step": 4470 + }, + { + "entropy": 1.9657244265079499, + "epoch": 0.6278246855621343, + "grad_norm": 4.09375, + "learning_rate": 1.959439528023599e-06, + "loss": 2.0544254302978517, + "mean_token_accuracy": 0.5568137258291245, + "num_tokens": 26353027.0, + "step": 4480 + }, + { + "entropy": 1.9609319418668747, + "epoch": 0.6292260799495498, + "grad_norm": 3.546875, + "learning_rate": 1.9520648967551625e-06, + "loss": 2.0087684631347655, + "mean_token_accuracy": 0.5550144128501415, + "num_tokens": 26408537.0, + "step": 4490 + }, + { + "entropy": 2.010950264334679, + "epoch": 0.6306274743369653, + "grad_norm": 3.46875, + "learning_rate": 1.944690265486726e-06, + "loss": 2.0559103012084963, + "mean_token_accuracy": 0.5485326498746872, + "num_tokens": 26468140.0, + "step": 4500 + }, + { + "entropy": 2.04426332116127, + "epoch": 0.6320288687243808, + "grad_norm": 3.546875, + "learning_rate": 1.937315634218289e-06, + "loss": 2.0743776321411134, + "mean_token_accuracy": 0.5475040890276432, + "num_tokens": 26527853.0, + "step": 4510 + }, + { + "entropy": 2.017905443906784, + "epoch": 0.6334302631117963, + "grad_norm": 3.84375, + "learning_rate": 1.929941002949853e-06, + "loss": 2.0734704971313476, + "mean_token_accuracy": 0.5491776756942273, + "num_tokens": 26587202.0, + "step": 4520 + }, + { + "entropy": 2.043527716398239, + "epoch": 0.6348316574992117, + "grad_norm": 3.921875, + "learning_rate": 1.922566371681416e-06, + "loss": 2.088183784484863, + "mean_token_accuracy": 0.5455729715526104, + "num_tokens": 26646466.0, + "step": 4530 + }, + { + "entropy": 1.961167186498642, + "epoch": 0.6362330518866272, + "grad_norm": 4.1875, + "learning_rate": 1.9151917404129794e-06, + "loss": 1.994313621520996, + "mean_token_accuracy": 0.5559468373656273, + "num_tokens": 26704075.0, + "step": 4540 + }, + { + "entropy": 2.0513493835926058, + "epoch": 0.6376344462740426, + "grad_norm": 3.828125, + "learning_rate": 1.907817109144543e-06, + "loss": 2.097141075134277, + "mean_token_accuracy": 0.5411770515143871, + "num_tokens": 26762440.0, + "step": 4550 + }, + { + "entropy": 2.047315889596939, + "epoch": 0.6390358406614581, + "grad_norm": 4.125, + "learning_rate": 1.9004424778761064e-06, + "loss": 2.0906604766845702, + "mean_token_accuracy": 0.5442274920642376, + "num_tokens": 26823392.0, + "step": 4560 + }, + { + "entropy": 2.043235424160957, + "epoch": 0.6404372350488736, + "grad_norm": 3.46875, + "learning_rate": 1.8930678466076699e-06, + "loss": 2.1020296096801756, + "mean_token_accuracy": 0.5450766839087009, + "num_tokens": 26882187.0, + "step": 4570 + }, + { + "entropy": 2.037255918979645, + "epoch": 0.6418386294362891, + "grad_norm": 3.359375, + "learning_rate": 1.8856932153392332e-06, + "loss": 2.0846155166625975, + "mean_token_accuracy": 0.5447261914610863, + "num_tokens": 26943183.0, + "step": 4580 + }, + { + "entropy": 1.9987845778465272, + "epoch": 0.6432400238237046, + "grad_norm": 4.5625, + "learning_rate": 1.8783185840707964e-06, + "loss": 2.027919387817383, + "mean_token_accuracy": 0.5525832653045655, + "num_tokens": 27000841.0, + "step": 4590 + }, + { + "entropy": 1.9617159694433213, + "epoch": 0.6446414182111201, + "grad_norm": 4.125, + "learning_rate": 1.8709439528023602e-06, + "loss": 2.006016731262207, + "mean_token_accuracy": 0.5595106884837151, + "num_tokens": 27057746.0, + "step": 4600 + }, + { + "entropy": 2.0060916900634767, + "epoch": 0.6460428125985356, + "grad_norm": 3.65625, + "learning_rate": 1.8635693215339234e-06, + "loss": 2.0546661376953126, + "mean_token_accuracy": 0.551572060585022, + "num_tokens": 27116382.0, + "step": 4610 + }, + { + "entropy": 2.0146182805299757, + "epoch": 0.6474442069859511, + "grad_norm": 3.515625, + "learning_rate": 1.8561946902654867e-06, + "loss": 2.068709945678711, + "mean_token_accuracy": 0.5452509075403214, + "num_tokens": 27175277.0, + "step": 4620 + }, + { + "entropy": 1.999513065814972, + "epoch": 0.6488456013733666, + "grad_norm": 4.15625, + "learning_rate": 1.8488200589970504e-06, + "loss": 2.0343143463134767, + "mean_token_accuracy": 0.5572051353752613, + "num_tokens": 27230042.0, + "step": 4630 + }, + { + "entropy": 2.0813112437725065, + "epoch": 0.6502469957607819, + "grad_norm": 3.84375, + "learning_rate": 1.8414454277286137e-06, + "loss": 2.1107179641723635, + "mean_token_accuracy": 0.5401345692574978, + "num_tokens": 27289023.0, + "step": 4640 + }, + { + "entropy": 1.990957334637642, + "epoch": 0.6516483901481974, + "grad_norm": 3.5, + "learning_rate": 1.8340707964601772e-06, + "loss": 2.055299186706543, + "mean_token_accuracy": 0.5579043760895729, + "num_tokens": 27347136.0, + "step": 4650 + }, + { + "entropy": 2.0554713308811188, + "epoch": 0.6530497845356129, + "grad_norm": 3.828125, + "learning_rate": 1.8266961651917405e-06, + "loss": 2.0880794525146484, + "mean_token_accuracy": 0.539712043851614, + "num_tokens": 27406387.0, + "step": 4660 + }, + { + "entropy": 2.026723089814186, + "epoch": 0.6544511789230284, + "grad_norm": 3.53125, + "learning_rate": 1.819321533923304e-06, + "loss": 2.0620615005493166, + "mean_token_accuracy": 0.5442560493946076, + "num_tokens": 27468338.0, + "step": 4670 + }, + { + "entropy": 2.0433737605810167, + "epoch": 0.6558525733104439, + "grad_norm": 3.546875, + "learning_rate": 1.8119469026548675e-06, + "loss": 2.089468002319336, + "mean_token_accuracy": 0.5462387673556804, + "num_tokens": 27526762.0, + "step": 4680 + }, + { + "entropy": 2.0639784425497054, + "epoch": 0.6572539676978594, + "grad_norm": 3.484375, + "learning_rate": 1.8045722713864308e-06, + "loss": 2.09127140045166, + "mean_token_accuracy": 0.5396267674863339, + "num_tokens": 27586667.0, + "step": 4690 + }, + { + "entropy": 2.0777137130498886, + "epoch": 0.6586553620852749, + "grad_norm": 3.484375, + "learning_rate": 1.797197640117994e-06, + "loss": 2.12203311920166, + "mean_token_accuracy": 0.5387140743434429, + "num_tokens": 27647812.0, + "step": 4700 + }, + { + "entropy": 2.0052779465913773, + "epoch": 0.6600567564726904, + "grad_norm": 3.75, + "learning_rate": 1.7898230088495578e-06, + "loss": 2.0488203048706053, + "mean_token_accuracy": 0.5510134190320969, + "num_tokens": 27704905.0, + "step": 4710 + }, + { + "entropy": 2.037332388758659, + "epoch": 0.6614581508601058, + "grad_norm": 3.46875, + "learning_rate": 1.782448377581121e-06, + "loss": 2.0693235397338867, + "mean_token_accuracy": 0.550404942035675, + "num_tokens": 27765888.0, + "step": 4720 + }, + { + "entropy": 1.9879785537719727, + "epoch": 0.6628595452475213, + "grad_norm": 3.46875, + "learning_rate": 1.7750737463126846e-06, + "loss": 2.019399642944336, + "mean_token_accuracy": 0.5548613980412483, + "num_tokens": 27825414.0, + "step": 4730 + }, + { + "entropy": 2.074928048253059, + "epoch": 0.6642609396349367, + "grad_norm": 3.578125, + "learning_rate": 1.767699115044248e-06, + "loss": 2.095464897155762, + "mean_token_accuracy": 0.5411224909126758, + "num_tokens": 27887165.0, + "step": 4740 + }, + { + "entropy": 1.9969034761190414, + "epoch": 0.6656623340223522, + "grad_norm": 3.796875, + "learning_rate": 1.7603244837758113e-06, + "loss": 2.0468374252319337, + "mean_token_accuracy": 0.5477425627410412, + "num_tokens": 27946311.0, + "step": 4750 + }, + { + "entropy": 2.0368546158075334, + "epoch": 0.6670637284097677, + "grad_norm": 4.09375, + "learning_rate": 1.7529498525073748e-06, + "loss": 2.0636785507202147, + "mean_token_accuracy": 0.5443605974316597, + "num_tokens": 28004202.0, + "step": 4760 + }, + { + "entropy": 2.0267171233892443, + "epoch": 0.6684651227971832, + "grad_norm": 3.609375, + "learning_rate": 1.7455752212389381e-06, + "loss": 2.05112361907959, + "mean_token_accuracy": 0.5438154451549053, + "num_tokens": 28066214.0, + "step": 4770 + }, + { + "entropy": 1.9797705829143524, + "epoch": 0.6698665171845987, + "grad_norm": 3.6875, + "learning_rate": 1.7382005899705016e-06, + "loss": 2.0299072265625, + "mean_token_accuracy": 0.5556055679917336, + "num_tokens": 28121726.0, + "step": 4780 + }, + { + "entropy": 2.011865311861038, + "epoch": 0.6712679115720142, + "grad_norm": 3.765625, + "learning_rate": 1.7308259587020651e-06, + "loss": 2.04732723236084, + "mean_token_accuracy": 0.5537573218345642, + "num_tokens": 28182568.0, + "step": 4790 + }, + { + "entropy": 2.0864431977272035, + "epoch": 0.6726693059594296, + "grad_norm": 4.03125, + "learning_rate": 1.7234513274336284e-06, + "loss": 2.127519416809082, + "mean_token_accuracy": 0.534342635422945, + "num_tokens": 28245237.0, + "step": 4800 + }, + { + "entropy": 1.998366579413414, + "epoch": 0.6740707003468451, + "grad_norm": 3.75, + "learning_rate": 1.716076696165192e-06, + "loss": 2.062630462646484, + "mean_token_accuracy": 0.5544887915253639, + "num_tokens": 28302356.0, + "step": 4810 + }, + { + "entropy": 2.0306527107954024, + "epoch": 0.6754720947342606, + "grad_norm": 3.828125, + "learning_rate": 1.7087020648967554e-06, + "loss": 2.0890117645263673, + "mean_token_accuracy": 0.5466470494866371, + "num_tokens": 28361601.0, + "step": 4820 + }, + { + "entropy": 2.0527277678251266, + "epoch": 0.6768734891216761, + "grad_norm": 3.4375, + "learning_rate": 1.7013274336283187e-06, + "loss": 2.0924203872680662, + "mean_token_accuracy": 0.5420490309596062, + "num_tokens": 28423623.0, + "step": 4830 + }, + { + "entropy": 2.056154838204384, + "epoch": 0.6782748835090916, + "grad_norm": 3.953125, + "learning_rate": 1.6939528023598822e-06, + "loss": 2.0951744079589845, + "mean_token_accuracy": 0.5420894406735897, + "num_tokens": 28485339.0, + "step": 4840 + }, + { + "entropy": 2.043560880422592, + "epoch": 0.679676277896507, + "grad_norm": 3.71875, + "learning_rate": 1.6865781710914457e-06, + "loss": 2.0973472595214844, + "mean_token_accuracy": 0.5462093599140644, + "num_tokens": 28544597.0, + "step": 4850 + }, + { + "entropy": 1.9837368726730347, + "epoch": 0.6810776722839225, + "grad_norm": 3.796875, + "learning_rate": 1.679203539823009e-06, + "loss": 2.024229049682617, + "mean_token_accuracy": 0.5542949989438057, + "num_tokens": 28600577.0, + "step": 4860 + }, + { + "entropy": 2.0445822417736053, + "epoch": 0.682479066671338, + "grad_norm": 3.28125, + "learning_rate": 1.6718289085545724e-06, + "loss": 2.080499267578125, + "mean_token_accuracy": 0.5457544267177582, + "num_tokens": 28658978.0, + "step": 4870 + }, + { + "entropy": 2.02373021543026, + "epoch": 0.6838804610587534, + "grad_norm": 3.765625, + "learning_rate": 1.6644542772861357e-06, + "loss": 2.0790000915527345, + "mean_token_accuracy": 0.549615528434515, + "num_tokens": 28718368.0, + "step": 4880 + }, + { + "entropy": 2.012890937924385, + "epoch": 0.6852818554461689, + "grad_norm": 3.9375, + "learning_rate": 1.6570796460176994e-06, + "loss": 2.0690385818481447, + "mean_token_accuracy": 0.5495897889137268, + "num_tokens": 28775826.0, + "step": 4890 + }, + { + "entropy": 1.9953327834606172, + "epoch": 0.6866832498335844, + "grad_norm": 4.0, + "learning_rate": 1.6497050147492627e-06, + "loss": 2.0699180603027343, + "mean_token_accuracy": 0.5566862612962723, + "num_tokens": 28832227.0, + "step": 4900 + }, + { + "entropy": 2.017250362038612, + "epoch": 0.6880846442209999, + "grad_norm": 3.421875, + "learning_rate": 1.642330383480826e-06, + "loss": 2.0600749969482424, + "mean_token_accuracy": 0.5505439385771751, + "num_tokens": 28893658.0, + "step": 4910 + }, + { + "entropy": 2.0216071248054504, + "epoch": 0.6894860386084154, + "grad_norm": 3.859375, + "learning_rate": 1.6349557522123895e-06, + "loss": 2.0286935806274413, + "mean_token_accuracy": 0.5535654246807098, + "num_tokens": 28951703.0, + "step": 4920 + }, + { + "entropy": 2.0413787811994553, + "epoch": 0.6908874329958309, + "grad_norm": 3.734375, + "learning_rate": 1.627581120943953e-06, + "loss": 2.0949050903320314, + "mean_token_accuracy": 0.5380190096795558, + "num_tokens": 29011546.0, + "step": 4930 + }, + { + "entropy": 2.0354854226112367, + "epoch": 0.6922888273832464, + "grad_norm": 4.1875, + "learning_rate": 1.6202064896755163e-06, + "loss": 2.0558343887329102, + "mean_token_accuracy": 0.5461363635957242, + "num_tokens": 29072027.0, + "step": 4940 + }, + { + "entropy": 1.9892939269542693, + "epoch": 0.6936902217706619, + "grad_norm": 3.671875, + "learning_rate": 1.6128318584070798e-06, + "loss": 2.0504005432128904, + "mean_token_accuracy": 0.552552143484354, + "num_tokens": 29127681.0, + "step": 4950 + }, + { + "entropy": 1.9891421705484391, + "epoch": 0.6950916161580772, + "grad_norm": 3.3125, + "learning_rate": 1.605457227138643e-06, + "loss": 2.0430601119995115, + "mean_token_accuracy": 0.5505351021885871, + "num_tokens": 29187210.0, + "step": 4960 + }, + { + "entropy": 2.0355088084936144, + "epoch": 0.6964930105454927, + "grad_norm": 3.421875, + "learning_rate": 1.5980825958702068e-06, + "loss": 2.065724563598633, + "mean_token_accuracy": 0.5397682934999466, + "num_tokens": 29248403.0, + "step": 4970 + }, + { + "entropy": 2.039190375804901, + "epoch": 0.6978944049329082, + "grad_norm": 3.671875, + "learning_rate": 1.59070796460177e-06, + "loss": 2.0834108352661134, + "mean_token_accuracy": 0.5436397984623909, + "num_tokens": 29308108.0, + "step": 4980 + }, + { + "entropy": 1.95891977250576, + "epoch": 0.6992957993203237, + "grad_norm": 3.515625, + "learning_rate": 1.5833333333333333e-06, + "loss": 2.0039615631103516, + "mean_token_accuracy": 0.5627473741769791, + "num_tokens": 29363890.0, + "step": 4990 + }, + { + "entropy": 2.00730918943882, + "epoch": 0.7006971937077392, + "grad_norm": 3.453125, + "learning_rate": 1.575958702064897e-06, + "loss": 2.0677875518798827, + "mean_token_accuracy": 0.550707983225584, + "num_tokens": 29421859.0, + "step": 5000 + }, + { + "entropy": 2.0345747053623198, + "epoch": 0.7020985880951547, + "grad_norm": 3.734375, + "learning_rate": 1.5685840707964603e-06, + "loss": 2.070606231689453, + "mean_token_accuracy": 0.5438873060047626, + "num_tokens": 29485269.0, + "step": 5010 + }, + { + "entropy": 1.9661437571048737, + "epoch": 0.7034999824825702, + "grad_norm": 3.890625, + "learning_rate": 1.5612094395280236e-06, + "loss": 2.0235414505004883, + "mean_token_accuracy": 0.5593526065349579, + "num_tokens": 29540316.0, + "step": 5020 + }, + { + "entropy": 1.9796199768781662, + "epoch": 0.7049013768699857, + "grad_norm": 3.359375, + "learning_rate": 1.5538348082595871e-06, + "loss": 2.022873115539551, + "mean_token_accuracy": 0.5520857453346253, + "num_tokens": 29598590.0, + "step": 5030 + }, + { + "entropy": 1.9959565669298172, + "epoch": 0.7063027712574012, + "grad_norm": 3.34375, + "learning_rate": 1.5464601769911506e-06, + "loss": 2.041946220397949, + "mean_token_accuracy": 0.5531593382358551, + "num_tokens": 29657145.0, + "step": 5040 + }, + { + "entropy": 1.9920789122581481, + "epoch": 0.7077041656448166, + "grad_norm": 4.09375, + "learning_rate": 1.5390855457227141e-06, + "loss": 2.0239227294921873, + "mean_token_accuracy": 0.5522009208798409, + "num_tokens": 29715642.0, + "step": 5050 + }, + { + "entropy": 2.01208633184433, + "epoch": 0.709105560032232, + "grad_norm": 3.34375, + "learning_rate": 1.5317109144542774e-06, + "loss": 2.041418266296387, + "mean_token_accuracy": 0.5467449180781842, + "num_tokens": 29778010.0, + "step": 5060 + }, + { + "entropy": 2.0474745452404024, + "epoch": 0.7105069544196475, + "grad_norm": 4.15625, + "learning_rate": 1.5243362831858407e-06, + "loss": 2.08483829498291, + "mean_token_accuracy": 0.5433097027242184, + "num_tokens": 29840933.0, + "step": 5070 + }, + { + "entropy": 1.9889600902795792, + "epoch": 0.711908348807063, + "grad_norm": 3.6875, + "learning_rate": 1.5169616519174044e-06, + "loss": 2.027958869934082, + "mean_token_accuracy": 0.5541149020195008, + "num_tokens": 29897460.0, + "step": 5080 + }, + { + "entropy": 1.9934295445680619, + "epoch": 0.7133097431944785, + "grad_norm": 3.515625, + "learning_rate": 1.5095870206489677e-06, + "loss": 2.053207588195801, + "mean_token_accuracy": 0.5522679515182972, + "num_tokens": 29955103.0, + "step": 5090 + }, + { + "entropy": 2.008519572019577, + "epoch": 0.714711137581894, + "grad_norm": 3.515625, + "learning_rate": 1.502212389380531e-06, + "loss": 2.075584602355957, + "mean_token_accuracy": 0.548991971462965, + "num_tokens": 30016790.0, + "step": 5100 + }, + { + "entropy": 2.039276033639908, + "epoch": 0.7161125319693095, + "grad_norm": 3.5625, + "learning_rate": 1.4948377581120947e-06, + "loss": 2.0914531707763673, + "mean_token_accuracy": 0.5397449642419815, + "num_tokens": 30077591.0, + "step": 5110 + }, + { + "entropy": 2.025232970714569, + "epoch": 0.717513926356725, + "grad_norm": 3.734375, + "learning_rate": 1.487463126843658e-06, + "loss": 2.062969970703125, + "mean_token_accuracy": 0.5480619043111801, + "num_tokens": 30137969.0, + "step": 5120 + }, + { + "entropy": 2.009069937467575, + "epoch": 0.7189153207441404, + "grad_norm": 3.796875, + "learning_rate": 1.4800884955752215e-06, + "loss": 2.0655084609985352, + "mean_token_accuracy": 0.5510629720985889, + "num_tokens": 30196112.0, + "step": 5130 + }, + { + "entropy": 2.0180150896310804, + "epoch": 0.7203167151315559, + "grad_norm": 3.953125, + "learning_rate": 1.4727138643067847e-06, + "loss": 2.0467395782470703, + "mean_token_accuracy": 0.5497609481215477, + "num_tokens": 30258499.0, + "step": 5140 + }, + { + "entropy": 2.03435495197773, + "epoch": 0.7217181095189714, + "grad_norm": 3.28125, + "learning_rate": 1.465339233038348e-06, + "loss": 2.0752939224243163, + "mean_token_accuracy": 0.5446124017238617, + "num_tokens": 30319512.0, + "step": 5150 + }, + { + "entropy": 1.966948103904724, + "epoch": 0.7231195039063869, + "grad_norm": 3.96875, + "learning_rate": 1.4579646017699117e-06, + "loss": 2.028528594970703, + "mean_token_accuracy": 0.5552498593926429, + "num_tokens": 30377947.0, + "step": 5160 + }, + { + "entropy": 2.0390039294958116, + "epoch": 0.7245208982938023, + "grad_norm": 4.125, + "learning_rate": 1.450589970501475e-06, + "loss": 2.072515678405762, + "mean_token_accuracy": 0.5410834163427353, + "num_tokens": 30438293.0, + "step": 5170 + }, + { + "entropy": 2.02644604742527, + "epoch": 0.7259222926812178, + "grad_norm": 5.375, + "learning_rate": 1.4432153392330383e-06, + "loss": 2.0944353103637696, + "mean_token_accuracy": 0.5451689854264259, + "num_tokens": 30498524.0, + "step": 5180 + }, + { + "entropy": 2.062913554906845, + "epoch": 0.7273236870686333, + "grad_norm": 3.46875, + "learning_rate": 1.435840707964602e-06, + "loss": 2.120368003845215, + "mean_token_accuracy": 0.5419623903930187, + "num_tokens": 30559568.0, + "step": 5190 + }, + { + "entropy": 1.9839348793029785, + "epoch": 0.7287250814560488, + "grad_norm": 4.8125, + "learning_rate": 1.4284660766961653e-06, + "loss": 2.010845756530762, + "mean_token_accuracy": 0.5561483658850193, + "num_tokens": 30616257.0, + "step": 5200 + }, + { + "entropy": 1.9909240514039994, + "epoch": 0.7301264758434642, + "grad_norm": 3.609375, + "learning_rate": 1.4210914454277288e-06, + "loss": 2.038786506652832, + "mean_token_accuracy": 0.5507546961307526, + "num_tokens": 30676130.0, + "step": 5210 + }, + { + "entropy": 2.0434360295534133, + "epoch": 0.7315278702308797, + "grad_norm": 3.5, + "learning_rate": 1.413716814159292e-06, + "loss": 2.087807464599609, + "mean_token_accuracy": 0.543889294564724, + "num_tokens": 30733476.0, + "step": 5220 + }, + { + "entropy": 2.0257026076316835, + "epoch": 0.7329292646182952, + "grad_norm": 3.875, + "learning_rate": 1.4063421828908556e-06, + "loss": 2.0706718444824217, + "mean_token_accuracy": 0.5436087146401405, + "num_tokens": 30794786.0, + "step": 5230 + }, + { + "entropy": 1.9865258991718293, + "epoch": 0.7343306590057107, + "grad_norm": 3.453125, + "learning_rate": 1.398967551622419e-06, + "loss": 2.031447982788086, + "mean_token_accuracy": 0.5527370892465114, + "num_tokens": 30851784.0, + "step": 5240 + }, + { + "entropy": 2.0543854385614395, + "epoch": 0.7357320533931262, + "grad_norm": 3.484375, + "learning_rate": 1.3915929203539824e-06, + "loss": 2.098319244384766, + "mean_token_accuracy": 0.5480484694242478, + "num_tokens": 30907244.0, + "step": 5250 + }, + { + "entropy": 2.046630707383156, + "epoch": 0.7371334477805417, + "grad_norm": 3.6875, + "learning_rate": 1.3842182890855456e-06, + "loss": 2.0829851150512697, + "mean_token_accuracy": 0.5406535662710666, + "num_tokens": 30968532.0, + "step": 5260 + }, + { + "entropy": 2.090639790892601, + "epoch": 0.7385348421679571, + "grad_norm": 3.765625, + "learning_rate": 1.3768436578171094e-06, + "loss": 2.108531951904297, + "mean_token_accuracy": 0.5410657115280628, + "num_tokens": 31032380.0, + "step": 5270 + }, + { + "entropy": 2.0192977368831633, + "epoch": 0.7399362365553726, + "grad_norm": 3.984375, + "learning_rate": 1.3694690265486726e-06, + "loss": 2.06378288269043, + "mean_token_accuracy": 0.5463333018124104, + "num_tokens": 31089758.0, + "step": 5280 + }, + { + "entropy": 2.0674374043941497, + "epoch": 0.741337630942788, + "grad_norm": 3.53125, + "learning_rate": 1.3620943952802361e-06, + "loss": 2.1007968902587892, + "mean_token_accuracy": 0.5407262332737446, + "num_tokens": 31151558.0, + "step": 5290 + }, + { + "entropy": 2.038059750199318, + "epoch": 0.7427390253302035, + "grad_norm": 4.15625, + "learning_rate": 1.3547197640117996e-06, + "loss": 2.0754085540771485, + "mean_token_accuracy": 0.5470300734043121, + "num_tokens": 31210314.0, + "step": 5300 + }, + { + "entropy": 2.046363744139671, + "epoch": 0.744140419717619, + "grad_norm": 4.125, + "learning_rate": 1.347345132743363e-06, + "loss": 2.0913890838623046, + "mean_token_accuracy": 0.5413794197142124, + "num_tokens": 31268829.0, + "step": 5310 + }, + { + "entropy": 1.9908478796482085, + "epoch": 0.7455418141050345, + "grad_norm": 4.1875, + "learning_rate": 1.3399705014749264e-06, + "loss": 2.0599996566772463, + "mean_token_accuracy": 0.5544260628521442, + "num_tokens": 31327236.0, + "step": 5320 + }, + { + "entropy": 2.0618211805820463, + "epoch": 0.74694320849245, + "grad_norm": 4.125, + "learning_rate": 1.3325958702064897e-06, + "loss": 2.089054489135742, + "mean_token_accuracy": 0.5435502268373966, + "num_tokens": 31386149.0, + "step": 5330 + }, + { + "entropy": 2.03034026324749, + "epoch": 0.7483446028798655, + "grad_norm": 3.1875, + "learning_rate": 1.3252212389380532e-06, + "loss": 2.061849594116211, + "mean_token_accuracy": 0.5444556280970574, + "num_tokens": 31445055.0, + "step": 5340 + }, + { + "entropy": 1.954394268989563, + "epoch": 0.749745997267281, + "grad_norm": 3.8125, + "learning_rate": 1.3178466076696167e-06, + "loss": 2.0087169647216796, + "mean_token_accuracy": 0.5599333278834819, + "num_tokens": 31501584.0, + "step": 5350 + }, + { + "entropy": 2.0153422504663467, + "epoch": 0.7511473916546965, + "grad_norm": 3.921875, + "learning_rate": 1.31047197640118e-06, + "loss": 2.0687637329101562, + "mean_token_accuracy": 0.5506028674542904, + "num_tokens": 31558998.0, + "step": 5360 + }, + { + "entropy": 2.0368154883384704, + "epoch": 0.752548786042112, + "grad_norm": 3.640625, + "learning_rate": 1.3030973451327437e-06, + "loss": 2.0791160583496096, + "mean_token_accuracy": 0.5412822388112545, + "num_tokens": 31620793.0, + "step": 5370 + }, + { + "entropy": 2.0666065216064453, + "epoch": 0.7539501804295273, + "grad_norm": 3.71875, + "learning_rate": 1.295722713864307e-06, + "loss": 2.1090770721435548, + "mean_token_accuracy": 0.5386351138353348, + "num_tokens": 31680897.0, + "step": 5380 + }, + { + "entropy": 1.9791485756635665, + "epoch": 0.7553515748169428, + "grad_norm": 3.5625, + "learning_rate": 1.2883480825958703e-06, + "loss": 2.023344612121582, + "mean_token_accuracy": 0.5536239571869374, + "num_tokens": 31738850.0, + "step": 5390 + }, + { + "entropy": 1.9926690608263016, + "epoch": 0.7567529692043583, + "grad_norm": 4.125, + "learning_rate": 1.2809734513274338e-06, + "loss": 2.0447708129882813, + "mean_token_accuracy": 0.5517847746610641, + "num_tokens": 31797186.0, + "step": 5400 + }, + { + "entropy": 1.9964224964380264, + "epoch": 0.7581543635917738, + "grad_norm": 4.84375, + "learning_rate": 1.2735988200589973e-06, + "loss": 2.045303726196289, + "mean_token_accuracy": 0.5528283767402172, + "num_tokens": 31855775.0, + "step": 5410 + }, + { + "entropy": 2.0050380915403365, + "epoch": 0.7595557579791893, + "grad_norm": 3.0625, + "learning_rate": 1.2662241887905605e-06, + "loss": 2.0495418548583983, + "mean_token_accuracy": 0.5511241793632508, + "num_tokens": 31917779.0, + "step": 5420 + }, + { + "entropy": 2.032631465792656, + "epoch": 0.7609571523666048, + "grad_norm": 3.6875, + "learning_rate": 1.258849557522124e-06, + "loss": 2.086847496032715, + "mean_token_accuracy": 0.5456423066556454, + "num_tokens": 31975252.0, + "step": 5430 + }, + { + "entropy": 2.039604443311691, + "epoch": 0.7623585467540203, + "grad_norm": 3.34375, + "learning_rate": 1.2514749262536873e-06, + "loss": 2.066813278198242, + "mean_token_accuracy": 0.541644936800003, + "num_tokens": 32039447.0, + "step": 5440 + }, + { + "entropy": 1.9697502493858337, + "epoch": 0.7637599411414357, + "grad_norm": 3.75, + "learning_rate": 1.2441002949852508e-06, + "loss": 2.009023094177246, + "mean_token_accuracy": 0.5575101591646672, + "num_tokens": 32094443.0, + "step": 5450 + }, + { + "entropy": 2.061708700656891, + "epoch": 0.7651613355288512, + "grad_norm": 3.3125, + "learning_rate": 1.2367256637168143e-06, + "loss": 2.124795341491699, + "mean_token_accuracy": 0.5439794048666954, + "num_tokens": 32153470.0, + "step": 5460 + }, + { + "entropy": 1.9518501698970794, + "epoch": 0.7665627299162667, + "grad_norm": 3.375, + "learning_rate": 1.2293510324483776e-06, + "loss": 2.02158145904541, + "mean_token_accuracy": 0.5608993351459504, + "num_tokens": 32207417.0, + "step": 5470 + }, + { + "entropy": 1.9832956045866013, + "epoch": 0.7679641243036822, + "grad_norm": 3.875, + "learning_rate": 1.221976401179941e-06, + "loss": 2.0236433029174803, + "mean_token_accuracy": 0.5541934780776501, + "num_tokens": 32265269.0, + "step": 5480 + }, + { + "entropy": 2.064125454425812, + "epoch": 0.7693655186910976, + "grad_norm": 3.53125, + "learning_rate": 1.2146017699115046e-06, + "loss": 2.1089569091796876, + "mean_token_accuracy": 0.5377648368477821, + "num_tokens": 32324979.0, + "step": 5490 + }, + { + "entropy": 1.9474216997623444, + "epoch": 0.7707669130785131, + "grad_norm": 3.171875, + "learning_rate": 1.2072271386430679e-06, + "loss": 2.0038455963134765, + "mean_token_accuracy": 0.5647109769284725, + "num_tokens": 32380726.0, + "step": 5500 + }, + { + "entropy": 2.0227566003799438, + "epoch": 0.7721683074659286, + "grad_norm": 4.625, + "learning_rate": 1.1998525073746314e-06, + "loss": 2.062344551086426, + "mean_token_accuracy": 0.5454733707010746, + "num_tokens": 32438897.0, + "step": 5510 + }, + { + "entropy": 2.060624200105667, + "epoch": 0.7735697018533441, + "grad_norm": 3.640625, + "learning_rate": 1.1924778761061947e-06, + "loss": 2.114695930480957, + "mean_token_accuracy": 0.5415135055780411, + "num_tokens": 32501137.0, + "step": 5520 + }, + { + "entropy": 1.963140258193016, + "epoch": 0.7749710962407595, + "grad_norm": 3.8125, + "learning_rate": 1.1851032448377582e-06, + "loss": 2.0178470611572266, + "mean_token_accuracy": 0.5575778961181641, + "num_tokens": 32557905.0, + "step": 5530 + }, + { + "entropy": 1.9923629015684128, + "epoch": 0.776372490628175, + "grad_norm": 3.546875, + "learning_rate": 1.1777286135693217e-06, + "loss": 2.038427734375, + "mean_token_accuracy": 0.5486880376935005, + "num_tokens": 32617221.0, + "step": 5540 + }, + { + "entropy": 2.0610381811857224, + "epoch": 0.7777738850155905, + "grad_norm": 3.78125, + "learning_rate": 1.170353982300885e-06, + "loss": 2.089842987060547, + "mean_token_accuracy": 0.5413602493703366, + "num_tokens": 32677171.0, + "step": 5550 + }, + { + "entropy": 2.0449792385101317, + "epoch": 0.779175279403006, + "grad_norm": 3.84375, + "learning_rate": 1.1629793510324484e-06, + "loss": 2.083336067199707, + "mean_token_accuracy": 0.544983584433794, + "num_tokens": 32734176.0, + "step": 5560 + }, + { + "entropy": 2.0048939406871797, + "epoch": 0.7805766737904215, + "grad_norm": 3.828125, + "learning_rate": 1.155604719764012e-06, + "loss": 2.0253765106201174, + "mean_token_accuracy": 0.5513779424130917, + "num_tokens": 32792833.0, + "step": 5570 + }, + { + "entropy": 1.9710610508918762, + "epoch": 0.781978068177837, + "grad_norm": 4.125, + "learning_rate": 1.1482300884955754e-06, + "loss": 2.0077726364135744, + "mean_token_accuracy": 0.555688152462244, + "num_tokens": 32848426.0, + "step": 5580 + }, + { + "entropy": 2.030847093462944, + "epoch": 0.7833794625652524, + "grad_norm": 4.0625, + "learning_rate": 1.1408554572271387e-06, + "loss": 2.0594655990600588, + "mean_token_accuracy": 0.55287881270051, + "num_tokens": 32908103.0, + "step": 5590 + }, + { + "entropy": 1.9961274981498718, + "epoch": 0.7847808569526679, + "grad_norm": 3.84375, + "learning_rate": 1.1334808259587022e-06, + "loss": 2.0341890335083006, + "mean_token_accuracy": 0.5506459511816502, + "num_tokens": 32969324.0, + "step": 5600 + }, + { + "entropy": 2.034320372343063, + "epoch": 0.7861822513400833, + "grad_norm": 3.1875, + "learning_rate": 1.1261061946902655e-06, + "loss": 2.0603561401367188, + "mean_token_accuracy": 0.5476031564176083, + "num_tokens": 33028844.0, + "step": 5610 + }, + { + "entropy": 2.0152987748384477, + "epoch": 0.7875836457274988, + "grad_norm": 3.421875, + "learning_rate": 1.118731563421829e-06, + "loss": 2.0534236907958983, + "mean_token_accuracy": 0.5472868226468564, + "num_tokens": 33088470.0, + "step": 5620 + }, + { + "entropy": 2.0418001085519792, + "epoch": 0.7889850401149143, + "grad_norm": 3.1875, + "learning_rate": 1.1113569321533923e-06, + "loss": 2.077309417724609, + "mean_token_accuracy": 0.5479334965348244, + "num_tokens": 33147480.0, + "step": 5630 + }, + { + "entropy": 2.0254188984632493, + "epoch": 0.7903864345023298, + "grad_norm": 3.9375, + "learning_rate": 1.1039823008849558e-06, + "loss": 2.0901409149169923, + "mean_token_accuracy": 0.5460968457162381, + "num_tokens": 33207254.0, + "step": 5640 + }, + { + "entropy": 2.078515759110451, + "epoch": 0.7917878288897453, + "grad_norm": 3.859375, + "learning_rate": 1.0966076696165193e-06, + "loss": 2.1065073013305664, + "mean_token_accuracy": 0.538357075303793, + "num_tokens": 33269654.0, + "step": 5650 + }, + { + "entropy": 2.030597913265228, + "epoch": 0.7931892232771608, + "grad_norm": 3.5625, + "learning_rate": 1.0892330383480828e-06, + "loss": 2.069440460205078, + "mean_token_accuracy": 0.5432138055562973, + "num_tokens": 33330503.0, + "step": 5660 + }, + { + "entropy": 2.0529883772134783, + "epoch": 0.7945906176645763, + "grad_norm": 3.671875, + "learning_rate": 1.081858407079646e-06, + "loss": 2.0848726272583007, + "mean_token_accuracy": 0.53982138261199, + "num_tokens": 33390091.0, + "step": 5670 + }, + { + "entropy": 1.9861307680606841, + "epoch": 0.7959920120519918, + "grad_norm": 3.609375, + "learning_rate": 1.0744837758112095e-06, + "loss": 2.0331771850585936, + "mean_token_accuracy": 0.5522069059312343, + "num_tokens": 33446385.0, + "step": 5680 + }, + { + "entropy": 2.0443527430295942, + "epoch": 0.7973934064394073, + "grad_norm": 3.375, + "learning_rate": 1.067109144542773e-06, + "loss": 2.085647773742676, + "mean_token_accuracy": 0.5428236566483975, + "num_tokens": 33509176.0, + "step": 5690 + }, + { + "entropy": 1.9681357473134995, + "epoch": 0.7987948008268226, + "grad_norm": 3.78125, + "learning_rate": 1.0597345132743363e-06, + "loss": 2.0029911041259765, + "mean_token_accuracy": 0.5614252880215644, + "num_tokens": 33564141.0, + "step": 5700 + }, + { + "entropy": 2.0032998740673067, + "epoch": 0.8001961952142381, + "grad_norm": 3.984375, + "learning_rate": 1.0523598820058998e-06, + "loss": 2.0492082595825196, + "mean_token_accuracy": 0.5448523163795471, + "num_tokens": 33622834.0, + "step": 5710 + }, + { + "entropy": 2.037500596046448, + "epoch": 0.8015975896016536, + "grad_norm": 4.15625, + "learning_rate": 1.0449852507374631e-06, + "loss": 2.0980915069580077, + "mean_token_accuracy": 0.5466709710657597, + "num_tokens": 33682071.0, + "step": 5720 + }, + { + "entropy": 2.0753049701452255, + "epoch": 0.8029989839890691, + "grad_norm": 3.328125, + "learning_rate": 1.0376106194690266e-06, + "loss": 2.0992870330810547, + "mean_token_accuracy": 0.5368607066571712, + "num_tokens": 33745428.0, + "step": 5730 + }, + { + "entropy": 2.0023007065057756, + "epoch": 0.8044003783764846, + "grad_norm": 3.84375, + "learning_rate": 1.03023598820059e-06, + "loss": 2.057754707336426, + "mean_token_accuracy": 0.5510963708162308, + "num_tokens": 33802458.0, + "step": 5740 + }, + { + "entropy": 1.9657213807106018, + "epoch": 0.8058017727639001, + "grad_norm": 4.0, + "learning_rate": 1.0228613569321534e-06, + "loss": 2.002410125732422, + "mean_token_accuracy": 0.5550228297710419, + "num_tokens": 33858048.0, + "step": 5750 + }, + { + "entropy": 2.0697003692388534, + "epoch": 0.8072031671513156, + "grad_norm": 3.6875, + "learning_rate": 1.0154867256637169e-06, + "loss": 2.1153873443603515, + "mean_token_accuracy": 0.539206364005804, + "num_tokens": 33919315.0, + "step": 5760 + }, + { + "entropy": 1.966435581445694, + "epoch": 0.8086045615387311, + "grad_norm": 3.28125, + "learning_rate": 1.0081120943952804e-06, + "loss": 1.9911382675170899, + "mean_token_accuracy": 0.5573266044259071, + "num_tokens": 33977087.0, + "step": 5770 + }, + { + "entropy": 2.0584985971450807, + "epoch": 0.8100059559261465, + "grad_norm": 3.546875, + "learning_rate": 1.0007374631268439e-06, + "loss": 2.1007591247558595, + "mean_token_accuracy": 0.5422012776136398, + "num_tokens": 34037587.0, + "step": 5780 + }, + { + "entropy": 1.9936932235956193, + "epoch": 0.811407350313562, + "grad_norm": 3.796875, + "learning_rate": 9.933628318584072e-07, + "loss": 2.022954750061035, + "mean_token_accuracy": 0.5503713063895702, + "num_tokens": 34094984.0, + "step": 5790 + }, + { + "entropy": 2.005845013260841, + "epoch": 0.8128087447009775, + "grad_norm": 3.1875, + "learning_rate": 9.859882005899705e-07, + "loss": 2.0530033111572266, + "mean_token_accuracy": 0.5474283389747143, + "num_tokens": 34153566.0, + "step": 5800 + }, + { + "entropy": 2.0169087648391724, + "epoch": 0.8142101390883929, + "grad_norm": 3.375, + "learning_rate": 9.78613569321534e-07, + "loss": 2.05322380065918, + "mean_token_accuracy": 0.5500776514410972, + "num_tokens": 34211590.0, + "step": 5810 + }, + { + "entropy": 1.9852912485599519, + "epoch": 0.8156115334758084, + "grad_norm": 3.59375, + "learning_rate": 9.712389380530974e-07, + "loss": 2.043269729614258, + "mean_token_accuracy": 0.5516661286354065, + "num_tokens": 34267748.0, + "step": 5820 + }, + { + "entropy": 2.050777268409729, + "epoch": 0.8170129278632239, + "grad_norm": 3.4375, + "learning_rate": 9.638643067846607e-07, + "loss": 2.0954652786254884, + "mean_token_accuracy": 0.5400247015058994, + "num_tokens": 34328233.0, + "step": 5830 + }, + { + "entropy": 2.018732896447182, + "epoch": 0.8184143222506394, + "grad_norm": 3.671875, + "learning_rate": 9.564896755162242e-07, + "loss": 2.044917678833008, + "mean_token_accuracy": 0.546955619752407, + "num_tokens": 34388325.0, + "step": 5840 + }, + { + "entropy": 2.0328523486852648, + "epoch": 0.8198157166380549, + "grad_norm": 3.734375, + "learning_rate": 9.491150442477877e-07, + "loss": 2.0901863098144533, + "mean_token_accuracy": 0.5440246790647507, + "num_tokens": 34449564.0, + "step": 5850 + }, + { + "entropy": 1.998683288693428, + "epoch": 0.8212171110254703, + "grad_norm": 3.65625, + "learning_rate": 9.417404129793511e-07, + "loss": 2.029113006591797, + "mean_token_accuracy": 0.5511540561914444, + "num_tokens": 34507918.0, + "step": 5860 + }, + { + "entropy": 1.9962221890687943, + "epoch": 0.8226185054128858, + "grad_norm": 3.703125, + "learning_rate": 9.343657817109145e-07, + "loss": 2.0319952011108398, + "mean_token_accuracy": 0.5507189773023129, + "num_tokens": 34564368.0, + "step": 5870 + }, + { + "entropy": 2.0780009627342224, + "epoch": 0.8240198998003013, + "grad_norm": 3.296875, + "learning_rate": 9.269911504424779e-07, + "loss": 2.1300451278686525, + "mean_token_accuracy": 0.5382222034037113, + "num_tokens": 34626239.0, + "step": 5880 + }, + { + "entropy": 2.0017063498497008, + "epoch": 0.8254212941877168, + "grad_norm": 3.5625, + "learning_rate": 9.196165191740414e-07, + "loss": 2.05452823638916, + "mean_token_accuracy": 0.5506385952234268, + "num_tokens": 34682351.0, + "step": 5890 + }, + { + "entropy": 2.0254274785518644, + "epoch": 0.8268226885751323, + "grad_norm": 3.46875, + "learning_rate": 9.122418879056048e-07, + "loss": 2.0523189544677733, + "mean_token_accuracy": 0.55109326466918, + "num_tokens": 34740163.0, + "step": 5900 + }, + { + "entropy": 1.9847439497709274, + "epoch": 0.8282240829625477, + "grad_norm": 3.46875, + "learning_rate": 9.048672566371682e-07, + "loss": 2.033793830871582, + "mean_token_accuracy": 0.5541432574391365, + "num_tokens": 34797066.0, + "step": 5910 + }, + { + "entropy": 1.9410735696554184, + "epoch": 0.8296254773499632, + "grad_norm": 3.53125, + "learning_rate": 8.974926253687316e-07, + "loss": 2.0152511596679688, + "mean_token_accuracy": 0.5639293536543846, + "num_tokens": 34851537.0, + "step": 5920 + }, + { + "entropy": 1.9899339884519578, + "epoch": 0.8310268717373787, + "grad_norm": 3.46875, + "learning_rate": 8.901179941002951e-07, + "loss": 2.0402029037475584, + "mean_token_accuracy": 0.5545332990586758, + "num_tokens": 34906669.0, + "step": 5930 + }, + { + "entropy": 2.0425183057785032, + "epoch": 0.8324282661247941, + "grad_norm": 3.515625, + "learning_rate": 8.827433628318586e-07, + "loss": 2.0895105361938477, + "mean_token_accuracy": 0.5444886229932309, + "num_tokens": 34968897.0, + "step": 5940 + }, + { + "entropy": 2.063935214281082, + "epoch": 0.8338296605122096, + "grad_norm": 3.65625, + "learning_rate": 8.753687315634218e-07, + "loss": 2.101520538330078, + "mean_token_accuracy": 0.5415826089680195, + "num_tokens": 35028785.0, + "step": 5950 + }, + { + "entropy": 2.054108539223671, + "epoch": 0.8352310548996251, + "grad_norm": 3.484375, + "learning_rate": 8.679941002949853e-07, + "loss": 2.1111446380615235, + "mean_token_accuracy": 0.5441468000411988, + "num_tokens": 35087811.0, + "step": 5960 + }, + { + "entropy": 2.0053718417882918, + "epoch": 0.8366324492870406, + "grad_norm": 4.75, + "learning_rate": 8.606194690265487e-07, + "loss": 2.033930778503418, + "mean_token_accuracy": 0.5514634430408478, + "num_tokens": 35146653.0, + "step": 5970 + }, + { + "entropy": 2.0017360031604765, + "epoch": 0.8380338436744561, + "grad_norm": 4.03125, + "learning_rate": 8.532448377581122e-07, + "loss": 2.0413970947265625, + "mean_token_accuracy": 0.54992650821805, + "num_tokens": 35204683.0, + "step": 5980 + }, + { + "entropy": 2.040289434790611, + "epoch": 0.8394352380618716, + "grad_norm": 4.0, + "learning_rate": 8.458702064896755e-07, + "loss": 2.082858085632324, + "mean_token_accuracy": 0.5428684964776039, + "num_tokens": 35264118.0, + "step": 5990 + }, + { + "entropy": 2.057906711101532, + "epoch": 0.8408366324492871, + "grad_norm": 3.578125, + "learning_rate": 8.38495575221239e-07, + "loss": 2.1240556716918944, + "mean_token_accuracy": 0.5388853140175343, + "num_tokens": 35325192.0, + "step": 6000 + }, + { + "entropy": 1.9824541568756104, + "epoch": 0.8422380268367026, + "grad_norm": 4.3125, + "learning_rate": 8.311209439528024e-07, + "loss": 2.0088882446289062, + "mean_token_accuracy": 0.5598554380238057, + "num_tokens": 35381469.0, + "step": 6010 + }, + { + "entropy": 2.0172985911369326, + "epoch": 0.843639421224118, + "grad_norm": 3.5, + "learning_rate": 8.237463126843659e-07, + "loss": 2.047108459472656, + "mean_token_accuracy": 0.5510746836662292, + "num_tokens": 35441252.0, + "step": 6020 + }, + { + "entropy": 2.056035044789314, + "epoch": 0.8450408156115334, + "grad_norm": 3.859375, + "learning_rate": 8.163716814159292e-07, + "loss": 2.094436836242676, + "mean_token_accuracy": 0.5377640590071678, + "num_tokens": 35504029.0, + "step": 6030 + }, + { + "entropy": 2.0101221829652784, + "epoch": 0.8464422099989489, + "grad_norm": 3.546875, + "learning_rate": 8.089970501474927e-07, + "loss": 2.035587501525879, + "mean_token_accuracy": 0.5499837107956409, + "num_tokens": 35562638.0, + "step": 6040 + }, + { + "entropy": 2.047288802266121, + "epoch": 0.8478436043863644, + "grad_norm": 3.609375, + "learning_rate": 8.016224188790561e-07, + "loss": 2.0719959259033205, + "mean_token_accuracy": 0.5444797173142433, + "num_tokens": 35623543.0, + "step": 6050 + }, + { + "entropy": 2.019099435210228, + "epoch": 0.8492449987737799, + "grad_norm": 4.03125, + "learning_rate": 7.942477876106196e-07, + "loss": 2.0497373580932616, + "mean_token_accuracy": 0.5494643032550812, + "num_tokens": 35679985.0, + "step": 6060 + }, + { + "entropy": 2.0061439841985704, + "epoch": 0.8506463931611954, + "grad_norm": 3.5, + "learning_rate": 7.868731563421829e-07, + "loss": 2.0653059005737306, + "mean_token_accuracy": 0.5476897947490216, + "num_tokens": 35739820.0, + "step": 6070 + }, + { + "entropy": 2.0328592479228975, + "epoch": 0.8520477875486109, + "grad_norm": 3.46875, + "learning_rate": 7.794985250737463e-07, + "loss": 2.086130905151367, + "mean_token_accuracy": 0.5431623153388501, + "num_tokens": 35796661.0, + "step": 6080 + }, + { + "entropy": 1.8993887215852738, + "epoch": 0.8534491819360264, + "grad_norm": 4.375, + "learning_rate": 7.721238938053098e-07, + "loss": 1.962714385986328, + "mean_token_accuracy": 0.5716985687613487, + "num_tokens": 35847825.0, + "step": 6090 + }, + { + "entropy": 2.075725582242012, + "epoch": 0.8548505763234419, + "grad_norm": 3.671875, + "learning_rate": 7.647492625368732e-07, + "loss": 2.107008171081543, + "mean_token_accuracy": 0.5390588149428368, + "num_tokens": 35907717.0, + "step": 6100 + }, + { + "entropy": 1.8984440177679063, + "epoch": 0.8562519707108573, + "grad_norm": 3.921875, + "learning_rate": 7.573746312684366e-07, + "loss": 1.9352567672729493, + "mean_token_accuracy": 0.5684593334794045, + "num_tokens": 35962545.0, + "step": 6110 + }, + { + "entropy": 2.058637836575508, + "epoch": 0.8576533650982728, + "grad_norm": 3.484375, + "learning_rate": 7.5e-07, + "loss": 2.0837944030761717, + "mean_token_accuracy": 0.5434879846870899, + "num_tokens": 36022968.0, + "step": 6120 + }, + { + "entropy": 1.9400978535413742, + "epoch": 0.8590547594856882, + "grad_norm": 4.25, + "learning_rate": 7.426253687315635e-07, + "loss": 1.9954360961914062, + "mean_token_accuracy": 0.5575679615139961, + "num_tokens": 36076221.0, + "step": 6130 + }, + { + "entropy": 1.9835266083478929, + "epoch": 0.8604561538731037, + "grad_norm": 3.96875, + "learning_rate": 7.352507374631269e-07, + "loss": 2.0139698028564452, + "mean_token_accuracy": 0.5524957269430161, + "num_tokens": 36132536.0, + "step": 6140 + }, + { + "entropy": 2.0456617742776873, + "epoch": 0.8618575482605192, + "grad_norm": 4.28125, + "learning_rate": 7.278761061946903e-07, + "loss": 2.112765884399414, + "mean_token_accuracy": 0.5496705166995526, + "num_tokens": 36190324.0, + "step": 6150 + }, + { + "entropy": 2.035894200205803, + "epoch": 0.8632589426479347, + "grad_norm": 3.609375, + "learning_rate": 7.205014749262537e-07, + "loss": 2.0914268493652344, + "mean_token_accuracy": 0.5396786376833915, + "num_tokens": 36248280.0, + "step": 6160 + }, + { + "entropy": 1.9879158645868302, + "epoch": 0.8646603370353502, + "grad_norm": 4.46875, + "learning_rate": 7.131268436578172e-07, + "loss": 2.035540008544922, + "mean_token_accuracy": 0.5554796390235424, + "num_tokens": 36306248.0, + "step": 6170 + }, + { + "entropy": 2.0229848116636275, + "epoch": 0.8660617314227657, + "grad_norm": 3.9375, + "learning_rate": 7.057522123893807e-07, + "loss": 2.0664846420288088, + "mean_token_accuracy": 0.5501019656658173, + "num_tokens": 36366071.0, + "step": 6180 + }, + { + "entropy": 2.0001530408859254, + "epoch": 0.8674631258101811, + "grad_norm": 3.765625, + "learning_rate": 6.98377581120944e-07, + "loss": 2.0575767517089845, + "mean_token_accuracy": 0.548371671885252, + "num_tokens": 36423193.0, + "step": 6190 + }, + { + "entropy": 2.0265861362218858, + "epoch": 0.8688645201975966, + "grad_norm": 3.578125, + "learning_rate": 6.910029498525074e-07, + "loss": 2.0700199127197267, + "mean_token_accuracy": 0.5455373786389828, + "num_tokens": 36483882.0, + "step": 6200 + }, + { + "entropy": 2.028146120905876, + "epoch": 0.8702659145850121, + "grad_norm": 3.125, + "learning_rate": 6.836283185840709e-07, + "loss": 2.070331573486328, + "mean_token_accuracy": 0.5485374145209789, + "num_tokens": 36545648.0, + "step": 6210 + }, + { + "entropy": 1.987005740404129, + "epoch": 0.8716673089724276, + "grad_norm": 3.859375, + "learning_rate": 6.762536873156344e-07, + "loss": 2.0263193130493162, + "mean_token_accuracy": 0.5584463804960251, + "num_tokens": 36599833.0, + "step": 6220 + }, + { + "entropy": 1.9288410902023316, + "epoch": 0.873068703359843, + "grad_norm": 3.8125, + "learning_rate": 6.688790560471976e-07, + "loss": 1.9976137161254883, + "mean_token_accuracy": 0.5675419226288796, + "num_tokens": 36652349.0, + "step": 6230 + }, + { + "entropy": 2.0247942060232162, + "epoch": 0.8744700977472585, + "grad_norm": 3.46875, + "learning_rate": 6.615044247787611e-07, + "loss": 2.077358627319336, + "mean_token_accuracy": 0.5490225307643414, + "num_tokens": 36709926.0, + "step": 6240 + }, + { + "entropy": 2.037145656347275, + "epoch": 0.875871492134674, + "grad_norm": 3.640625, + "learning_rate": 6.541297935103245e-07, + "loss": 2.0747695922851563, + "mean_token_accuracy": 0.5489478342235088, + "num_tokens": 36766997.0, + "step": 6250 + }, + { + "entropy": 2.08789139688015, + "epoch": 0.8772728865220895, + "grad_norm": 3.453125, + "learning_rate": 6.46755162241888e-07, + "loss": 2.159014892578125, + "mean_token_accuracy": 0.5406771205365658, + "num_tokens": 36825729.0, + "step": 6260 + }, + { + "entropy": 2.02270690202713, + "epoch": 0.878674280909505, + "grad_norm": 3.40625, + "learning_rate": 6.393805309734513e-07, + "loss": 2.078984260559082, + "mean_token_accuracy": 0.5482322216033936, + "num_tokens": 36882508.0, + "step": 6270 + }, + { + "entropy": 2.062803938984871, + "epoch": 0.8800756752969204, + "grad_norm": 3.6875, + "learning_rate": 6.320058997050148e-07, + "loss": 2.1231290817260744, + "mean_token_accuracy": 0.5389542184770107, + "num_tokens": 36943197.0, + "step": 6280 + }, + { + "entropy": 2.0048464626073836, + "epoch": 0.8814770696843359, + "grad_norm": 3.546875, + "learning_rate": 6.246312684365782e-07, + "loss": 2.059210014343262, + "mean_token_accuracy": 0.5485001653432846, + "num_tokens": 37005085.0, + "step": 6290 + }, + { + "entropy": 2.025335270166397, + "epoch": 0.8828784640717514, + "grad_norm": 3.640625, + "learning_rate": 6.172566371681416e-07, + "loss": 2.060206413269043, + "mean_token_accuracy": 0.5467525869607925, + "num_tokens": 37063136.0, + "step": 6300 + }, + { + "entropy": 2.020196759700775, + "epoch": 0.8842798584591669, + "grad_norm": 3.296875, + "learning_rate": 6.098820058997051e-07, + "loss": 2.0538991928100585, + "mean_token_accuracy": 0.550513831526041, + "num_tokens": 37119334.0, + "step": 6310 + }, + { + "entropy": 2.0399301528930662, + "epoch": 0.8856812528465824, + "grad_norm": 4.03125, + "learning_rate": 6.025073746312685e-07, + "loss": 2.0788429260253904, + "mean_token_accuracy": 0.5436412036418915, + "num_tokens": 37178429.0, + "step": 6320 + }, + { + "entropy": 2.000591477751732, + "epoch": 0.8870826472339979, + "grad_norm": 3.546875, + "learning_rate": 5.951327433628319e-07, + "loss": 2.0320425033569336, + "mean_token_accuracy": 0.5498329490423203, + "num_tokens": 37236981.0, + "step": 6330 + }, + { + "entropy": 1.9805440604686737, + "epoch": 0.8884840416214133, + "grad_norm": 4.09375, + "learning_rate": 5.877581120943953e-07, + "loss": 2.0307077407836913, + "mean_token_accuracy": 0.5537135571241378, + "num_tokens": 37294776.0, + "step": 6340 + }, + { + "entropy": 1.936901894211769, + "epoch": 0.8898854360088287, + "grad_norm": 3.75, + "learning_rate": 5.803834808259588e-07, + "loss": 1.9797849655151367, + "mean_token_accuracy": 0.5618837036192417, + "num_tokens": 37352430.0, + "step": 6350 + }, + { + "entropy": 2.053365245461464, + "epoch": 0.8912868303962442, + "grad_norm": 3.546875, + "learning_rate": 5.730088495575221e-07, + "loss": 2.0904542922973635, + "mean_token_accuracy": 0.542573357373476, + "num_tokens": 37416418.0, + "step": 6360 + }, + { + "entropy": 2.000337392091751, + "epoch": 0.8926882247836597, + "grad_norm": 3.625, + "learning_rate": 5.656342182890856e-07, + "loss": 2.0344776153564452, + "mean_token_accuracy": 0.5477051064372063, + "num_tokens": 37475972.0, + "step": 6370 + }, + { + "entropy": 2.0170745253562927, + "epoch": 0.8940896191710752, + "grad_norm": 3.34375, + "learning_rate": 5.58259587020649e-07, + "loss": 2.0558750152587892, + "mean_token_accuracy": 0.5457292802631855, + "num_tokens": 37537323.0, + "step": 6380 + }, + { + "entropy": 1.9845432758331298, + "epoch": 0.8954910135584907, + "grad_norm": 3.546875, + "learning_rate": 5.508849557522124e-07, + "loss": 2.023291015625, + "mean_token_accuracy": 0.5518418118357659, + "num_tokens": 37595925.0, + "step": 6390 + }, + { + "entropy": 2.011869651079178, + "epoch": 0.8968924079459062, + "grad_norm": 3.625, + "learning_rate": 5.435103244837758e-07, + "loss": 2.056960868835449, + "mean_token_accuracy": 0.5488930225372315, + "num_tokens": 37654306.0, + "step": 6400 + }, + { + "entropy": 2.038878303766251, + "epoch": 0.8982938023333217, + "grad_norm": 3.71875, + "learning_rate": 5.361356932153393e-07, + "loss": 2.076828956604004, + "mean_token_accuracy": 0.5448395848274231, + "num_tokens": 37711374.0, + "step": 6410 + }, + { + "entropy": 2.039568394422531, + "epoch": 0.8996951967207372, + "grad_norm": 4.15625, + "learning_rate": 5.287610619469027e-07, + "loss": 2.073482704162598, + "mean_token_accuracy": 0.5399616301059723, + "num_tokens": 37770694.0, + "step": 6420 + }, + { + "entropy": 2.0115788161754606, + "epoch": 0.9010965911081527, + "grad_norm": 3.46875, + "learning_rate": 5.213864306784661e-07, + "loss": 2.050179290771484, + "mean_token_accuracy": 0.5504628643393517, + "num_tokens": 37826837.0, + "step": 6430 + }, + { + "entropy": 2.015952408313751, + "epoch": 0.9024979854955681, + "grad_norm": 3.40625, + "learning_rate": 5.140117994100295e-07, + "loss": 2.071290969848633, + "mean_token_accuracy": 0.5500116847455502, + "num_tokens": 37884941.0, + "step": 6440 + }, + { + "entropy": 2.0158112794160843, + "epoch": 0.9038993798829835, + "grad_norm": 3.84375, + "learning_rate": 5.06637168141593e-07, + "loss": 2.086159324645996, + "mean_token_accuracy": 0.5468775622546673, + "num_tokens": 37944909.0, + "step": 6450 + }, + { + "entropy": 2.0077433407306673, + "epoch": 0.905300774270399, + "grad_norm": 3.65625, + "learning_rate": 4.992625368731564e-07, + "loss": 2.0421270370483398, + "mean_token_accuracy": 0.5518474072217942, + "num_tokens": 38004058.0, + "step": 6460 + }, + { + "entropy": 1.9993306159973145, + "epoch": 0.9067021686578145, + "grad_norm": 3.515625, + "learning_rate": 4.918879056047199e-07, + "loss": 2.06262149810791, + "mean_token_accuracy": 0.5521749749779701, + "num_tokens": 38063422.0, + "step": 6470 + }, + { + "entropy": 1.9897942543029785, + "epoch": 0.90810356304523, + "grad_norm": 4.0, + "learning_rate": 4.845132743362832e-07, + "loss": 2.022821807861328, + "mean_token_accuracy": 0.5545240730047226, + "num_tokens": 38122592.0, + "step": 6480 + }, + { + "entropy": 2.0235477566719053, + "epoch": 0.9095049574326455, + "grad_norm": 3.390625, + "learning_rate": 4.771386430678466e-07, + "loss": 2.066654014587402, + "mean_token_accuracy": 0.5462680235505104, + "num_tokens": 38180327.0, + "step": 6490 + }, + { + "entropy": 2.0121923983097076, + "epoch": 0.910906351820061, + "grad_norm": 3.328125, + "learning_rate": 4.6976401179941004e-07, + "loss": 2.052077293395996, + "mean_token_accuracy": 0.5453770585358143, + "num_tokens": 38244544.0, + "step": 6500 + }, + { + "entropy": 1.990377414226532, + "epoch": 0.9123077462074765, + "grad_norm": 3.421875, + "learning_rate": 4.623893805309735e-07, + "loss": 2.0432369232177736, + "mean_token_accuracy": 0.552157311886549, + "num_tokens": 38301630.0, + "step": 6510 + }, + { + "entropy": 2.0386833012104035, + "epoch": 0.9137091405948919, + "grad_norm": 4.15625, + "learning_rate": 4.550147492625369e-07, + "loss": 2.0687461853027345, + "mean_token_accuracy": 0.5471295416355133, + "num_tokens": 38362047.0, + "step": 6520 + }, + { + "entropy": 2.016241931915283, + "epoch": 0.9151105349823074, + "grad_norm": 3.5625, + "learning_rate": 4.476401179941003e-07, + "loss": 2.0638471603393556, + "mean_token_accuracy": 0.5452109411358833, + "num_tokens": 38420546.0, + "step": 6530 + }, + { + "entropy": 2.0142155259847643, + "epoch": 0.9165119293697229, + "grad_norm": 3.8125, + "learning_rate": 4.402654867256637e-07, + "loss": 2.0311935424804686, + "mean_token_accuracy": 0.5492551133036614, + "num_tokens": 38477064.0, + "step": 6540 + }, + { + "entropy": 2.041605365276337, + "epoch": 0.9179133237571383, + "grad_norm": 3.921875, + "learning_rate": 4.328908554572272e-07, + "loss": 2.0858449935913086, + "mean_token_accuracy": 0.5439679443836212, + "num_tokens": 38537050.0, + "step": 6550 + }, + { + "entropy": 1.9411323845386506, + "epoch": 0.9193147181445538, + "grad_norm": 4.03125, + "learning_rate": 4.255162241887906e-07, + "loss": 1.9711341857910156, + "mean_token_accuracy": 0.5628354027867317, + "num_tokens": 38589735.0, + "step": 6560 + }, + { + "entropy": 2.021375334262848, + "epoch": 0.9207161125319693, + "grad_norm": 4.1875, + "learning_rate": 4.1814159292035404e-07, + "loss": 2.0553340911865234, + "mean_token_accuracy": 0.548372670263052, + "num_tokens": 38647281.0, + "step": 6570 + }, + { + "entropy": 1.9824792742729187, + "epoch": 0.9221175069193848, + "grad_norm": 4.21875, + "learning_rate": 4.1076696165191743e-07, + "loss": 2.0263160705566405, + "mean_token_accuracy": 0.561814583837986, + "num_tokens": 38704914.0, + "step": 6580 + }, + { + "entropy": 1.9721422612667083, + "epoch": 0.9235189013068003, + "grad_norm": 3.90625, + "learning_rate": 4.033923303834809e-07, + "loss": 1.9989608764648437, + "mean_token_accuracy": 0.5563309617340565, + "num_tokens": 38762710.0, + "step": 6590 + }, + { + "entropy": 1.9227455765008927, + "epoch": 0.9249202956942157, + "grad_norm": 4.15625, + "learning_rate": 3.9601769911504427e-07, + "loss": 1.9534294128417968, + "mean_token_accuracy": 0.5688752263784409, + "num_tokens": 38813817.0, + "step": 6600 + }, + { + "entropy": 2.0381629914045334, + "epoch": 0.9263216900816312, + "grad_norm": 3.515625, + "learning_rate": 3.886430678466077e-07, + "loss": 2.085663414001465, + "mean_token_accuracy": 0.5444460518658161, + "num_tokens": 38873859.0, + "step": 6610 + }, + { + "entropy": 1.9983567535877227, + "epoch": 0.9277230844690467, + "grad_norm": 4.125, + "learning_rate": 3.812684365781711e-07, + "loss": 2.040988540649414, + "mean_token_accuracy": 0.5537363089621067, + "num_tokens": 38930081.0, + "step": 6620 + }, + { + "entropy": 2.054008278250694, + "epoch": 0.9291244788564622, + "grad_norm": 3.75, + "learning_rate": 3.7389380530973454e-07, + "loss": 2.078899955749512, + "mean_token_accuracy": 0.5432993650436402, + "num_tokens": 38992167.0, + "step": 6630 + }, + { + "entropy": 1.9960554003715516, + "epoch": 0.9305258732438777, + "grad_norm": 3.796875, + "learning_rate": 3.6651917404129794e-07, + "loss": 2.0496959686279297, + "mean_token_accuracy": 0.5463558658957481, + "num_tokens": 39050033.0, + "step": 6640 + }, + { + "entropy": 2.0130848824977874, + "epoch": 0.9319272676312932, + "grad_norm": 3.8125, + "learning_rate": 3.591445427728614e-07, + "loss": 2.0514385223388674, + "mean_token_accuracy": 0.5462015986442565, + "num_tokens": 39109813.0, + "step": 6650 + }, + { + "entropy": 2.0008685261011125, + "epoch": 0.9333286620187086, + "grad_norm": 3.984375, + "learning_rate": 3.5176991150442477e-07, + "loss": 2.0396297454833983, + "mean_token_accuracy": 0.5541946470737458, + "num_tokens": 39169418.0, + "step": 6660 + }, + { + "entropy": 2.007505992054939, + "epoch": 0.934730056406124, + "grad_norm": 4.0625, + "learning_rate": 3.4439528023598827e-07, + "loss": 2.028615188598633, + "mean_token_accuracy": 0.5482863061130047, + "num_tokens": 39231387.0, + "step": 6670 + }, + { + "entropy": 1.9650995761156083, + "epoch": 0.9361314507935395, + "grad_norm": 3.734375, + "learning_rate": 3.370206489675516e-07, + "loss": 2.010177803039551, + "mean_token_accuracy": 0.553785203397274, + "num_tokens": 39289423.0, + "step": 6680 + }, + { + "entropy": 2.006561702489853, + "epoch": 0.937532845180955, + "grad_norm": 3.5, + "learning_rate": 3.296460176991151e-07, + "loss": 2.041889762878418, + "mean_token_accuracy": 0.5466282285749913, + "num_tokens": 39348639.0, + "step": 6690 + }, + { + "entropy": 2.0054247260093687, + "epoch": 0.9389342395683705, + "grad_norm": 3.8125, + "learning_rate": 3.222713864306785e-07, + "loss": 2.0583980560302733, + "mean_token_accuracy": 0.549737986177206, + "num_tokens": 39406443.0, + "step": 6700 + }, + { + "entropy": 2.0257751524448393, + "epoch": 0.940335633955786, + "grad_norm": 3.265625, + "learning_rate": 3.1489675516224194e-07, + "loss": 2.0774940490722655, + "mean_token_accuracy": 0.5478138782083988, + "num_tokens": 39465975.0, + "step": 6710 + }, + { + "entropy": 2.0418166309595107, + "epoch": 0.9417370283432015, + "grad_norm": 3.46875, + "learning_rate": 3.0752212389380533e-07, + "loss": 2.0822931289672852, + "mean_token_accuracy": 0.5457756318151951, + "num_tokens": 39525372.0, + "step": 6720 + }, + { + "entropy": 2.040261897444725, + "epoch": 0.943138422730617, + "grad_norm": 3.71875, + "learning_rate": 3.0014749262536877e-07, + "loss": 2.0909551620483398, + "mean_token_accuracy": 0.5410762540996075, + "num_tokens": 39586099.0, + "step": 6730 + }, + { + "entropy": 1.9952420234680175, + "epoch": 0.9445398171180325, + "grad_norm": 3.703125, + "learning_rate": 2.927728613569322e-07, + "loss": 2.024325942993164, + "mean_token_accuracy": 0.5516095891594887, + "num_tokens": 39647194.0, + "step": 6740 + }, + { + "entropy": 2.010332950949669, + "epoch": 0.945941211505448, + "grad_norm": 3.34375, + "learning_rate": 2.853982300884956e-07, + "loss": 2.0505157470703126, + "mean_token_accuracy": 0.5492790564894676, + "num_tokens": 39706766.0, + "step": 6750 + }, + { + "entropy": 1.996803131699562, + "epoch": 0.9473426058928635, + "grad_norm": 3.34375, + "learning_rate": 2.7802359882005905e-07, + "loss": 2.043563461303711, + "mean_token_accuracy": 0.5545439317822456, + "num_tokens": 39765877.0, + "step": 6760 + }, + { + "entropy": 2.0132049292325975, + "epoch": 0.9487440002802788, + "grad_norm": 3.671875, + "learning_rate": 2.7064896755162244e-07, + "loss": 2.056793975830078, + "mean_token_accuracy": 0.545245599001646, + "num_tokens": 39826022.0, + "step": 6770 + }, + { + "entropy": 2.005643293261528, + "epoch": 0.9501453946676943, + "grad_norm": 3.484375, + "learning_rate": 2.632743362831859e-07, + "loss": 2.038430595397949, + "mean_token_accuracy": 0.5540463931858539, + "num_tokens": 39884745.0, + "step": 6780 + }, + { + "entropy": 2.059726729989052, + "epoch": 0.9515467890551098, + "grad_norm": 3.890625, + "learning_rate": 2.558997050147493e-07, + "loss": 2.0972423553466797, + "mean_token_accuracy": 0.5440350763499737, + "num_tokens": 39942285.0, + "step": 6790 + }, + { + "entropy": 2.0223002463579176, + "epoch": 0.9529481834425253, + "grad_norm": 3.65625, + "learning_rate": 2.485250737463127e-07, + "loss": 2.0618755340576174, + "mean_token_accuracy": 0.5452701196074485, + "num_tokens": 40000931.0, + "step": 6800 + }, + { + "entropy": 2.0402153849601747, + "epoch": 0.9543495778299408, + "grad_norm": 3.8125, + "learning_rate": 2.4115044247787616e-07, + "loss": 2.067853546142578, + "mean_token_accuracy": 0.546335106343031, + "num_tokens": 40058952.0, + "step": 6810 + }, + { + "entropy": 1.9442283779382705, + "epoch": 0.9557509722173563, + "grad_norm": 3.734375, + "learning_rate": 2.3377581120943955e-07, + "loss": 1.9712425231933595, + "mean_token_accuracy": 0.5628250107169152, + "num_tokens": 40114799.0, + "step": 6820 + }, + { + "entropy": 2.054434522986412, + "epoch": 0.9571523666047718, + "grad_norm": 3.390625, + "learning_rate": 2.2640117994100297e-07, + "loss": 2.1256746292114257, + "mean_token_accuracy": 0.5392848886549473, + "num_tokens": 40178091.0, + "step": 6830 + }, + { + "entropy": 1.985149982571602, + "epoch": 0.9585537609921873, + "grad_norm": 3.59375, + "learning_rate": 2.1902654867256642e-07, + "loss": 2.0216928482055665, + "mean_token_accuracy": 0.5527814209461213, + "num_tokens": 40234308.0, + "step": 6840 + }, + { + "entropy": 2.005727228522301, + "epoch": 0.9599551553796027, + "grad_norm": 3.84375, + "learning_rate": 2.1165191740412983e-07, + "loss": 2.044818878173828, + "mean_token_accuracy": 0.5569542542099952, + "num_tokens": 40290671.0, + "step": 6850 + }, + { + "entropy": 1.9439302861690522, + "epoch": 0.9613565497670182, + "grad_norm": 3.34375, + "learning_rate": 2.0427728613569325e-07, + "loss": 2.042837905883789, + "mean_token_accuracy": 0.5579884998500347, + "num_tokens": 40348303.0, + "step": 6860 + }, + { + "entropy": 1.985863122344017, + "epoch": 0.9627579441544336, + "grad_norm": 3.953125, + "learning_rate": 1.9690265486725667e-07, + "loss": 2.0241420745849608, + "mean_token_accuracy": 0.5558819644153118, + "num_tokens": 40404033.0, + "step": 6870 + }, + { + "entropy": 2.0353716880083086, + "epoch": 0.9641593385418491, + "grad_norm": 3.5, + "learning_rate": 1.8952802359882009e-07, + "loss": 2.0784717559814454, + "mean_token_accuracy": 0.5437360376119613, + "num_tokens": 40462297.0, + "step": 6880 + }, + { + "entropy": 1.9631452977657318, + "epoch": 0.9655607329292646, + "grad_norm": 3.578125, + "learning_rate": 1.821533923303835e-07, + "loss": 1.9946128845214843, + "mean_token_accuracy": 0.5578184209764003, + "num_tokens": 40518672.0, + "step": 6890 + }, + { + "entropy": 1.974769440293312, + "epoch": 0.9669621273166801, + "grad_norm": 3.578125, + "learning_rate": 1.7477876106194692e-07, + "loss": 2.008790969848633, + "mean_token_accuracy": 0.5549022503197193, + "num_tokens": 40577347.0, + "step": 6900 + }, + { + "entropy": 2.0247413128614427, + "epoch": 0.9683635217040956, + "grad_norm": 3.84375, + "learning_rate": 1.6740412979351036e-07, + "loss": 2.0420900344848634, + "mean_token_accuracy": 0.550466337800026, + "num_tokens": 40634927.0, + "step": 6910 + }, + { + "entropy": 2.04134958088398, + "epoch": 0.969764916091511, + "grad_norm": 3.734375, + "learning_rate": 1.6002949852507378e-07, + "loss": 2.07739372253418, + "mean_token_accuracy": 0.5457551784813404, + "num_tokens": 40691846.0, + "step": 6920 + }, + { + "entropy": 1.9664236783981324, + "epoch": 0.9711663104789265, + "grad_norm": 3.96875, + "learning_rate": 1.5265486725663717e-07, + "loss": 2.0042022705078124, + "mean_token_accuracy": 0.5581375107169151, + "num_tokens": 40750168.0, + "step": 6930 + }, + { + "entropy": 2.0273396879434586, + "epoch": 0.972567704866342, + "grad_norm": 2.921875, + "learning_rate": 1.452802359882006e-07, + "loss": 2.0899295806884766, + "mean_token_accuracy": 0.5483457125723362, + "num_tokens": 40808730.0, + "step": 6940 + }, + { + "entropy": 2.02415617108345, + "epoch": 0.9739690992537575, + "grad_norm": 3.671875, + "learning_rate": 1.3790560471976403e-07, + "loss": 2.074420356750488, + "mean_token_accuracy": 0.5455530092120171, + "num_tokens": 40868156.0, + "step": 6950 + }, + { + "entropy": 2.0240610927343368, + "epoch": 0.975370493641173, + "grad_norm": 3.4375, + "learning_rate": 1.3053097345132745e-07, + "loss": 2.0607715606689454, + "mean_token_accuracy": 0.5471958234906197, + "num_tokens": 40923682.0, + "step": 6960 + }, + { + "entropy": 2.0293728321790696, + "epoch": 0.9767718880285885, + "grad_norm": 3.34375, + "learning_rate": 1.2315634218289087e-07, + "loss": 2.0637628555297853, + "mean_token_accuracy": 0.544766490906477, + "num_tokens": 40983981.0, + "step": 6970 + }, + { + "entropy": 2.071514305472374, + "epoch": 0.9781732824160039, + "grad_norm": 3.40625, + "learning_rate": 1.1578171091445429e-07, + "loss": 2.109389877319336, + "mean_token_accuracy": 0.5343147926032543, + "num_tokens": 41047569.0, + "step": 6980 + }, + { + "entropy": 2.0054246932268143, + "epoch": 0.9795746768034194, + "grad_norm": 3.390625, + "learning_rate": 1.084070796460177e-07, + "loss": 2.064986801147461, + "mean_token_accuracy": 0.5516049854457379, + "num_tokens": 41108322.0, + "step": 6990 + }, + { + "entropy": 2.007992023229599, + "epoch": 0.9809760711908349, + "grad_norm": 3.234375, + "learning_rate": 1.0103244837758113e-07, + "loss": 2.0503591537475585, + "mean_token_accuracy": 0.5488371036946773, + "num_tokens": 41166844.0, + "step": 7000 + }, + { + "entropy": 2.0042567580938337, + "epoch": 0.9823774655782503, + "grad_norm": 3.734375, + "learning_rate": 9.365781710914455e-08, + "loss": 2.061623382568359, + "mean_token_accuracy": 0.548622415214777, + "num_tokens": 41225177.0, + "step": 7010 + }, + { + "entropy": 1.92800931930542, + "epoch": 0.9837788599656658, + "grad_norm": 3.625, + "learning_rate": 8.628318584070797e-08, + "loss": 1.9699708938598632, + "mean_token_accuracy": 0.5659283697605133, + "num_tokens": 41279035.0, + "step": 7020 + }, + { + "entropy": 2.0187560588121416, + "epoch": 0.9851802543530813, + "grad_norm": 4.3125, + "learning_rate": 7.890855457227139e-08, + "loss": 2.0564207077026366, + "mean_token_accuracy": 0.5510451331734657, + "num_tokens": 41336172.0, + "step": 7030 + }, + { + "entropy": 2.0258085399866106, + "epoch": 0.9865816487404968, + "grad_norm": 4.34375, + "learning_rate": 7.153392330383482e-08, + "loss": 2.0803592681884764, + "mean_token_accuracy": 0.5455730646848679, + "num_tokens": 41393023.0, + "step": 7040 + }, + { + "entropy": 1.9908759355545045, + "epoch": 0.9879830431279123, + "grad_norm": 3.90625, + "learning_rate": 6.415929203539823e-08, + "loss": 2.038408088684082, + "mean_token_accuracy": 0.5520909205079079, + "num_tokens": 41450492.0, + "step": 7050 + }, + { + "entropy": 2.0316341012716292, + "epoch": 0.9893844375153278, + "grad_norm": 3.71875, + "learning_rate": 5.678466076696166e-08, + "loss": 2.0807302474975584, + "mean_token_accuracy": 0.5429827772080899, + "num_tokens": 41513079.0, + "step": 7060 + }, + { + "entropy": 1.9735469341278076, + "epoch": 0.9907858319027433, + "grad_norm": 3.5, + "learning_rate": 4.9410029498525076e-08, + "loss": 2.0276817321777343, + "mean_token_accuracy": 0.556085791438818, + "num_tokens": 41573403.0, + "step": 7070 + }, + { + "entropy": 2.0149279564619063, + "epoch": 0.9921872262901587, + "grad_norm": 3.578125, + "learning_rate": 4.20353982300885e-08, + "loss": 2.053173828125, + "mean_token_accuracy": 0.5515789903700352, + "num_tokens": 41632678.0, + "step": 7080 + }, + { + "entropy": 1.994994157552719, + "epoch": 0.9935886206775741, + "grad_norm": 3.78125, + "learning_rate": 3.466076696165192e-08, + "loss": 2.040749931335449, + "mean_token_accuracy": 0.5496800921857357, + "num_tokens": 41690157.0, + "step": 7090 + }, + { + "entropy": 1.9715099543333054, + "epoch": 0.9949900150649896, + "grad_norm": 4.0625, + "learning_rate": 2.728613569321534e-08, + "loss": 2.0029624938964843, + "mean_token_accuracy": 0.5555107049643994, + "num_tokens": 41746029.0, + "step": 7100 + }, + { + "entropy": 2.0422003865242004, + "epoch": 0.9963914094524051, + "grad_norm": 3.9375, + "learning_rate": 1.9911504424778762e-08, + "loss": 2.0568687438964846, + "mean_token_accuracy": 0.5468258865177631, + "num_tokens": 41803572.0, + "step": 7110 + }, + { + "entropy": 1.9422328054904938, + "epoch": 0.9977928038398206, + "grad_norm": 3.71875, + "learning_rate": 1.2536873156342184e-08, + "loss": 1.9889080047607421, + "mean_token_accuracy": 0.5637653365731239, + "num_tokens": 41856226.0, + "step": 7120 + }, + { + "entropy": 2.0215059161186217, + "epoch": 0.9991941982272361, + "grad_norm": 3.421875, + "learning_rate": 5.162241887905605e-09, + "loss": 2.051664924621582, + "mean_token_accuracy": 0.5480418384075165, + "num_tokens": 41915331.0, + "step": 7130 + } + ], + "logging_steps": 10, + "max_steps": 7136, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.15865159523498e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-7136/training_args.bin b/checkpoint-7136/training_args.bin new file mode 100644 index 0000000..b126b81 --- /dev/null +++ b/checkpoint-7136/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c570d44b5dfac423532e77f40f4b5a1be2f35231009d8aad93eb07d41a9a26c +size 5713 diff --git a/checkpoint-896/chat_template.jinja b/checkpoint-896/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-896/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-896/config.json b/checkpoint-896/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-896/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-896/generation_config.json b/checkpoint-896/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-896/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-896/model.safetensors b/checkpoint-896/model.safetensors new file mode 100644 index 0000000..42241b3 --- /dev/null +++ b/checkpoint-896/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2206d85a361a65a287958180def127b992592f0eb19c603f5b8c475bcb58dda5 +size 3441185608 diff --git a/checkpoint-896/optimizer.pt b/checkpoint-896/optimizer.pt new file mode 100644 index 0000000..17b0f2d --- /dev/null +++ b/checkpoint-896/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:122160cad57140cf5fab5f32298e738756bd5581c3cde74b388d313837a417df +size 6882572207 diff --git a/checkpoint-896/rng_state.pth b/checkpoint-896/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-896/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-896/scheduler.pt b/checkpoint-896/scheduler.pt new file mode 100644 index 0000000..3f1eab6 --- /dev/null +++ b/checkpoint-896/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2b8f7d0b80c816b08ea7e04f404377b6b56ff8b0da07c9c66b6046193d8972a +size 1465 diff --git a/checkpoint-896/tokenizer.json b/checkpoint-896/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-896/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-896/tokenizer_config.json b/checkpoint-896/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/checkpoint-896/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-896/trainer_state.json b/checkpoint-896/trainer_state.json new file mode 100644 index 0000000..5c2b82d --- /dev/null +++ b/checkpoint-896/trainer_state.json @@ -0,0 +1,924 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 896, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.2742946662008763, + "epoch": 0.011166945840312675, + "grad_norm": 57.75, + "learning_rate": 2.02247191011236e-06, + "loss": 2.0921743392944334, + "mean_token_accuracy": 0.6198021434247494, + "num_tokens": 27228.0, + "step": 10 + }, + { + "entropy": 1.4035808108747005, + "epoch": 0.02233389168062535, + "grad_norm": 27.5, + "learning_rate": 4.269662921348315e-06, + "loss": 1.8684164047241212, + "mean_token_accuracy": 0.6308974869549274, + "num_tokens": 54263.0, + "step": 20 + }, + { + "entropy": 1.3806322045624255, + "epoch": 0.03350083752093802, + "grad_norm": 7.4375, + "learning_rate": 6.51685393258427e-06, + "loss": 1.5451434135437012, + "mean_token_accuracy": 0.6709699966013432, + "num_tokens": 80355.0, + "step": 30 + }, + { + "entropy": 1.2432026788592339, + "epoch": 0.0446677833612507, + "grad_norm": 4.4375, + "learning_rate": 8.764044943820226e-06, + "loss": 1.3852472305297852, + "mean_token_accuracy": 0.727528478205204, + "num_tokens": 107262.0, + "step": 40 + }, + { + "entropy": 1.2347759172320365, + "epoch": 0.05583472920156337, + "grad_norm": 3.96875, + "learning_rate": 1.101123595505618e-05, + "loss": 1.3566075325012208, + "mean_token_accuracy": 0.7324926406145096, + "num_tokens": 134872.0, + "step": 50 + }, + { + "entropy": 1.1513216629624368, + "epoch": 0.06700167504187604, + "grad_norm": 4.0, + "learning_rate": 1.3258426966292135e-05, + "loss": 1.2471713066101073, + "mean_token_accuracy": 0.7584743887186051, + "num_tokens": 159459.0, + "step": 60 + }, + { + "entropy": 1.156738107651472, + "epoch": 0.07816862088218872, + "grad_norm": 3.578125, + "learning_rate": 1.5505617977528093e-05, + "loss": 1.239414596557617, + "mean_token_accuracy": 0.749166414886713, + "num_tokens": 184576.0, + "step": 70 + }, + { + "entropy": 1.157951571792364, + "epoch": 0.0893355667225014, + "grad_norm": 3.640625, + "learning_rate": 1.7752808988764045e-05, + "loss": 1.2582244873046875, + "mean_token_accuracy": 0.7544261917471886, + "num_tokens": 210131.0, + "step": 80 + }, + { + "entropy": 1.1601063475012778, + "epoch": 0.10050251256281408, + "grad_norm": 3.625, + "learning_rate": 2e-05, + "loss": 1.2618935585021973, + "mean_token_accuracy": 0.747503137588501, + "num_tokens": 237729.0, + "step": 90 + }, + { + "entropy": 1.0921222895383835, + "epoch": 0.11166945840312674, + "grad_norm": 3.59375, + "learning_rate": 1.975216852540273e-05, + "loss": 1.1785853385925293, + "mean_token_accuracy": 0.7695864170789719, + "num_tokens": 262275.0, + "step": 100 + }, + { + "entropy": 1.1327178589999676, + "epoch": 0.12283640424343942, + "grad_norm": 3.609375, + "learning_rate": 1.9504337050805453e-05, + "loss": 1.216773509979248, + "mean_token_accuracy": 0.7555348932743072, + "num_tokens": 288546.0, + "step": 110 + }, + { + "entropy": 1.035539500042796, + "epoch": 0.13400335008375208, + "grad_norm": 3.78125, + "learning_rate": 1.925650557620818e-05, + "loss": 1.1224644660949707, + "mean_token_accuracy": 0.7721244119107723, + "num_tokens": 312401.0, + "step": 120 + }, + { + "entropy": 1.1713032595813275, + "epoch": 0.14517029592406477, + "grad_norm": 3.671875, + "learning_rate": 1.9008674101610908e-05, + "loss": 1.3123122215270997, + "mean_token_accuracy": 0.7460119985044003, + "num_tokens": 339734.0, + "step": 130 + }, + { + "entropy": 1.1201179042458533, + "epoch": 0.15633724176437744, + "grad_norm": 3.65625, + "learning_rate": 1.8760842627013632e-05, + "loss": 1.2395885467529297, + "mean_token_accuracy": 0.7611079879105092, + "num_tokens": 364808.0, + "step": 140 + }, + { + "entropy": 1.1376964189112186, + "epoch": 0.16750418760469013, + "grad_norm": 3.171875, + "learning_rate": 1.851301115241636e-05, + "loss": 1.2277887344360352, + "mean_token_accuracy": 0.7598711617290974, + "num_tokens": 391934.0, + "step": 150 + }, + { + "entropy": 1.0489437825977803, + "epoch": 0.1786711334450028, + "grad_norm": 3.96875, + "learning_rate": 1.8265179677819083e-05, + "loss": 1.1456377029418945, + "mean_token_accuracy": 0.7739411622285843, + "num_tokens": 416529.0, + "step": 160 + }, + { + "entropy": 1.0241498839110137, + "epoch": 0.18983807928531546, + "grad_norm": 3.6875, + "learning_rate": 1.801734820322181e-05, + "loss": 1.1425954818725585, + "mean_token_accuracy": 0.77560595870018, + "num_tokens": 441104.0, + "step": 170 + }, + { + "entropy": 1.1297610633075237, + "epoch": 0.20100502512562815, + "grad_norm": 3.3125, + "learning_rate": 1.7769516728624535e-05, + "loss": 1.2210969924926758, + "mean_token_accuracy": 0.7550569862127304, + "num_tokens": 468069.0, + "step": 180 + }, + { + "entropy": 1.112941750138998, + "epoch": 0.21217197096594081, + "grad_norm": 3.65625, + "learning_rate": 1.7521685254027262e-05, + "loss": 1.192617416381836, + "mean_token_accuracy": 0.7619749017059803, + "num_tokens": 494025.0, + "step": 190 + }, + { + "entropy": 1.063616494834423, + "epoch": 0.22333891680625348, + "grad_norm": 3.828125, + "learning_rate": 1.727385377942999e-05, + "loss": 1.188753604888916, + "mean_token_accuracy": 0.7676796585321426, + "num_tokens": 519158.0, + "step": 200 + }, + { + "entropy": 1.1531932204961777, + "epoch": 0.23450586264656617, + "grad_norm": 3.59375, + "learning_rate": 1.7026022304832714e-05, + "loss": 1.2400163650512694, + "mean_token_accuracy": 0.7510207004845142, + "num_tokens": 546677.0, + "step": 210 + }, + { + "entropy": 1.1051580917090178, + "epoch": 0.24567280848687884, + "grad_norm": 3.703125, + "learning_rate": 1.677819083023544e-05, + "loss": 1.1891214370727539, + "mean_token_accuracy": 0.7638134479522705, + "num_tokens": 571855.0, + "step": 220 + }, + { + "entropy": 1.1669820021837949, + "epoch": 0.2568397543271915, + "grad_norm": 3.546875, + "learning_rate": 1.653035935563817e-05, + "loss": 1.2794105529785156, + "mean_token_accuracy": 0.7478987194597722, + "num_tokens": 599409.0, + "step": 230 + }, + { + "entropy": 1.118950032442808, + "epoch": 0.26800670016750416, + "grad_norm": 4.0, + "learning_rate": 1.6282527881040892e-05, + "loss": 1.2216855049133302, + "mean_token_accuracy": 0.7556364260613918, + "num_tokens": 625984.0, + "step": 240 + }, + { + "entropy": 1.1487022332847119, + "epoch": 0.27917364600781686, + "grad_norm": 3.0625, + "learning_rate": 1.603469640644362e-05, + "loss": 1.3041958808898926, + "mean_token_accuracy": 0.7531310901045799, + "num_tokens": 654422.0, + "step": 250 + }, + { + "entropy": 1.067482265457511, + "epoch": 0.29034059184812955, + "grad_norm": 3.84375, + "learning_rate": 1.5786864931846347e-05, + "loss": 1.1802674293518067, + "mean_token_accuracy": 0.7648798644542694, + "num_tokens": 679950.0, + "step": 260 + }, + { + "entropy": 1.1374815497547388, + "epoch": 0.3015075376884422, + "grad_norm": 3.234375, + "learning_rate": 1.553903345724907e-05, + "loss": 1.2403667449951172, + "mean_token_accuracy": 0.7576499588787555, + "num_tokens": 707691.0, + "step": 270 + }, + { + "entropy": 1.0954229425638915, + "epoch": 0.3126744835287549, + "grad_norm": 3.8125, + "learning_rate": 1.52912019826518e-05, + "loss": 1.1737814903259278, + "mean_token_accuracy": 0.765420364588499, + "num_tokens": 732904.0, + "step": 280 + }, + { + "entropy": 1.1397546224296093, + "epoch": 0.32384142936906757, + "grad_norm": 3.5625, + "learning_rate": 1.5043370508054524e-05, + "loss": 1.2469436645507812, + "mean_token_accuracy": 0.7499822400510311, + "num_tokens": 760829.0, + "step": 290 + }, + { + "entropy": 1.0366052724421024, + "epoch": 0.33500837520938026, + "grad_norm": 3.84375, + "learning_rate": 1.4795539033457252e-05, + "loss": 1.140453815460205, + "mean_token_accuracy": 0.7735538497567177, + "num_tokens": 785442.0, + "step": 300 + }, + { + "entropy": 1.1235052689909935, + "epoch": 0.3461753210496929, + "grad_norm": 3.21875, + "learning_rate": 1.4547707558859976e-05, + "loss": 1.23864803314209, + "mean_token_accuracy": 0.7578027009963989, + "num_tokens": 813321.0, + "step": 310 + }, + { + "entropy": 1.0663026701658964, + "epoch": 0.3573422668900056, + "grad_norm": 3.296875, + "learning_rate": 1.4299876084262703e-05, + "loss": 1.1644706726074219, + "mean_token_accuracy": 0.7657143622636795, + "num_tokens": 840571.0, + "step": 320 + }, + { + "entropy": 1.1613866232335568, + "epoch": 0.3685092127303183, + "grad_norm": 3.296875, + "learning_rate": 1.4052044609665429e-05, + "loss": 1.215484046936035, + "mean_token_accuracy": 0.7551229789853096, + "num_tokens": 868204.0, + "step": 330 + }, + { + "entropy": 1.038713937997818, + "epoch": 0.3796761585706309, + "grad_norm": 3.609375, + "learning_rate": 1.3804213135068155e-05, + "loss": 1.133761501312256, + "mean_token_accuracy": 0.7695340342819691, + "num_tokens": 893234.0, + "step": 340 + }, + { + "entropy": 1.117867962270975, + "epoch": 0.3908431044109436, + "grad_norm": 3.359375, + "learning_rate": 1.355638166047088e-05, + "loss": 1.2365485191345216, + "mean_token_accuracy": 0.751798415929079, + "num_tokens": 920370.0, + "step": 350 + }, + { + "entropy": 1.066711399704218, + "epoch": 0.4020100502512563, + "grad_norm": 3.25, + "learning_rate": 1.3308550185873608e-05, + "loss": 1.149217128753662, + "mean_token_accuracy": 0.7712466239929199, + "num_tokens": 944799.0, + "step": 360 + }, + { + "entropy": 1.09353599101305, + "epoch": 0.41317699609156894, + "grad_norm": 3.640625, + "learning_rate": 1.3060718711276332e-05, + "loss": 1.1921247482299804, + "mean_token_accuracy": 0.7622878901660443, + "num_tokens": 970357.0, + "step": 370 + }, + { + "entropy": 1.0636055015027523, + "epoch": 0.42434394193188163, + "grad_norm": 3.40625, + "learning_rate": 1.281288723667906e-05, + "loss": 1.1449426651000976, + "mean_token_accuracy": 0.7751508951187134, + "num_tokens": 995627.0, + "step": 380 + }, + { + "entropy": 1.0997799094766378, + "epoch": 0.4355108877721943, + "grad_norm": 3.46875, + "learning_rate": 1.2565055762081787e-05, + "loss": 1.1890180587768555, + "mean_token_accuracy": 0.7632672250270843, + "num_tokens": 1021258.0, + "step": 390 + }, + { + "entropy": 1.0799225345253944, + "epoch": 0.44667783361250696, + "grad_norm": 3.328125, + "learning_rate": 1.231722428748451e-05, + "loss": 1.1613880157470704, + "mean_token_accuracy": 0.7609394922852516, + "num_tokens": 1048009.0, + "step": 400 + }, + { + "entropy": 1.0300600126385688, + "epoch": 0.45784477945281965, + "grad_norm": 3.296875, + "learning_rate": 1.2069392812887238e-05, + "loss": 1.1121676445007325, + "mean_token_accuracy": 0.7761823385953903, + "num_tokens": 1072790.0, + "step": 410 + }, + { + "entropy": 1.03407681286335, + "epoch": 0.46901172529313234, + "grad_norm": 3.421875, + "learning_rate": 1.1821561338289964e-05, + "loss": 1.1337100982666015, + "mean_token_accuracy": 0.779456903040409, + "num_tokens": 1096424.0, + "step": 420 + }, + { + "entropy": 1.0816764250397681, + "epoch": 0.480178671133445, + "grad_norm": 3.09375, + "learning_rate": 1.1573729863692691e-05, + "loss": 1.168564796447754, + "mean_token_accuracy": 0.7695191375911236, + "num_tokens": 1121279.0, + "step": 430 + }, + { + "entropy": 1.0950494274497031, + "epoch": 0.49134561697375767, + "grad_norm": 3.671875, + "learning_rate": 1.1325898389095415e-05, + "loss": 1.220743751525879, + "mean_token_accuracy": 0.7663791351020336, + "num_tokens": 1146975.0, + "step": 440 + }, + { + "entropy": 1.0629482321441173, + "epoch": 0.5025125628140703, + "grad_norm": 3.359375, + "learning_rate": 1.1078066914498143e-05, + "loss": 1.17555513381958, + "mean_token_accuracy": 0.7702355854213238, + "num_tokens": 1171720.0, + "step": 450 + }, + { + "entropy": 1.0769638925790788, + "epoch": 0.513679508654383, + "grad_norm": 3.203125, + "learning_rate": 1.083023543990087e-05, + "loss": 1.1584683418273927, + "mean_token_accuracy": 0.7703170344233513, + "num_tokens": 1197191.0, + "step": 460 + }, + { + "entropy": 1.0924316361546516, + "epoch": 0.5248464544946957, + "grad_norm": 3.53125, + "learning_rate": 1.0582403965303594e-05, + "loss": 1.197894287109375, + "mean_token_accuracy": 0.7625605218112469, + "num_tokens": 1223732.0, + "step": 470 + }, + { + "entropy": 1.1760938204824924, + "epoch": 0.5360134003350083, + "grad_norm": 3.21875, + "learning_rate": 1.0334572490706321e-05, + "loss": 1.3036236763000488, + "mean_token_accuracy": 0.7476884454488755, + "num_tokens": 1252654.0, + "step": 480 + }, + { + "entropy": 1.1041141584515572, + "epoch": 0.5471803461753211, + "grad_norm": 3.328125, + "learning_rate": 1.0086741016109047e-05, + "loss": 1.1862168312072754, + "mean_token_accuracy": 0.7658242657780647, + "num_tokens": 1278284.0, + "step": 490 + }, + { + "entropy": 1.0600057408213615, + "epoch": 0.5583472920156337, + "grad_norm": 3.640625, + "learning_rate": 9.838909541511773e-06, + "loss": 1.1444557189941407, + "mean_token_accuracy": 0.7672035470604897, + "num_tokens": 1303786.0, + "step": 500 + }, + { + "entropy": 1.1602862119674682, + "epoch": 0.5695142378559463, + "grad_norm": 3.546875, + "learning_rate": 9.591078066914498e-06, + "loss": 1.2625015258789063, + "mean_token_accuracy": 0.7455270260572433, + "num_tokens": 1330403.0, + "step": 510 + }, + { + "entropy": 1.0500171076506377, + "epoch": 0.5806811836962591, + "grad_norm": 3.34375, + "learning_rate": 9.343246592317226e-06, + "loss": 1.1100471496582032, + "mean_token_accuracy": 0.7771873719990253, + "num_tokens": 1355913.0, + "step": 520 + }, + { + "entropy": 1.0421214148402214, + "epoch": 0.5918481295365717, + "grad_norm": 3.421875, + "learning_rate": 9.095415117719952e-06, + "loss": 1.1434844970703124, + "mean_token_accuracy": 0.7771047562360763, + "num_tokens": 1381048.0, + "step": 530 + }, + { + "entropy": 1.1398710921406745, + "epoch": 0.6030150753768844, + "grad_norm": 3.765625, + "learning_rate": 8.847583643122677e-06, + "loss": 1.237742042541504, + "mean_token_accuracy": 0.7521290302276611, + "num_tokens": 1407965.0, + "step": 540 + }, + { + "entropy": 1.0272238925099373, + "epoch": 0.6141820212171971, + "grad_norm": 3.28125, + "learning_rate": 8.599752168525403e-06, + "loss": 1.1173413276672364, + "mean_token_accuracy": 0.7779941506683826, + "num_tokens": 1432806.0, + "step": 550 + }, + { + "entropy": 1.0616241727024316, + "epoch": 0.6253489670575098, + "grad_norm": 3.15625, + "learning_rate": 8.351920693928129e-06, + "loss": 1.1724609375, + "mean_token_accuracy": 0.7691180802881717, + "num_tokens": 1458745.0, + "step": 560 + }, + { + "entropy": 1.1184002067893744, + "epoch": 0.6365159128978225, + "grad_norm": 3.65625, + "learning_rate": 8.104089219330854e-06, + "loss": 1.228511905670166, + "mean_token_accuracy": 0.7584543086588382, + "num_tokens": 1485413.0, + "step": 570 + }, + { + "entropy": 1.123507371917367, + "epoch": 0.6476828587381351, + "grad_norm": 3.4375, + "learning_rate": 7.856257744733582e-06, + "loss": 1.2577102661132813, + "mean_token_accuracy": 0.7598744049668312, + "num_tokens": 1513305.0, + "step": 580 + }, + { + "entropy": 1.0770755916833878, + "epoch": 0.6588498045784478, + "grad_norm": 3.28125, + "learning_rate": 7.608426270136308e-06, + "loss": 1.1567678451538086, + "mean_token_accuracy": 0.7634399652481079, + "num_tokens": 1539813.0, + "step": 590 + }, + { + "entropy": 0.9889072474092245, + "epoch": 0.6700167504187605, + "grad_norm": 3.65625, + "learning_rate": 7.360594795539034e-06, + "loss": 1.0486254692077637, + "mean_token_accuracy": 0.7880329728126526, + "num_tokens": 1563321.0, + "step": 600 + }, + { + "entropy": 1.0968003824353219, + "epoch": 0.6811836962590732, + "grad_norm": 3.28125, + "learning_rate": 7.11276332094176e-06, + "loss": 1.1853549003601074, + "mean_token_accuracy": 0.7634060740470886, + "num_tokens": 1590121.0, + "step": 610 + }, + { + "entropy": 1.0346317429095506, + "epoch": 0.6923506420993858, + "grad_norm": 3.078125, + "learning_rate": 6.8649318463444856e-06, + "loss": 1.09486722946167, + "mean_token_accuracy": 0.7758483737707138, + "num_tokens": 1616645.0, + "step": 620 + }, + { + "entropy": 1.1052446074783802, + "epoch": 0.7035175879396985, + "grad_norm": 4.09375, + "learning_rate": 6.617100371747213e-06, + "loss": 1.18992280960083, + "mean_token_accuracy": 0.7618899635970593, + "num_tokens": 1642059.0, + "step": 630 + }, + { + "entropy": 1.0749453879892825, + "epoch": 0.7146845337800112, + "grad_norm": 3.4375, + "learning_rate": 6.369268897149939e-06, + "loss": 1.1491153717041016, + "mean_token_accuracy": 0.7696318380534649, + "num_tokens": 1666939.0, + "step": 640 + }, + { + "entropy": 1.0601326443254948, + "epoch": 0.7258514796203238, + "grad_norm": 3.640625, + "learning_rate": 6.121437422552665e-06, + "loss": 1.1444756507873535, + "mean_token_accuracy": 0.7738051861524582, + "num_tokens": 1692634.0, + "step": 650 + }, + { + "entropy": 1.1047037810087204, + "epoch": 0.7370184254606366, + "grad_norm": 3.53125, + "learning_rate": 5.873605947955391e-06, + "loss": 1.2263466835021972, + "mean_token_accuracy": 0.7588796854019165, + "num_tokens": 1720055.0, + "step": 660 + }, + { + "entropy": 1.0764735087752342, + "epoch": 0.7481853713009492, + "grad_norm": 3.40625, + "learning_rate": 5.625774473358117e-06, + "loss": 1.2022334098815919, + "mean_token_accuracy": 0.7653821043670177, + "num_tokens": 1746264.0, + "step": 670 + }, + { + "entropy": 1.0576500050723552, + "epoch": 0.7593523171412618, + "grad_norm": 3.21875, + "learning_rate": 5.377942998760843e-06, + "loss": 1.1265974998474122, + "mean_token_accuracy": 0.7686478443443775, + "num_tokens": 1773301.0, + "step": 680 + }, + { + "entropy": 1.1367420602589846, + "epoch": 0.7705192629815746, + "grad_norm": 3.4375, + "learning_rate": 5.130111524163569e-06, + "loss": 1.2707669258117675, + "mean_token_accuracy": 0.7557471729815006, + "num_tokens": 1800329.0, + "step": 690 + }, + { + "entropy": 1.0998500097543, + "epoch": 0.7816862088218872, + "grad_norm": 3.875, + "learning_rate": 4.8822800495662955e-06, + "loss": 1.184847068786621, + "mean_token_accuracy": 0.7647615008056163, + "num_tokens": 1825475.0, + "step": 700 + }, + { + "entropy": 1.1467237778007984, + "epoch": 0.7928531546621999, + "grad_norm": 3.171875, + "learning_rate": 4.634448574969021e-06, + "loss": 1.2255634307861327, + "mean_token_accuracy": 0.7532035551965237, + "num_tokens": 1852394.0, + "step": 710 + }, + { + "entropy": 1.079410395771265, + "epoch": 0.8040201005025126, + "grad_norm": 3.59375, + "learning_rate": 4.386617100371748e-06, + "loss": 1.2125642776489258, + "mean_token_accuracy": 0.767810083925724, + "num_tokens": 1878744.0, + "step": 720 + }, + { + "entropy": 1.0579048234969377, + "epoch": 0.8151870463428252, + "grad_norm": 3.296875, + "learning_rate": 4.1387856257744735e-06, + "loss": 1.1596202850341797, + "mean_token_accuracy": 0.7713351771235466, + "num_tokens": 1903148.0, + "step": 730 + }, + { + "entropy": 1.0731349058449269, + "epoch": 0.8263539921831379, + "grad_norm": 3.328125, + "learning_rate": 3.8909541511772e-06, + "loss": 1.1340813636779785, + "mean_token_accuracy": 0.7696133933961391, + "num_tokens": 1928626.0, + "step": 740 + }, + { + "entropy": 1.1284345269203186, + "epoch": 0.8375209380234506, + "grad_norm": 3.625, + "learning_rate": 3.643122676579926e-06, + "loss": 1.2018820762634277, + "mean_token_accuracy": 0.7566148206591606, + "num_tokens": 1955732.0, + "step": 750 + }, + { + "entropy": 1.1177599750459195, + "epoch": 0.8486878838637633, + "grad_norm": 3.703125, + "learning_rate": 3.3952912019826523e-06, + "loss": 1.2435114860534668, + "mean_token_accuracy": 0.7585679024457932, + "num_tokens": 1981894.0, + "step": 760 + }, + { + "entropy": 1.0989192590117454, + "epoch": 0.8598548297040759, + "grad_norm": 3.40625, + "learning_rate": 3.147459727385378e-06, + "loss": 1.1960742950439454, + "mean_token_accuracy": 0.7629640743136406, + "num_tokens": 2008438.0, + "step": 770 + }, + { + "entropy": 1.0289450511336327, + "epoch": 0.8710217755443886, + "grad_norm": 3.453125, + "learning_rate": 2.899628252788104e-06, + "loss": 1.112326717376709, + "mean_token_accuracy": 0.7783081293106079, + "num_tokens": 2033527.0, + "step": 780 + }, + { + "entropy": 1.1425920329988002, + "epoch": 0.8821887213847013, + "grad_norm": 3.453125, + "learning_rate": 2.6517967781908303e-06, + "loss": 1.2318533897399901, + "mean_token_accuracy": 0.7519856467843056, + "num_tokens": 2060494.0, + "step": 790 + }, + { + "entropy": 1.1216130927205086, + "epoch": 0.8933556672250139, + "grad_norm": 3.671875, + "learning_rate": 2.4039653035935564e-06, + "loss": 1.195895004272461, + "mean_token_accuracy": 0.7575728356838226, + "num_tokens": 2086503.0, + "step": 800 + }, + { + "entropy": 1.1177167922258378, + "epoch": 0.9045226130653267, + "grad_norm": 3.765625, + "learning_rate": 2.1561338289962826e-06, + "loss": 1.180656909942627, + "mean_token_accuracy": 0.7616930149495602, + "num_tokens": 2112895.0, + "step": 810 + }, + { + "entropy": 1.0543908223509788, + "epoch": 0.9156895589056393, + "grad_norm": 3.65625, + "learning_rate": 1.908302354399009e-06, + "loss": 1.1741219520568849, + "mean_token_accuracy": 0.7708107247948647, + "num_tokens": 2138001.0, + "step": 820 + }, + { + "entropy": 1.087718739360571, + "epoch": 0.9268565047459519, + "grad_norm": 3.28125, + "learning_rate": 1.6604708798017348e-06, + "loss": 1.177952480316162, + "mean_token_accuracy": 0.7635545134544373, + "num_tokens": 2164192.0, + "step": 830 + }, + { + "entropy": 1.060328460112214, + "epoch": 0.9380234505862647, + "grad_norm": 3.71875, + "learning_rate": 1.4126394052044612e-06, + "loss": 1.1673462867736817, + "mean_token_accuracy": 0.7744978621602059, + "num_tokens": 2189210.0, + "step": 840 + }, + { + "entropy": 1.0364069953560828, + "epoch": 0.9491903964265773, + "grad_norm": 3.484375, + "learning_rate": 1.1648079306071871e-06, + "loss": 1.1158721923828125, + "mean_token_accuracy": 0.7778892047703266, + "num_tokens": 2214446.0, + "step": 850 + }, + { + "entropy": 1.0322446286678315, + "epoch": 0.96035734226689, + "grad_norm": 3.015625, + "learning_rate": 9.169764560099133e-07, + "loss": 1.0796822547912597, + "mean_token_accuracy": 0.7774909734725952, + "num_tokens": 2239557.0, + "step": 860 + }, + { + "entropy": 1.1237488869577645, + "epoch": 0.9715242881072027, + "grad_norm": 3.265625, + "learning_rate": 6.691449814126394e-07, + "loss": 1.2529041290283203, + "mean_token_accuracy": 0.7569354966282844, + "num_tokens": 2266609.0, + "step": 870 + }, + { + "entropy": 1.02969014570117, + "epoch": 0.9826912339475153, + "grad_norm": 3.265625, + "learning_rate": 4.213135068153656e-07, + "loss": 1.1265899658203125, + "mean_token_accuracy": 0.7792067192494869, + "num_tokens": 2291191.0, + "step": 880 + }, + { + "entropy": 1.0750355511903762, + "epoch": 0.993858179787828, + "grad_norm": 3.453125, + "learning_rate": 1.7348203221809172e-07, + "loss": 1.143712043762207, + "mean_token_accuracy": 0.7723280422389507, + "num_tokens": 2316691.0, + "step": 890 + } + ], + "logging_steps": 10, + "max_steps": 896, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.4891590857728e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-896/training_args.bin b/checkpoint-896/training_args.bin new file mode 100644 index 0000000..79b7383 --- /dev/null +++ b/checkpoint-896/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b155d610518979b16cb5dabd7c7fc4b2c6efcd2cc727476b697ee4228298492 +size 5713 diff --git a/config.json b/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/data_manifest.json b/data_manifest.json new file mode 100644 index 0000000..56dcf08 --- /dev/null +++ b/data_manifest.json @@ -0,0 +1,84 @@ +{ + "total_train": 6396, + "raw_before_dedup": 6396, + "config": { + "model": { + "id": "Qwen/Qwen3-1.7B", + "cache_dir": "/scratch/hf_cache" + }, + "system_prompt": "You are a knowledgeable assistant. For multiple-choice questions, reason step by step, then write only the letter of the correct answer inside \\boxed{}.\n", + "data": { + "sources": [ + "/scratch/data/gk_jennifer_knowledge_v6.jsonl" + ], + "max_samples": null, + "sampling_strategy": "source_balanced", + "max_seq_length": 4096, + "max_think_chars": 0, + "seed": 7 + }, + "training": { + "output_dir": "/scratch/checkpoints/gk_thinking_v6_knowledge_base", + "num_train_epochs": 2, + "learning_rate": 7e-06, + "per_device_train_batch_size": 2, + "gradient_accumulation_steps": 8, + "warmup_ratio": 0.05, + "seed": 7, + "bf16": true, + "logging_steps": 10, + "save_strategy": "epoch", + "gradient_checkpointing": true + }, + "generation": { + "do_sample": true, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643 + } + }, + "source_counts": { + "cot_distill/jennifer/other": 201, + "cot_distill/jennifer/business": 91, + "cot_distill/jennifer/mmlu": 4602, + "cot_distill/jennifer/biology": 140, + "cot_distill/jennifer/engineering": 49, + "cot_distill/jennifer/economics": 203, + "cot_distill/jennifer/math": 196, + "cot_distill/jennifer/health": 170, + "cot_distill/jennifer/physics": 118, + "cot_distill/jennifer/law": 150, + "cot_distill/jennifer/psychology": 161, + "cot_distill/jennifer/computer science": 60, + "cot_distill/jennifer/history": 56, + "cot_distill/jennifer/philosophy": 93, + "cot_distill/jennifer/chemistry": 106 + }, + "answer_letter_counts": { + "A": 1364, + "B": 1414, + "C": 1456, + "D": 1185, + "E": 383, + "F": 115, + "G": 101, + "H": 80, + "I": 76, + "J": 50, + "K": 29, + "L": 36, + "M": 32, + "N": 19, + "O": 16, + "P": 9, + "Q": 9, + "R": 13, + "S": 5, + "T": 4 + } +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..af2a8dc --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..1064856 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bac0341b3ac0211a9d714f8033b8cc41540a2e5883ada65da043db89fa5c1de +size 3441185608 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..79984c8 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dff8e567c2d962375693a532e5305e36f6a51c2b316d666c005b3bfe2d70466 +size 5777