From 7c4ed61457bdf02778817d4aaa42c26b3fc8d506 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 13 Jun 2026 09:58:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: cs-552-2026-MMRF/general_knowledge_model Source: Original Platform --- .gitattributes | 49 + README.md | 66 ++ chat_template.jinja | 89 ++ checkpoint-1000/chat_template.jinja | 89 ++ checkpoint-1000/config.json | 63 ++ checkpoint-1000/generation_config.json | 12 + checkpoint-1000/model.safetensors | 3 + checkpoint-1000/optimizer.pt | 3 + checkpoint-1000/rng_state.pth | 3 + checkpoint-1000/scheduler.pt | 3 + checkpoint-1000/tokenizer.json | 3 + checkpoint-1000/tokenizer_config.json | 30 + checkpoint-1000/trainer_state.json | 234 +++++ checkpoint-1000/training_args.bin | 3 + checkpoint-1500/chat_template.jinja | 89 ++ checkpoint-1500/config.json | 63 ++ checkpoint-1500/generation_config.json | 12 + checkpoint-1500/model.safetensors | 3 + checkpoint-1500/optimizer.pt | 3 + checkpoint-1500/rng_state.pth | 3 + checkpoint-1500/scheduler.pt | 3 + checkpoint-1500/tokenizer.json | 3 + checkpoint-1500/tokenizer_config.json | 30 + checkpoint-1500/trainer_state.json | 334 +++++++ checkpoint-1500/training_args.bin | 3 + checkpoint-2000/chat_template.jinja | 89 ++ checkpoint-2000/config.json | 63 ++ checkpoint-2000/generation_config.json | 12 + checkpoint-2000/model.safetensors | 3 + checkpoint-2000/optimizer.pt | 3 + checkpoint-2000/rng_state.pth | 3 + checkpoint-2000/scheduler.pt | 3 + checkpoint-2000/tokenizer.json | 3 + checkpoint-2000/tokenizer_config.json | 30 + checkpoint-2000/trainer_state.json | 434 ++++++++ checkpoint-2000/training_args.bin | 3 + checkpoint-2500/chat_template.jinja | 89 ++ checkpoint-2500/config.json | 63 ++ checkpoint-2500/generation_config.json | 12 + checkpoint-2500/model.safetensors | 3 + checkpoint-2500/optimizer.pt | 3 + checkpoint-2500/rng_state.pth | 3 + checkpoint-2500/scheduler.pt | 3 + checkpoint-2500/tokenizer.json | 3 + checkpoint-2500/tokenizer_config.json | 30 + checkpoint-2500/trainer_state.json | 534 ++++++++++ checkpoint-2500/training_args.bin | 3 + checkpoint-3000/chat_template.jinja | 89 ++ checkpoint-3000/config.json | 63 ++ checkpoint-3000/generation_config.json | 12 + checkpoint-3000/model.safetensors | 3 + checkpoint-3000/optimizer.pt | 3 + checkpoint-3000/rng_state.pth | 3 + checkpoint-3000/scheduler.pt | 3 + checkpoint-3000/tokenizer.json | 3 + checkpoint-3000/tokenizer_config.json | 30 + checkpoint-3000/trainer_state.json | 634 ++++++++++++ checkpoint-3000/training_args.bin | 3 + checkpoint-3500/chat_template.jinja | 89 ++ checkpoint-3500/config.json | 63 ++ checkpoint-3500/generation_config.json | 12 + checkpoint-3500/model.safetensors | 3 + checkpoint-3500/optimizer.pt | 3 + checkpoint-3500/rng_state.pth | 3 + checkpoint-3500/scheduler.pt | 3 + checkpoint-3500/tokenizer.json | 3 + checkpoint-3500/tokenizer_config.json | 30 + checkpoint-3500/trainer_state.json | 734 ++++++++++++++ checkpoint-3500/training_args.bin | 3 + checkpoint-4000/chat_template.jinja | 89 ++ checkpoint-4000/config.json | 63 ++ checkpoint-4000/generation_config.json | 12 + checkpoint-4000/model.safetensors | 3 + checkpoint-4000/optimizer.pt | 3 + checkpoint-4000/rng_state.pth | 3 + checkpoint-4000/scheduler.pt | 3 + checkpoint-4000/tokenizer.json | 3 + checkpoint-4000/tokenizer_config.json | 30 + checkpoint-4000/trainer_state.json | 834 ++++++++++++++++ checkpoint-4000/training_args.bin | 3 + checkpoint-4500/chat_template.jinja | 89 ++ checkpoint-4500/config.json | 63 ++ checkpoint-4500/generation_config.json | 12 + checkpoint-4500/model.safetensors | 3 + checkpoint-4500/optimizer.pt | 3 + checkpoint-4500/rng_state.pth | 3 + checkpoint-4500/scheduler.pt | 3 + checkpoint-4500/tokenizer.json | 3 + checkpoint-4500/tokenizer_config.json | 30 + checkpoint-4500/trainer_state.json | 934 +++++++++++++++++ checkpoint-4500/training_args.bin | 3 + checkpoint-500/chat_template.jinja | 89 ++ checkpoint-500/config.json | 63 ++ checkpoint-500/generation_config.json | 12 + checkpoint-500/model.safetensors | 3 + checkpoint-500/optimizer.pt | 3 + checkpoint-500/rng_state.pth | 3 + checkpoint-500/scheduler.pt | 3 + checkpoint-500/tokenizer.json | 3 + checkpoint-500/tokenizer_config.json | 30 + checkpoint-500/trainer_state.json | 134 +++ checkpoint-500/training_args.bin | 3 + checkpoint-5000/chat_template.jinja | 89 ++ checkpoint-5000/config.json | 63 ++ checkpoint-5000/generation_config.json | 12 + checkpoint-5000/model.safetensors | 3 + checkpoint-5000/optimizer.pt | 3 + checkpoint-5000/rng_state.pth | 3 + checkpoint-5000/scheduler.pt | 3 + checkpoint-5000/tokenizer.json | 3 + checkpoint-5000/tokenizer_config.json | 30 + checkpoint-5000/trainer_state.json | 1034 +++++++++++++++++++ checkpoint-5000/training_args.bin | 3 + checkpoint-5500/chat_template.jinja | 89 ++ checkpoint-5500/config.json | 63 ++ checkpoint-5500/generation_config.json | 12 + checkpoint-5500/model.safetensors | 3 + checkpoint-5500/optimizer.pt | 3 + checkpoint-5500/rng_state.pth | 3 + checkpoint-5500/scheduler.pt | 3 + checkpoint-5500/tokenizer.json | 3 + checkpoint-5500/tokenizer_config.json | 30 + checkpoint-5500/trainer_state.json | 1134 +++++++++++++++++++++ checkpoint-5500/training_args.bin | 3 + checkpoint-6000/chat_template.jinja | 89 ++ checkpoint-6000/config.json | 63 ++ checkpoint-6000/generation_config.json | 12 + checkpoint-6000/model.safetensors | 3 + checkpoint-6000/optimizer.pt | 3 + checkpoint-6000/rng_state.pth | 3 + checkpoint-6000/scheduler.pt | 3 + checkpoint-6000/tokenizer.json | 3 + checkpoint-6000/tokenizer_config.json | 30 + checkpoint-6000/trainer_state.json | 1234 +++++++++++++++++++++++ checkpoint-6000/training_args.bin | 3 + checkpoint-6241/chat_template.jinja | 89 ++ checkpoint-6241/config.json | 63 ++ checkpoint-6241/generation_config.json | 12 + checkpoint-6241/model.safetensors | 3 + checkpoint-6241/optimizer.pt | 3 + checkpoint-6241/rng_state.pth | 3 + checkpoint-6241/scheduler.pt | 3 + checkpoint-6241/tokenizer.json | 3 + checkpoint-6241/tokenizer_config.json | 30 + checkpoint-6241/trainer_state.json | 1274 ++++++++++++++++++++++++ checkpoint-6241/training_args.bin | 3 + config.json | 63 ++ generation_config.json | 13 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + training_args.bin | 3 + 152 files changed, 12557 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-1000/chat_template.jinja create mode 100644 checkpoint-1000/config.json create mode 100644 checkpoint-1000/generation_config.json create mode 100644 checkpoint-1000/model.safetensors create mode 100644 checkpoint-1000/optimizer.pt create mode 100644 checkpoint-1000/rng_state.pth create mode 100644 checkpoint-1000/scheduler.pt create mode 100644 checkpoint-1000/tokenizer.json create mode 100644 checkpoint-1000/tokenizer_config.json create mode 100644 checkpoint-1000/trainer_state.json create mode 100644 checkpoint-1000/training_args.bin create mode 100644 checkpoint-1500/chat_template.jinja create mode 100644 checkpoint-1500/config.json create mode 100644 checkpoint-1500/generation_config.json create mode 100644 checkpoint-1500/model.safetensors create mode 100644 checkpoint-1500/optimizer.pt create mode 100644 checkpoint-1500/rng_state.pth create mode 100644 checkpoint-1500/scheduler.pt create mode 100644 checkpoint-1500/tokenizer.json create mode 100644 checkpoint-1500/tokenizer_config.json create mode 100644 checkpoint-1500/trainer_state.json create mode 100644 checkpoint-1500/training_args.bin create mode 100644 checkpoint-2000/chat_template.jinja create mode 100644 checkpoint-2000/config.json create mode 100644 checkpoint-2000/generation_config.json create mode 100644 checkpoint-2000/model.safetensors create mode 100644 checkpoint-2000/optimizer.pt create mode 100644 checkpoint-2000/rng_state.pth create mode 100644 checkpoint-2000/scheduler.pt create mode 100644 checkpoint-2000/tokenizer.json create mode 100644 checkpoint-2000/tokenizer_config.json create mode 100644 checkpoint-2000/trainer_state.json create mode 100644 checkpoint-2000/training_args.bin create mode 100644 checkpoint-2500/chat_template.jinja create mode 100644 checkpoint-2500/config.json create mode 100644 checkpoint-2500/generation_config.json create mode 100644 checkpoint-2500/model.safetensors create mode 100644 checkpoint-2500/optimizer.pt create mode 100644 checkpoint-2500/rng_state.pth create mode 100644 checkpoint-2500/scheduler.pt create mode 100644 checkpoint-2500/tokenizer.json create mode 100644 checkpoint-2500/tokenizer_config.json create mode 100644 checkpoint-2500/trainer_state.json create mode 100644 checkpoint-2500/training_args.bin create mode 100644 checkpoint-3000/chat_template.jinja create mode 100644 checkpoint-3000/config.json create mode 100644 checkpoint-3000/generation_config.json create mode 100644 checkpoint-3000/model.safetensors create mode 100644 checkpoint-3000/optimizer.pt create mode 100644 checkpoint-3000/rng_state.pth create mode 100644 checkpoint-3000/scheduler.pt create mode 100644 checkpoint-3000/tokenizer.json create mode 100644 checkpoint-3000/tokenizer_config.json create mode 100644 checkpoint-3000/trainer_state.json create mode 100644 checkpoint-3000/training_args.bin create mode 100644 checkpoint-3500/chat_template.jinja create mode 100644 checkpoint-3500/config.json create mode 100644 checkpoint-3500/generation_config.json create mode 100644 checkpoint-3500/model.safetensors create mode 100644 checkpoint-3500/optimizer.pt create mode 100644 checkpoint-3500/rng_state.pth create mode 100644 checkpoint-3500/scheduler.pt create mode 100644 checkpoint-3500/tokenizer.json create mode 100644 checkpoint-3500/tokenizer_config.json create mode 100644 checkpoint-3500/trainer_state.json create mode 100644 checkpoint-3500/training_args.bin create mode 100644 checkpoint-4000/chat_template.jinja create mode 100644 checkpoint-4000/config.json create mode 100644 checkpoint-4000/generation_config.json create mode 100644 checkpoint-4000/model.safetensors create mode 100644 checkpoint-4000/optimizer.pt create mode 100644 checkpoint-4000/rng_state.pth create mode 100644 checkpoint-4000/scheduler.pt create mode 100644 checkpoint-4000/tokenizer.json create mode 100644 checkpoint-4000/tokenizer_config.json create mode 100644 checkpoint-4000/trainer_state.json create mode 100644 checkpoint-4000/training_args.bin create mode 100644 checkpoint-4500/chat_template.jinja create mode 100644 checkpoint-4500/config.json create mode 100644 checkpoint-4500/generation_config.json create mode 100644 checkpoint-4500/model.safetensors create mode 100644 checkpoint-4500/optimizer.pt create mode 100644 checkpoint-4500/rng_state.pth create mode 100644 checkpoint-4500/scheduler.pt create mode 100644 checkpoint-4500/tokenizer.json create mode 100644 checkpoint-4500/tokenizer_config.json create mode 100644 checkpoint-4500/trainer_state.json create mode 100644 checkpoint-4500/training_args.bin create mode 100644 checkpoint-500/chat_template.jinja create mode 100644 checkpoint-500/config.json create mode 100644 checkpoint-500/generation_config.json create mode 100644 checkpoint-500/model.safetensors create mode 100644 checkpoint-500/optimizer.pt create mode 100644 checkpoint-500/rng_state.pth create mode 100644 checkpoint-500/scheduler.pt create mode 100644 checkpoint-500/tokenizer.json create mode 100644 checkpoint-500/tokenizer_config.json create mode 100644 checkpoint-500/trainer_state.json create mode 100644 checkpoint-500/training_args.bin create mode 100644 checkpoint-5000/chat_template.jinja create mode 100644 checkpoint-5000/config.json create mode 100644 checkpoint-5000/generation_config.json create mode 100644 checkpoint-5000/model.safetensors create mode 100644 checkpoint-5000/optimizer.pt create mode 100644 checkpoint-5000/rng_state.pth create mode 100644 checkpoint-5000/scheduler.pt create mode 100644 checkpoint-5000/tokenizer.json create mode 100644 checkpoint-5000/tokenizer_config.json create mode 100644 checkpoint-5000/trainer_state.json create mode 100644 checkpoint-5000/training_args.bin create mode 100644 checkpoint-5500/chat_template.jinja create mode 100644 checkpoint-5500/config.json create mode 100644 checkpoint-5500/generation_config.json create mode 100644 checkpoint-5500/model.safetensors create mode 100644 checkpoint-5500/optimizer.pt create mode 100644 checkpoint-5500/rng_state.pth create mode 100644 checkpoint-5500/scheduler.pt create mode 100644 checkpoint-5500/tokenizer.json create mode 100644 checkpoint-5500/tokenizer_config.json create mode 100644 checkpoint-5500/trainer_state.json create mode 100644 checkpoint-5500/training_args.bin create mode 100644 checkpoint-6000/chat_template.jinja create mode 100644 checkpoint-6000/config.json create mode 100644 checkpoint-6000/generation_config.json create mode 100644 checkpoint-6000/model.safetensors create mode 100644 checkpoint-6000/optimizer.pt create mode 100644 checkpoint-6000/rng_state.pth create mode 100644 checkpoint-6000/scheduler.pt create mode 100644 checkpoint-6000/tokenizer.json create mode 100644 checkpoint-6000/tokenizer_config.json create mode 100644 checkpoint-6000/trainer_state.json create mode 100644 checkpoint-6000/training_args.bin create mode 100644 checkpoint-6241/chat_template.jinja create mode 100644 checkpoint-6241/config.json create mode 100644 checkpoint-6241/generation_config.json create mode 100644 checkpoint-6241/model.safetensors create mode 100644 checkpoint-6241/optimizer.pt create mode 100644 checkpoint-6241/rng_state.pth create mode 100644 checkpoint-6241/scheduler.pt create mode 100644 checkpoint-6241/tokenizer.json create mode 100644 checkpoint-6241/tokenizer_config.json create mode 100644 checkpoint-6241/trainer_state.json create mode 100644 checkpoint-6241/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e10e872 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,49 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-3500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-4000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-4500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-5500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-6000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-6241/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b858ad0 --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +--- +library_name: transformers +model_name: general_knowledge_model_v8 +tags: +- generated_from_trainer +- grpo +- trl +licence: license +--- + +# Model Card for general_knowledge_model_v8 + +This model is a fine-tuned version of [None](https://huggingface.co/None). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300). + +### Framework versions + +- TRL: 1.3.0 +- Transformers: 5.7.0 +- Pytorch: 2.10.0+cu128 +- Datasets: 4.8.5 +- Tokenizers: 0.22.2 + +## Citations + +Cite GRPO as: + +```bibtex +@article{shao2024deepseekmath, + title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}}, + author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo}, + year = 2024, + eprint = {arXiv:2402.03300}, +} +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1000/chat_template.jinja b/checkpoint-1000/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-1000/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1000/config.json b/checkpoint-1000/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-1000/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-1000/generation_config.json b/checkpoint-1000/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-1000/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-1000/model.safetensors b/checkpoint-1000/model.safetensors new file mode 100644 index 0000000..e8a65e5 --- /dev/null +++ b/checkpoint-1000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa6a5ffa325a42521faee189eed67b3922773291e206e209e90b3c770b4d4539 +size 3441185608 diff --git a/checkpoint-1000/optimizer.pt b/checkpoint-1000/optimizer.pt new file mode 100644 index 0000000..f40d49e --- /dev/null +++ b/checkpoint-1000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c51b9e4e783c1596ac7d185b6f7589123a938675998d0d32a5f077dfa2dfefd +size 6882572207 diff --git a/checkpoint-1000/rng_state.pth b/checkpoint-1000/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-1000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-1000/scheduler.pt b/checkpoint-1000/scheduler.pt new file mode 100644 index 0000000..97d98d5 --- /dev/null +++ b/checkpoint-1000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03766410d0ee20761a88c2327709b696e608f6be1dc35ba4d82dcb2eea2325f7 +size 1465 diff --git a/checkpoint-1000/tokenizer.json b/checkpoint-1000/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-1000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-1000/tokenizer_config.json b/checkpoint-1000/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-1000/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1000/trainer_state.json b/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000..5654c8a --- /dev/null +++ b/checkpoint-1000/trainer_state.json @@ -0,0 +1,234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.16024998998437562, + "eval_steps": 500, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.095551986099814e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1000/training_args.bin b/checkpoint-1000/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-1000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-1500/chat_template.jinja b/checkpoint-1500/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-1500/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1500/config.json b/checkpoint-1500/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-1500/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-1500/generation_config.json b/checkpoint-1500/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-1500/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-1500/model.safetensors b/checkpoint-1500/model.safetensors new file mode 100644 index 0000000..a549be5 --- /dev/null +++ b/checkpoint-1500/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:438e34183f5e94f608c7669e0b2cb247069e6a47ebcded96167ec3afb7af8101 +size 3441185608 diff --git a/checkpoint-1500/optimizer.pt b/checkpoint-1500/optimizer.pt new file mode 100644 index 0000000..9f3c9c6 --- /dev/null +++ b/checkpoint-1500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11e561d4c48540c45fb0712b82d5ffdd92c408f08ae4ed12ecbced4e409b1208 +size 6882572207 diff --git a/checkpoint-1500/rng_state.pth b/checkpoint-1500/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-1500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-1500/scheduler.pt b/checkpoint-1500/scheduler.pt new file mode 100644 index 0000000..d35c63e --- /dev/null +++ b/checkpoint-1500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b666bb023db840fb022cfa0357e113ba9d208c4303ad969fafd6098567ba3038 +size 1465 diff --git a/checkpoint-1500/tokenizer.json b/checkpoint-1500/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-1500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-1500/tokenizer_config.json b/checkpoint-1500/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-1500/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1500/trainer_state.json b/checkpoint-1500/trainer_state.json new file mode 100644 index 0000000..829cf4d --- /dev/null +++ b/checkpoint-1500/trainer_state.json @@ -0,0 +1,334 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.24037498497656343, + "eval_steps": 500, + "global_step": 1500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.064051705174999e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1500/training_args.bin b/checkpoint-1500/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-1500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-2000/chat_template.jinja b/checkpoint-2000/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-2000/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-2000/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-2000/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-2000/model.safetensors b/checkpoint-2000/model.safetensors new file mode 100644 index 0000000..809c23b --- /dev/null +++ b/checkpoint-2000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec88ee383178456b8f8ab7ee942be2785b0a32656809f229cae94c76dff1b810 +size 3441185608 diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt new file mode 100644 index 0000000..cc0c799 --- /dev/null +++ b/checkpoint-2000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea2dc818dc38c52000eeebc5b101562caf8fb11d4790f976bb066ee636e4f14d +size 6882572207 diff --git a/checkpoint-2000/rng_state.pth b/checkpoint-2000/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-2000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000..9187318 --- /dev/null +++ b/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11b128c7c709b8b18f78ca21ff87ee25957a3215823c4227b4ba298165c8d222 +size 1465 diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000..9db5338 --- /dev/null +++ b/checkpoint-2000/trainer_state.json @@ -0,0 +1,434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.32049997996875124, + "eval_steps": 500, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4174735647666176e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-2500/chat_template.jinja b/checkpoint-2500/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-2500/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-2500/config.json b/checkpoint-2500/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-2500/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2500/generation_config.json b/checkpoint-2500/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-2500/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-2500/model.safetensors b/checkpoint-2500/model.safetensors new file mode 100644 index 0000000..533aff5 --- /dev/null +++ b/checkpoint-2500/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98a04b91a11ec1c0b526f91d12bdfb28ff4c3c69e104271b9e28a2f75f44f891 +size 3441185608 diff --git a/checkpoint-2500/optimizer.pt b/checkpoint-2500/optimizer.pt new file mode 100644 index 0000000..6526fb0 --- /dev/null +++ b/checkpoint-2500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7535b338ca59a9c376634a6fbbef04a836c5c21962e653fe7794832854643965 +size 6882572207 diff --git a/checkpoint-2500/rng_state.pth b/checkpoint-2500/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-2500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-2500/scheduler.pt b/checkpoint-2500/scheduler.pt new file mode 100644 index 0000000..659045e --- /dev/null +++ b/checkpoint-2500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c863cc66645dc4964c4c4f2cf30406fef13cbea85c7c7bd73c333d72670c821e +size 1465 diff --git a/checkpoint-2500/tokenizer.json b/checkpoint-2500/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-2500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-2500/tokenizer_config.json b/checkpoint-2500/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-2500/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2500/trainer_state.json b/checkpoint-2500/trainer_state.json new file mode 100644 index 0000000..38511de --- /dev/null +++ b/checkpoint-2500/trainer_state.json @@ -0,0 +1,534 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4006249749609391, + "eval_steps": 500, + "global_step": 2500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.773461452668027e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2500/training_args.bin b/checkpoint-2500/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-2500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-3000/chat_template.jinja b/checkpoint-3000/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-3000/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-3000/config.json b/checkpoint-3000/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-3000/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-3000/generation_config.json b/checkpoint-3000/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-3000/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-3000/model.safetensors b/checkpoint-3000/model.safetensors new file mode 100644 index 0000000..ff6cf74 --- /dev/null +++ b/checkpoint-3000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5b72e5752b081abdc1994e5b2076200e5122c26cf9578d62c85a820f8ff24de +size 3441185608 diff --git a/checkpoint-3000/optimizer.pt b/checkpoint-3000/optimizer.pt new file mode 100644 index 0000000..85ae37c --- /dev/null +++ b/checkpoint-3000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34d6d5e6b3baf98ae75f9122a60622751f2ca0f8eb5d5542a1fd00ed368520c2 +size 6882572207 diff --git a/checkpoint-3000/rng_state.pth b/checkpoint-3000/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-3000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-3000/scheduler.pt b/checkpoint-3000/scheduler.pt new file mode 100644 index 0000000..a884e1b --- /dev/null +++ b/checkpoint-3000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:875525fc96d4b4bb5c12817e379ae11ed73b09ebd4f315c5475a43de7d437801 +size 1465 diff --git a/checkpoint-3000/tokenizer.json b/checkpoint-3000/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-3000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-3000/tokenizer_config.json b/checkpoint-3000/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-3000/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-3000/trainer_state.json b/checkpoint-3000/trainer_state.json new file mode 100644 index 0000000..10b401c --- /dev/null +++ b/checkpoint-3000/trainer_state.json @@ -0,0 +1,634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.48074996995312685, + "eval_steps": 500, + "global_step": 3000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.126267005433856e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-3000/training_args.bin b/checkpoint-3000/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-3000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-3500/chat_template.jinja b/checkpoint-3500/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-3500/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-3500/config.json b/checkpoint-3500/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-3500/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-3500/generation_config.json b/checkpoint-3500/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-3500/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-3500/model.safetensors b/checkpoint-3500/model.safetensors new file mode 100644 index 0000000..60920a5 --- /dev/null +++ b/checkpoint-3500/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd3ed08012c528778e23bfc40ce9263d1183db5fe1b344fcc375190ceff69329 +size 3441185608 diff --git a/checkpoint-3500/optimizer.pt b/checkpoint-3500/optimizer.pt new file mode 100644 index 0000000..eef7695 --- /dev/null +++ b/checkpoint-3500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff2d89dd80a7f07fe151122ae98c1d25ce93198c70301c9848ea4314b9f22671 +size 6882572207 diff --git a/checkpoint-3500/rng_state.pth b/checkpoint-3500/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-3500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-3500/scheduler.pt b/checkpoint-3500/scheduler.pt new file mode 100644 index 0000000..b2087fa --- /dev/null +++ b/checkpoint-3500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e847bb7d94a2f7987ac5f9be2becb727ffced83ae79de46ff613b03a251110e +size 1465 diff --git a/checkpoint-3500/tokenizer.json b/checkpoint-3500/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-3500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-3500/tokenizer_config.json b/checkpoint-3500/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-3500/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-3500/trainer_state.json b/checkpoint-3500/trainer_state.json new file mode 100644 index 0000000..bac80ac --- /dev/null +++ b/checkpoint-3500/trainer_state.json @@ -0,0 +1,734 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5608749649453146, + "eval_steps": 500, + "global_step": 3500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + }, + { + "entropy": 2.0951381689310073, + "epoch": 0.48876246945234564, + "grad_norm": 3.234375, + "learning_rate": 1.0229129947123859e-05, + "loss": 2.108144073486328, + "mean_token_accuracy": 0.5269971822202205, + "num_tokens": 18293136.0, + "step": 3050 + }, + { + "entropy": 2.116977259516716, + "epoch": 0.4967749689515644, + "grad_norm": 3.421875, + "learning_rate": 1.0068899214869413e-05, + "loss": 2.1327830505371095, + "mean_token_accuracy": 0.5238270902633667, + "num_tokens": 18591481.0, + "step": 3100 + }, + { + "entropy": 2.08598925113678, + "epoch": 0.5047874684507833, + "grad_norm": 2.96875, + "learning_rate": 9.908668482614967e-06, + "loss": 2.1032344055175782, + "mean_token_accuracy": 0.5289211909472943, + "num_tokens": 18885221.0, + "step": 3150 + }, + { + "entropy": 2.097712662220001, + "epoch": 0.512799967950002, + "grad_norm": 3.40625, + "learning_rate": 9.74843775036052e-06, + "loss": 2.1150633239746095, + "mean_token_accuracy": 0.5288272814452648, + "num_tokens": 19184784.0, + "step": 3200 + }, + { + "entropy": 2.1063760107755662, + "epoch": 0.5208124674492208, + "grad_norm": 3.28125, + "learning_rate": 9.588207018106075e-06, + "loss": 2.1217109680175783, + "mean_token_accuracy": 0.526290989369154, + "num_tokens": 19482538.0, + "step": 3250 + }, + { + "entropy": 2.090182801485062, + "epoch": 0.5288249669484396, + "grad_norm": 2.875, + "learning_rate": 9.427976285851627e-06, + "loss": 2.1130311584472654, + "mean_token_accuracy": 0.5285439921915531, + "num_tokens": 19782544.0, + "step": 3300 + }, + { + "entropy": 2.103260806798935, + "epoch": 0.5368374664476584, + "grad_norm": 3.59375, + "learning_rate": 9.267745553597181e-06, + "loss": 2.116189727783203, + "mean_token_accuracy": 0.5273195558786392, + "num_tokens": 20086408.0, + "step": 3350 + }, + { + "entropy": 2.0673747235536575, + "epoch": 0.5448499659468772, + "grad_norm": 3.125, + "learning_rate": 9.107514821342735e-06, + "loss": 2.087482147216797, + "mean_token_accuracy": 0.532562478929758, + "num_tokens": 20380429.0, + "step": 3400 + }, + { + "entropy": 2.1030789083242416, + "epoch": 0.552862465446096, + "grad_norm": 3.296875, + "learning_rate": 8.947284089088289e-06, + "loss": 2.122591247558594, + "mean_token_accuracy": 0.5256703786551953, + "num_tokens": 20682849.0, + "step": 3450 + }, + { + "entropy": 2.0988716661930082, + "epoch": 0.5608749649453146, + "grad_norm": 3.546875, + "learning_rate": 8.787053356833841e-06, + "loss": 2.115867156982422, + "mean_token_accuracy": 0.5266730943322182, + "num_tokens": 20979225.0, + "step": 3500 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.4793245607162675e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-3500/training_args.bin b/checkpoint-3500/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-3500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-4000/chat_template.jinja b/checkpoint-4000/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-4000/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-4000/config.json b/checkpoint-4000/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-4000/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-4000/generation_config.json b/checkpoint-4000/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-4000/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-4000/model.safetensors b/checkpoint-4000/model.safetensors new file mode 100644 index 0000000..d3a746c --- /dev/null +++ b/checkpoint-4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f52f23dfe48c7547727ea25697ac3a9d18e4d016c8fa4c5c9e1b4dff8c31cb +size 3441185608 diff --git a/checkpoint-4000/optimizer.pt b/checkpoint-4000/optimizer.pt new file mode 100644 index 0000000..3903484 --- /dev/null +++ b/checkpoint-4000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c313769ae550920040bf3021b0ea64cd98c9f52d74cedfbdb319bf5c5369138 +size 6882572207 diff --git a/checkpoint-4000/rng_state.pth b/checkpoint-4000/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-4000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-4000/scheduler.pt b/checkpoint-4000/scheduler.pt new file mode 100644 index 0000000..03f8dd3 --- /dev/null +++ b/checkpoint-4000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78d0e196db341b3518b3d3e05e16be9cc71c297b4603d00fde73052aafae3e28 +size 1465 diff --git a/checkpoint-4000/tokenizer.json b/checkpoint-4000/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-4000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-4000/tokenizer_config.json b/checkpoint-4000/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-4000/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-4000/trainer_state.json b/checkpoint-4000/trainer_state.json new file mode 100644 index 0000000..7069506 --- /dev/null +++ b/checkpoint-4000/trainer_state.json @@ -0,0 +1,834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6409999599375025, + "eval_steps": 500, + "global_step": 4000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + }, + { + "entropy": 2.0951381689310073, + "epoch": 0.48876246945234564, + "grad_norm": 3.234375, + "learning_rate": 1.0229129947123859e-05, + "loss": 2.108144073486328, + "mean_token_accuracy": 0.5269971822202205, + "num_tokens": 18293136.0, + "step": 3050 + }, + { + "entropy": 2.116977259516716, + "epoch": 0.4967749689515644, + "grad_norm": 3.421875, + "learning_rate": 1.0068899214869413e-05, + "loss": 2.1327830505371095, + "mean_token_accuracy": 0.5238270902633667, + "num_tokens": 18591481.0, + "step": 3100 + }, + { + "entropy": 2.08598925113678, + "epoch": 0.5047874684507833, + "grad_norm": 2.96875, + "learning_rate": 9.908668482614967e-06, + "loss": 2.1032344055175782, + "mean_token_accuracy": 0.5289211909472943, + "num_tokens": 18885221.0, + "step": 3150 + }, + { + "entropy": 2.097712662220001, + "epoch": 0.512799967950002, + "grad_norm": 3.40625, + "learning_rate": 9.74843775036052e-06, + "loss": 2.1150633239746095, + "mean_token_accuracy": 0.5288272814452648, + "num_tokens": 19184784.0, + "step": 3200 + }, + { + "entropy": 2.1063760107755662, + "epoch": 0.5208124674492208, + "grad_norm": 3.28125, + "learning_rate": 9.588207018106075e-06, + "loss": 2.1217109680175783, + "mean_token_accuracy": 0.526290989369154, + "num_tokens": 19482538.0, + "step": 3250 + }, + { + "entropy": 2.090182801485062, + "epoch": 0.5288249669484396, + "grad_norm": 2.875, + "learning_rate": 9.427976285851627e-06, + "loss": 2.1130311584472654, + "mean_token_accuracy": 0.5285439921915531, + "num_tokens": 19782544.0, + "step": 3300 + }, + { + "entropy": 2.103260806798935, + "epoch": 0.5368374664476584, + "grad_norm": 3.59375, + "learning_rate": 9.267745553597181e-06, + "loss": 2.116189727783203, + "mean_token_accuracy": 0.5273195558786392, + "num_tokens": 20086408.0, + "step": 3350 + }, + { + "entropy": 2.0673747235536575, + "epoch": 0.5448499659468772, + "grad_norm": 3.125, + "learning_rate": 9.107514821342735e-06, + "loss": 2.087482147216797, + "mean_token_accuracy": 0.532562478929758, + "num_tokens": 20380429.0, + "step": 3400 + }, + { + "entropy": 2.1030789083242416, + "epoch": 0.552862465446096, + "grad_norm": 3.296875, + "learning_rate": 8.947284089088289e-06, + "loss": 2.122591247558594, + "mean_token_accuracy": 0.5256703786551953, + "num_tokens": 20682849.0, + "step": 3450 + }, + { + "entropy": 2.0988716661930082, + "epoch": 0.5608749649453146, + "grad_norm": 3.546875, + "learning_rate": 8.787053356833841e-06, + "loss": 2.115867156982422, + "mean_token_accuracy": 0.5266730943322182, + "num_tokens": 20979225.0, + "step": 3500 + }, + { + "entropy": 2.0954072701931, + "epoch": 0.5688874644445334, + "grad_norm": 3.859375, + "learning_rate": 8.626822624579395e-06, + "loss": 2.1116868591308595, + "mean_token_accuracy": 0.5267495161294937, + "num_tokens": 21284166.0, + "step": 3550 + }, + { + "entropy": 2.11238471031189, + "epoch": 0.5768999639437522, + "grad_norm": 3.484375, + "learning_rate": 8.466591892324949e-06, + "loss": 2.1292352294921875, + "mean_token_accuracy": 0.5249243360757828, + "num_tokens": 21584612.0, + "step": 3600 + }, + { + "entropy": 2.091464368700981, + "epoch": 0.584912463442971, + "grad_norm": 3.359375, + "learning_rate": 8.306361160070501e-06, + "loss": 2.1099319458007812, + "mean_token_accuracy": 0.5292035435140133, + "num_tokens": 21879345.0, + "step": 3650 + }, + { + "entropy": 2.095204224586487, + "epoch": 0.5929249629421898, + "grad_norm": 2.96875, + "learning_rate": 8.146130427816055e-06, + "loss": 2.1198716735839844, + "mean_token_accuracy": 0.5255991271138192, + "num_tokens": 22183486.0, + "step": 3700 + }, + { + "entropy": 2.0666573679447175, + "epoch": 0.6009374624414086, + "grad_norm": 2.90625, + "learning_rate": 7.98589969556161e-06, + "loss": 2.08689697265625, + "mean_token_accuracy": 0.5318664258718491, + "num_tokens": 22476438.0, + "step": 3750 + }, + { + "entropy": 2.122153177857399, + "epoch": 0.6089499619406273, + "grad_norm": 3.40625, + "learning_rate": 7.825668963307162e-06, + "loss": 2.138288879394531, + "mean_token_accuracy": 0.5232584895193577, + "num_tokens": 22780385.0, + "step": 3800 + }, + { + "entropy": 2.1190748488903046, + "epoch": 0.6169624614398461, + "grad_norm": 3.46875, + "learning_rate": 7.665438231052716e-06, + "loss": 2.139790344238281, + "mean_token_accuracy": 0.5243287613987923, + "num_tokens": 23080847.0, + "step": 3850 + }, + { + "entropy": 2.1312570214271545, + "epoch": 0.6249749609390649, + "grad_norm": 3.765625, + "learning_rate": 7.5052074987982706e-06, + "loss": 2.1471080017089843, + "mean_token_accuracy": 0.5232046774029732, + "num_tokens": 23384738.0, + "step": 3900 + }, + { + "entropy": 2.0779008239507677, + "epoch": 0.6329874604382837, + "grad_norm": 3.171875, + "learning_rate": 7.344976766543823e-06, + "loss": 2.0974729919433592, + "mean_token_accuracy": 0.5305448272824287, + "num_tokens": 23682750.0, + "step": 3950 + }, + { + "entropy": 2.1121455317735673, + "epoch": 0.6409999599375025, + "grad_norm": 3.921875, + "learning_rate": 7.184746034289377e-06, + "loss": 2.127069549560547, + "mean_token_accuracy": 0.5252936625480652, + "num_tokens": 23989029.0, + "step": 4000 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.832566128574546e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-4000/training_args.bin b/checkpoint-4000/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-4000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-4500/chat_template.jinja b/checkpoint-4500/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-4500/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-4500/config.json b/checkpoint-4500/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-4500/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-4500/generation_config.json b/checkpoint-4500/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-4500/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-4500/model.safetensors b/checkpoint-4500/model.safetensors new file mode 100644 index 0000000..9cff9c1 --- /dev/null +++ b/checkpoint-4500/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:748cbfd96fce4131cccae2cfbbf7e5943bb67a7960feaffb683370fbb897f84b +size 3441185608 diff --git a/checkpoint-4500/optimizer.pt b/checkpoint-4500/optimizer.pt new file mode 100644 index 0000000..9e49881 --- /dev/null +++ b/checkpoint-4500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:504fdf18c65054e63b9b27fd2ef1b64e410a2d62984669334f0bbfee95f68807 +size 6882572207 diff --git a/checkpoint-4500/rng_state.pth b/checkpoint-4500/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-4500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-4500/scheduler.pt b/checkpoint-4500/scheduler.pt new file mode 100644 index 0000000..65bfe7f --- /dev/null +++ b/checkpoint-4500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0c65d41abe4fed1f917d1493ac9e89afd04deb14307279675e66aa4431576fd +size 1465 diff --git a/checkpoint-4500/tokenizer.json b/checkpoint-4500/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-4500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-4500/tokenizer_config.json b/checkpoint-4500/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-4500/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-4500/trainer_state.json b/checkpoint-4500/trainer_state.json new file mode 100644 index 0000000..704a9da --- /dev/null +++ b/checkpoint-4500/trainer_state.json @@ -0,0 +1,934 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7211249549296903, + "eval_steps": 500, + "global_step": 4500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + }, + { + "entropy": 2.0951381689310073, + "epoch": 0.48876246945234564, + "grad_norm": 3.234375, + "learning_rate": 1.0229129947123859e-05, + "loss": 2.108144073486328, + "mean_token_accuracy": 0.5269971822202205, + "num_tokens": 18293136.0, + "step": 3050 + }, + { + "entropy": 2.116977259516716, + "epoch": 0.4967749689515644, + "grad_norm": 3.421875, + "learning_rate": 1.0068899214869413e-05, + "loss": 2.1327830505371095, + "mean_token_accuracy": 0.5238270902633667, + "num_tokens": 18591481.0, + "step": 3100 + }, + { + "entropy": 2.08598925113678, + "epoch": 0.5047874684507833, + "grad_norm": 2.96875, + "learning_rate": 9.908668482614967e-06, + "loss": 2.1032344055175782, + "mean_token_accuracy": 0.5289211909472943, + "num_tokens": 18885221.0, + "step": 3150 + }, + { + "entropy": 2.097712662220001, + "epoch": 0.512799967950002, + "grad_norm": 3.40625, + "learning_rate": 9.74843775036052e-06, + "loss": 2.1150633239746095, + "mean_token_accuracy": 0.5288272814452648, + "num_tokens": 19184784.0, + "step": 3200 + }, + { + "entropy": 2.1063760107755662, + "epoch": 0.5208124674492208, + "grad_norm": 3.28125, + "learning_rate": 9.588207018106075e-06, + "loss": 2.1217109680175783, + "mean_token_accuracy": 0.526290989369154, + "num_tokens": 19482538.0, + "step": 3250 + }, + { + "entropy": 2.090182801485062, + "epoch": 0.5288249669484396, + "grad_norm": 2.875, + "learning_rate": 9.427976285851627e-06, + "loss": 2.1130311584472654, + "mean_token_accuracy": 0.5285439921915531, + "num_tokens": 19782544.0, + "step": 3300 + }, + { + "entropy": 2.103260806798935, + "epoch": 0.5368374664476584, + "grad_norm": 3.59375, + "learning_rate": 9.267745553597181e-06, + "loss": 2.116189727783203, + "mean_token_accuracy": 0.5273195558786392, + "num_tokens": 20086408.0, + "step": 3350 + }, + { + "entropy": 2.0673747235536575, + "epoch": 0.5448499659468772, + "grad_norm": 3.125, + "learning_rate": 9.107514821342735e-06, + "loss": 2.087482147216797, + "mean_token_accuracy": 0.532562478929758, + "num_tokens": 20380429.0, + "step": 3400 + }, + { + "entropy": 2.1030789083242416, + "epoch": 0.552862465446096, + "grad_norm": 3.296875, + "learning_rate": 8.947284089088289e-06, + "loss": 2.122591247558594, + "mean_token_accuracy": 0.5256703786551953, + "num_tokens": 20682849.0, + "step": 3450 + }, + { + "entropy": 2.0988716661930082, + "epoch": 0.5608749649453146, + "grad_norm": 3.546875, + "learning_rate": 8.787053356833841e-06, + "loss": 2.115867156982422, + "mean_token_accuracy": 0.5266730943322182, + "num_tokens": 20979225.0, + "step": 3500 + }, + { + "entropy": 2.0954072701931, + "epoch": 0.5688874644445334, + "grad_norm": 3.859375, + "learning_rate": 8.626822624579395e-06, + "loss": 2.1116868591308595, + "mean_token_accuracy": 0.5267495161294937, + "num_tokens": 21284166.0, + "step": 3550 + }, + { + "entropy": 2.11238471031189, + "epoch": 0.5768999639437522, + "grad_norm": 3.484375, + "learning_rate": 8.466591892324949e-06, + "loss": 2.1292352294921875, + "mean_token_accuracy": 0.5249243360757828, + "num_tokens": 21584612.0, + "step": 3600 + }, + { + "entropy": 2.091464368700981, + "epoch": 0.584912463442971, + "grad_norm": 3.359375, + "learning_rate": 8.306361160070501e-06, + "loss": 2.1099319458007812, + "mean_token_accuracy": 0.5292035435140133, + "num_tokens": 21879345.0, + "step": 3650 + }, + { + "entropy": 2.095204224586487, + "epoch": 0.5929249629421898, + "grad_norm": 2.96875, + "learning_rate": 8.146130427816055e-06, + "loss": 2.1198716735839844, + "mean_token_accuracy": 0.5255991271138192, + "num_tokens": 22183486.0, + "step": 3700 + }, + { + "entropy": 2.0666573679447175, + "epoch": 0.6009374624414086, + "grad_norm": 2.90625, + "learning_rate": 7.98589969556161e-06, + "loss": 2.08689697265625, + "mean_token_accuracy": 0.5318664258718491, + "num_tokens": 22476438.0, + "step": 3750 + }, + { + "entropy": 2.122153177857399, + "epoch": 0.6089499619406273, + "grad_norm": 3.40625, + "learning_rate": 7.825668963307162e-06, + "loss": 2.138288879394531, + "mean_token_accuracy": 0.5232584895193577, + "num_tokens": 22780385.0, + "step": 3800 + }, + { + "entropy": 2.1190748488903046, + "epoch": 0.6169624614398461, + "grad_norm": 3.46875, + "learning_rate": 7.665438231052716e-06, + "loss": 2.139790344238281, + "mean_token_accuracy": 0.5243287613987923, + "num_tokens": 23080847.0, + "step": 3850 + }, + { + "entropy": 2.1312570214271545, + "epoch": 0.6249749609390649, + "grad_norm": 3.765625, + "learning_rate": 7.5052074987982706e-06, + "loss": 2.1471080017089843, + "mean_token_accuracy": 0.5232046774029732, + "num_tokens": 23384738.0, + "step": 3900 + }, + { + "entropy": 2.0779008239507677, + "epoch": 0.6329874604382837, + "grad_norm": 3.171875, + "learning_rate": 7.344976766543823e-06, + "loss": 2.0974729919433592, + "mean_token_accuracy": 0.5305448272824287, + "num_tokens": 23682750.0, + "step": 3950 + }, + { + "entropy": 2.1121455317735673, + "epoch": 0.6409999599375025, + "grad_norm": 3.921875, + "learning_rate": 7.184746034289377e-06, + "loss": 2.127069549560547, + "mean_token_accuracy": 0.5252936625480652, + "num_tokens": 23989029.0, + "step": 4000 + }, + { + "entropy": 2.0982888478040693, + "epoch": 0.6490124594367213, + "grad_norm": 4.0625, + "learning_rate": 7.024515302034931e-06, + "loss": 2.112042236328125, + "mean_token_accuracy": 0.5281323495507241, + "num_tokens": 24292063.0, + "step": 4050 + }, + { + "entropy": 2.098363963365555, + "epoch": 0.65702495893594, + "grad_norm": 3.046875, + "learning_rate": 6.864284569780485e-06, + "loss": 2.1156489562988283, + "mean_token_accuracy": 0.5275120075047016, + "num_tokens": 24590201.0, + "step": 4100 + }, + { + "entropy": 2.1065632277727127, + "epoch": 0.6650374584351588, + "grad_norm": 3.375, + "learning_rate": 6.704053837526038e-06, + "loss": 2.12911376953125, + "mean_token_accuracy": 0.5262684133648873, + "num_tokens": 24892122.0, + "step": 4150 + }, + { + "entropy": 2.1014072370529173, + "epoch": 0.6730499579343776, + "grad_norm": 2.84375, + "learning_rate": 6.543823105271592e-06, + "loss": 2.1130335998535155, + "mean_token_accuracy": 0.5262750512361527, + "num_tokens": 25194969.0, + "step": 4200 + }, + { + "entropy": 2.0905236744880678, + "epoch": 0.6810624574335964, + "grad_norm": 3.03125, + "learning_rate": 6.383592373017146e-06, + "loss": 2.1153533935546873, + "mean_token_accuracy": 0.527137616276741, + "num_tokens": 25499521.0, + "step": 4250 + }, + { + "entropy": 2.1032627063989637, + "epoch": 0.6890749569328152, + "grad_norm": 3.125, + "learning_rate": 6.223361640762698e-06, + "loss": 2.122831115722656, + "mean_token_accuracy": 0.5254572249948979, + "num_tokens": 25798933.0, + "step": 4300 + }, + { + "entropy": 2.118139310479164, + "epoch": 0.697087456432034, + "grad_norm": 3.28125, + "learning_rate": 6.063130908508252e-06, + "loss": 2.1376751708984374, + "mean_token_accuracy": 0.5243131402134895, + "num_tokens": 26096619.0, + "step": 4350 + }, + { + "entropy": 2.082554380297661, + "epoch": 0.7050999559312527, + "grad_norm": 3.390625, + "learning_rate": 5.902900176253806e-06, + "loss": 2.0949493408203126, + "mean_token_accuracy": 0.5316410692036152, + "num_tokens": 26394395.0, + "step": 4400 + }, + { + "entropy": 2.09870474755764, + "epoch": 0.7131124554304715, + "grad_norm": 3.421875, + "learning_rate": 5.7426694439993594e-06, + "loss": 2.1184346008300783, + "mean_token_accuracy": 0.5267573061585427, + "num_tokens": 26692333.0, + "step": 4450 + }, + { + "entropy": 2.0866885662078856, + "epoch": 0.7211249549296903, + "grad_norm": 3.71875, + "learning_rate": 5.5824387117449134e-06, + "loss": 2.102904968261719, + "mean_token_accuracy": 0.529726449996233, + "num_tokens": 26989384.0, + "step": 4500 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.1866411651588096e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-4500/training_args.bin b/checkpoint-4500/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-4500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-500/chat_template.jinja b/checkpoint-500/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-500/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-500/config.json b/checkpoint-500/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-500/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-500/generation_config.json b/checkpoint-500/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-500/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-500/model.safetensors b/checkpoint-500/model.safetensors new file mode 100644 index 0000000..cb8297b --- /dev/null +++ b/checkpoint-500/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:510bdc2405d027342acd0e2047289831efc0b3f25febcd5c74681c16385dd5c2 +size 3441185608 diff --git a/checkpoint-500/optimizer.pt b/checkpoint-500/optimizer.pt new file mode 100644 index 0000000..0e23027 --- /dev/null +++ b/checkpoint-500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3466f40535b1f6a6dd5a3dfad0962141622b90ac955b344d5278efbb20cd2ba1 +size 6882572207 diff --git a/checkpoint-500/rng_state.pth b/checkpoint-500/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-500/scheduler.pt b/checkpoint-500/scheduler.pt new file mode 100644 index 0000000..dc233f2 --- /dev/null +++ b/checkpoint-500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e36e50838b6f150abbf126d12ba88d9ca9bcc1b4d6927bad9696043424255e41 +size 1465 diff --git a/checkpoint-500/tokenizer.json b/checkpoint-500/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-500/tokenizer_config.json b/checkpoint-500/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-500/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-500/trainer_state.json b/checkpoint-500/trainer_state.json new file mode 100644 index 0000000..47a0332 --- /dev/null +++ b/checkpoint-500/trainer_state.json @@ -0,0 +1,134 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.08012499499218781, + "eval_steps": 500, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.560511422243635e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-500/training_args.bin b/checkpoint-500/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-5000/chat_template.jinja b/checkpoint-5000/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-5000/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-5000/config.json b/checkpoint-5000/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-5000/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-5000/generation_config.json b/checkpoint-5000/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-5000/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-5000/model.safetensors b/checkpoint-5000/model.safetensors new file mode 100644 index 0000000..591746f --- /dev/null +++ b/checkpoint-5000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9210c0f8c0427681f33f67f527b1ec49a79f01625fbc1759118f2ecc59421a63 +size 3441185608 diff --git a/checkpoint-5000/optimizer.pt b/checkpoint-5000/optimizer.pt new file mode 100644 index 0000000..8c34497 --- /dev/null +++ b/checkpoint-5000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0640613439df62a0358096494db2b4d610bf4a9169660130ece88bebfa231026 +size 6882572207 diff --git a/checkpoint-5000/rng_state.pth b/checkpoint-5000/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-5000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-5000/scheduler.pt b/checkpoint-5000/scheduler.pt new file mode 100644 index 0000000..f10b6dc --- /dev/null +++ b/checkpoint-5000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25aa8e41a989b4b3c58d4ee4aed8e685dad485c899acf803301071d8faa75034 +size 1465 diff --git a/checkpoint-5000/tokenizer.json b/checkpoint-5000/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-5000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-5000/tokenizer_config.json b/checkpoint-5000/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-5000/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-5000/trainer_state.json b/checkpoint-5000/trainer_state.json new file mode 100644 index 0000000..43cbf87 --- /dev/null +++ b/checkpoint-5000/trainer_state.json @@ -0,0 +1,1034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8012499499218781, + "eval_steps": 500, + "global_step": 5000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + }, + { + "entropy": 2.0951381689310073, + "epoch": 0.48876246945234564, + "grad_norm": 3.234375, + "learning_rate": 1.0229129947123859e-05, + "loss": 2.108144073486328, + "mean_token_accuracy": 0.5269971822202205, + "num_tokens": 18293136.0, + "step": 3050 + }, + { + "entropy": 2.116977259516716, + "epoch": 0.4967749689515644, + "grad_norm": 3.421875, + "learning_rate": 1.0068899214869413e-05, + "loss": 2.1327830505371095, + "mean_token_accuracy": 0.5238270902633667, + "num_tokens": 18591481.0, + "step": 3100 + }, + { + "entropy": 2.08598925113678, + "epoch": 0.5047874684507833, + "grad_norm": 2.96875, + "learning_rate": 9.908668482614967e-06, + "loss": 2.1032344055175782, + "mean_token_accuracy": 0.5289211909472943, + "num_tokens": 18885221.0, + "step": 3150 + }, + { + "entropy": 2.097712662220001, + "epoch": 0.512799967950002, + "grad_norm": 3.40625, + "learning_rate": 9.74843775036052e-06, + "loss": 2.1150633239746095, + "mean_token_accuracy": 0.5288272814452648, + "num_tokens": 19184784.0, + "step": 3200 + }, + { + "entropy": 2.1063760107755662, + "epoch": 0.5208124674492208, + "grad_norm": 3.28125, + "learning_rate": 9.588207018106075e-06, + "loss": 2.1217109680175783, + "mean_token_accuracy": 0.526290989369154, + "num_tokens": 19482538.0, + "step": 3250 + }, + { + "entropy": 2.090182801485062, + "epoch": 0.5288249669484396, + "grad_norm": 2.875, + "learning_rate": 9.427976285851627e-06, + "loss": 2.1130311584472654, + "mean_token_accuracy": 0.5285439921915531, + "num_tokens": 19782544.0, + "step": 3300 + }, + { + "entropy": 2.103260806798935, + "epoch": 0.5368374664476584, + "grad_norm": 3.59375, + "learning_rate": 9.267745553597181e-06, + "loss": 2.116189727783203, + "mean_token_accuracy": 0.5273195558786392, + "num_tokens": 20086408.0, + "step": 3350 + }, + { + "entropy": 2.0673747235536575, + "epoch": 0.5448499659468772, + "grad_norm": 3.125, + "learning_rate": 9.107514821342735e-06, + "loss": 2.087482147216797, + "mean_token_accuracy": 0.532562478929758, + "num_tokens": 20380429.0, + "step": 3400 + }, + { + "entropy": 2.1030789083242416, + "epoch": 0.552862465446096, + "grad_norm": 3.296875, + "learning_rate": 8.947284089088289e-06, + "loss": 2.122591247558594, + "mean_token_accuracy": 0.5256703786551953, + "num_tokens": 20682849.0, + "step": 3450 + }, + { + "entropy": 2.0988716661930082, + "epoch": 0.5608749649453146, + "grad_norm": 3.546875, + "learning_rate": 8.787053356833841e-06, + "loss": 2.115867156982422, + "mean_token_accuracy": 0.5266730943322182, + "num_tokens": 20979225.0, + "step": 3500 + }, + { + "entropy": 2.0954072701931, + "epoch": 0.5688874644445334, + "grad_norm": 3.859375, + "learning_rate": 8.626822624579395e-06, + "loss": 2.1116868591308595, + "mean_token_accuracy": 0.5267495161294937, + "num_tokens": 21284166.0, + "step": 3550 + }, + { + "entropy": 2.11238471031189, + "epoch": 0.5768999639437522, + "grad_norm": 3.484375, + "learning_rate": 8.466591892324949e-06, + "loss": 2.1292352294921875, + "mean_token_accuracy": 0.5249243360757828, + "num_tokens": 21584612.0, + "step": 3600 + }, + { + "entropy": 2.091464368700981, + "epoch": 0.584912463442971, + "grad_norm": 3.359375, + "learning_rate": 8.306361160070501e-06, + "loss": 2.1099319458007812, + "mean_token_accuracy": 0.5292035435140133, + "num_tokens": 21879345.0, + "step": 3650 + }, + { + "entropy": 2.095204224586487, + "epoch": 0.5929249629421898, + "grad_norm": 2.96875, + "learning_rate": 8.146130427816055e-06, + "loss": 2.1198716735839844, + "mean_token_accuracy": 0.5255991271138192, + "num_tokens": 22183486.0, + "step": 3700 + }, + { + "entropy": 2.0666573679447175, + "epoch": 0.6009374624414086, + "grad_norm": 2.90625, + "learning_rate": 7.98589969556161e-06, + "loss": 2.08689697265625, + "mean_token_accuracy": 0.5318664258718491, + "num_tokens": 22476438.0, + "step": 3750 + }, + { + "entropy": 2.122153177857399, + "epoch": 0.6089499619406273, + "grad_norm": 3.40625, + "learning_rate": 7.825668963307162e-06, + "loss": 2.138288879394531, + "mean_token_accuracy": 0.5232584895193577, + "num_tokens": 22780385.0, + "step": 3800 + }, + { + "entropy": 2.1190748488903046, + "epoch": 0.6169624614398461, + "grad_norm": 3.46875, + "learning_rate": 7.665438231052716e-06, + "loss": 2.139790344238281, + "mean_token_accuracy": 0.5243287613987923, + "num_tokens": 23080847.0, + "step": 3850 + }, + { + "entropy": 2.1312570214271545, + "epoch": 0.6249749609390649, + "grad_norm": 3.765625, + "learning_rate": 7.5052074987982706e-06, + "loss": 2.1471080017089843, + "mean_token_accuracy": 0.5232046774029732, + "num_tokens": 23384738.0, + "step": 3900 + }, + { + "entropy": 2.0779008239507677, + "epoch": 0.6329874604382837, + "grad_norm": 3.171875, + "learning_rate": 7.344976766543823e-06, + "loss": 2.0974729919433592, + "mean_token_accuracy": 0.5305448272824287, + "num_tokens": 23682750.0, + "step": 3950 + }, + { + "entropy": 2.1121455317735673, + "epoch": 0.6409999599375025, + "grad_norm": 3.921875, + "learning_rate": 7.184746034289377e-06, + "loss": 2.127069549560547, + "mean_token_accuracy": 0.5252936625480652, + "num_tokens": 23989029.0, + "step": 4000 + }, + { + "entropy": 2.0982888478040693, + "epoch": 0.6490124594367213, + "grad_norm": 4.0625, + "learning_rate": 7.024515302034931e-06, + "loss": 2.112042236328125, + "mean_token_accuracy": 0.5281323495507241, + "num_tokens": 24292063.0, + "step": 4050 + }, + { + "entropy": 2.098363963365555, + "epoch": 0.65702495893594, + "grad_norm": 3.046875, + "learning_rate": 6.864284569780485e-06, + "loss": 2.1156489562988283, + "mean_token_accuracy": 0.5275120075047016, + "num_tokens": 24590201.0, + "step": 4100 + }, + { + "entropy": 2.1065632277727127, + "epoch": 0.6650374584351588, + "grad_norm": 3.375, + "learning_rate": 6.704053837526038e-06, + "loss": 2.12911376953125, + "mean_token_accuracy": 0.5262684133648873, + "num_tokens": 24892122.0, + "step": 4150 + }, + { + "entropy": 2.1014072370529173, + "epoch": 0.6730499579343776, + "grad_norm": 2.84375, + "learning_rate": 6.543823105271592e-06, + "loss": 2.1130335998535155, + "mean_token_accuracy": 0.5262750512361527, + "num_tokens": 25194969.0, + "step": 4200 + }, + { + "entropy": 2.0905236744880678, + "epoch": 0.6810624574335964, + "grad_norm": 3.03125, + "learning_rate": 6.383592373017146e-06, + "loss": 2.1153533935546873, + "mean_token_accuracy": 0.527137616276741, + "num_tokens": 25499521.0, + "step": 4250 + }, + { + "entropy": 2.1032627063989637, + "epoch": 0.6890749569328152, + "grad_norm": 3.125, + "learning_rate": 6.223361640762698e-06, + "loss": 2.122831115722656, + "mean_token_accuracy": 0.5254572249948979, + "num_tokens": 25798933.0, + "step": 4300 + }, + { + "entropy": 2.118139310479164, + "epoch": 0.697087456432034, + "grad_norm": 3.28125, + "learning_rate": 6.063130908508252e-06, + "loss": 2.1376751708984374, + "mean_token_accuracy": 0.5243131402134895, + "num_tokens": 26096619.0, + "step": 4350 + }, + { + "entropy": 2.082554380297661, + "epoch": 0.7050999559312527, + "grad_norm": 3.390625, + "learning_rate": 5.902900176253806e-06, + "loss": 2.0949493408203126, + "mean_token_accuracy": 0.5316410692036152, + "num_tokens": 26394395.0, + "step": 4400 + }, + { + "entropy": 2.09870474755764, + "epoch": 0.7131124554304715, + "grad_norm": 3.421875, + "learning_rate": 5.7426694439993594e-06, + "loss": 2.1184346008300783, + "mean_token_accuracy": 0.5267573061585427, + "num_tokens": 26692333.0, + "step": 4450 + }, + { + "entropy": 2.0866885662078856, + "epoch": 0.7211249549296903, + "grad_norm": 3.71875, + "learning_rate": 5.5824387117449134e-06, + "loss": 2.102904968261719, + "mean_token_accuracy": 0.529726449996233, + "num_tokens": 26989384.0, + "step": 4500 + }, + { + "entropy": 2.114828346967697, + "epoch": 0.7291374544289091, + "grad_norm": 3.46875, + "learning_rate": 5.4222079794904666e-06, + "loss": 2.128611602783203, + "mean_token_accuracy": 0.5247649565339089, + "num_tokens": 27299927.0, + "step": 4550 + }, + { + "entropy": 2.09617226600647, + "epoch": 0.7371499539281279, + "grad_norm": 3.015625, + "learning_rate": 5.26197724723602e-06, + "loss": 2.1166041564941405, + "mean_token_accuracy": 0.5273530712723732, + "num_tokens": 27599011.0, + "step": 4600 + }, + { + "entropy": 2.0975282275676728, + "epoch": 0.7451624534273467, + "grad_norm": 3.3125, + "learning_rate": 5.101746514981574e-06, + "loss": 2.112121124267578, + "mean_token_accuracy": 0.5281847220659256, + "num_tokens": 27892355.0, + "step": 4650 + }, + { + "entropy": 2.095783769488335, + "epoch": 0.7531749529265654, + "grad_norm": 3.78125, + "learning_rate": 4.941515782727128e-06, + "loss": 2.110701904296875, + "mean_token_accuracy": 0.5272985179722309, + "num_tokens": 28187615.0, + "step": 4700 + }, + { + "entropy": 2.0964103603363036, + "epoch": 0.7611874524257842, + "grad_norm": 3.1875, + "learning_rate": 4.781285050472681e-06, + "loss": 2.11992919921875, + "mean_token_accuracy": 0.5271916574239731, + "num_tokens": 28485419.0, + "step": 4750 + }, + { + "entropy": 2.0820577853918074, + "epoch": 0.769199951925003, + "grad_norm": 3.5, + "learning_rate": 4.621054318218234e-06, + "loss": 2.095073394775391, + "mean_token_accuracy": 0.5297581899166107, + "num_tokens": 28780158.0, + "step": 4800 + }, + { + "entropy": 2.089591929912567, + "epoch": 0.7772124514242218, + "grad_norm": 3.125, + "learning_rate": 4.460823585963788e-06, + "loss": 2.109657897949219, + "mean_token_accuracy": 0.528961935043335, + "num_tokens": 29081052.0, + "step": 4850 + }, + { + "entropy": 2.092900555729866, + "epoch": 0.7852249509234406, + "grad_norm": 3.46875, + "learning_rate": 4.300592853709342e-06, + "loss": 2.106436920166016, + "mean_token_accuracy": 0.5284854583442211, + "num_tokens": 29376324.0, + "step": 4900 + }, + { + "entropy": 2.0630873823165894, + "epoch": 0.7932374504226594, + "grad_norm": 3.28125, + "learning_rate": 4.140362121454896e-06, + "loss": 2.088908233642578, + "mean_token_accuracy": 0.532561699450016, + "num_tokens": 29675269.0, + "step": 4950 + }, + { + "entropy": 2.0753238475322724, + "epoch": 0.8012499499218781, + "grad_norm": 3.21875, + "learning_rate": 3.980131389200449e-06, + "loss": 2.0961412048339843, + "mean_token_accuracy": 0.5304154875874519, + "num_tokens": 29974838.0, + "step": 5000 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.540934716676915e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-5000/training_args.bin b/checkpoint-5000/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-5000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-5500/chat_template.jinja b/checkpoint-5500/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-5500/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-5500/config.json b/checkpoint-5500/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-5500/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-5500/generation_config.json b/checkpoint-5500/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-5500/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-5500/model.safetensors b/checkpoint-5500/model.safetensors new file mode 100644 index 0000000..364fd7b --- /dev/null +++ b/checkpoint-5500/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2ea18394260e36468ba955f0edf3aed5e86e05cfddad9dd67132e0c90d1659f +size 3441185608 diff --git a/checkpoint-5500/optimizer.pt b/checkpoint-5500/optimizer.pt new file mode 100644 index 0000000..67abe68 --- /dev/null +++ b/checkpoint-5500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf82561ff68c22f9de236f8f7a595bd320f78bca6143d43d6339be6cf6d5f3e +size 6882572207 diff --git a/checkpoint-5500/rng_state.pth b/checkpoint-5500/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-5500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-5500/scheduler.pt b/checkpoint-5500/scheduler.pt new file mode 100644 index 0000000..e1440b6 --- /dev/null +++ b/checkpoint-5500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8240268d8934f8e0ca422959548ab4fb30a8e1118fb691e38334f307db3923e8 +size 1465 diff --git a/checkpoint-5500/tokenizer.json b/checkpoint-5500/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-5500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-5500/tokenizer_config.json b/checkpoint-5500/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-5500/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-5500/trainer_state.json b/checkpoint-5500/trainer_state.json new file mode 100644 index 0000000..3c73eab --- /dev/null +++ b/checkpoint-5500/trainer_state.json @@ -0,0 +1,1134 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.881374944914066, + "eval_steps": 500, + "global_step": 5500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + }, + { + "entropy": 2.0951381689310073, + "epoch": 0.48876246945234564, + "grad_norm": 3.234375, + "learning_rate": 1.0229129947123859e-05, + "loss": 2.108144073486328, + "mean_token_accuracy": 0.5269971822202205, + "num_tokens": 18293136.0, + "step": 3050 + }, + { + "entropy": 2.116977259516716, + "epoch": 0.4967749689515644, + "grad_norm": 3.421875, + "learning_rate": 1.0068899214869413e-05, + "loss": 2.1327830505371095, + "mean_token_accuracy": 0.5238270902633667, + "num_tokens": 18591481.0, + "step": 3100 + }, + { + "entropy": 2.08598925113678, + "epoch": 0.5047874684507833, + "grad_norm": 2.96875, + "learning_rate": 9.908668482614967e-06, + "loss": 2.1032344055175782, + "mean_token_accuracy": 0.5289211909472943, + "num_tokens": 18885221.0, + "step": 3150 + }, + { + "entropy": 2.097712662220001, + "epoch": 0.512799967950002, + "grad_norm": 3.40625, + "learning_rate": 9.74843775036052e-06, + "loss": 2.1150633239746095, + "mean_token_accuracy": 0.5288272814452648, + "num_tokens": 19184784.0, + "step": 3200 + }, + { + "entropy": 2.1063760107755662, + "epoch": 0.5208124674492208, + "grad_norm": 3.28125, + "learning_rate": 9.588207018106075e-06, + "loss": 2.1217109680175783, + "mean_token_accuracy": 0.526290989369154, + "num_tokens": 19482538.0, + "step": 3250 + }, + { + "entropy": 2.090182801485062, + "epoch": 0.5288249669484396, + "grad_norm": 2.875, + "learning_rate": 9.427976285851627e-06, + "loss": 2.1130311584472654, + "mean_token_accuracy": 0.5285439921915531, + "num_tokens": 19782544.0, + "step": 3300 + }, + { + "entropy": 2.103260806798935, + "epoch": 0.5368374664476584, + "grad_norm": 3.59375, + "learning_rate": 9.267745553597181e-06, + "loss": 2.116189727783203, + "mean_token_accuracy": 0.5273195558786392, + "num_tokens": 20086408.0, + "step": 3350 + }, + { + "entropy": 2.0673747235536575, + "epoch": 0.5448499659468772, + "grad_norm": 3.125, + "learning_rate": 9.107514821342735e-06, + "loss": 2.087482147216797, + "mean_token_accuracy": 0.532562478929758, + "num_tokens": 20380429.0, + "step": 3400 + }, + { + "entropy": 2.1030789083242416, + "epoch": 0.552862465446096, + "grad_norm": 3.296875, + "learning_rate": 8.947284089088289e-06, + "loss": 2.122591247558594, + "mean_token_accuracy": 0.5256703786551953, + "num_tokens": 20682849.0, + "step": 3450 + }, + { + "entropy": 2.0988716661930082, + "epoch": 0.5608749649453146, + "grad_norm": 3.546875, + "learning_rate": 8.787053356833841e-06, + "loss": 2.115867156982422, + "mean_token_accuracy": 0.5266730943322182, + "num_tokens": 20979225.0, + "step": 3500 + }, + { + "entropy": 2.0954072701931, + "epoch": 0.5688874644445334, + "grad_norm": 3.859375, + "learning_rate": 8.626822624579395e-06, + "loss": 2.1116868591308595, + "mean_token_accuracy": 0.5267495161294937, + "num_tokens": 21284166.0, + "step": 3550 + }, + { + "entropy": 2.11238471031189, + "epoch": 0.5768999639437522, + "grad_norm": 3.484375, + "learning_rate": 8.466591892324949e-06, + "loss": 2.1292352294921875, + "mean_token_accuracy": 0.5249243360757828, + "num_tokens": 21584612.0, + "step": 3600 + }, + { + "entropy": 2.091464368700981, + "epoch": 0.584912463442971, + "grad_norm": 3.359375, + "learning_rate": 8.306361160070501e-06, + "loss": 2.1099319458007812, + "mean_token_accuracy": 0.5292035435140133, + "num_tokens": 21879345.0, + "step": 3650 + }, + { + "entropy": 2.095204224586487, + "epoch": 0.5929249629421898, + "grad_norm": 2.96875, + "learning_rate": 8.146130427816055e-06, + "loss": 2.1198716735839844, + "mean_token_accuracy": 0.5255991271138192, + "num_tokens": 22183486.0, + "step": 3700 + }, + { + "entropy": 2.0666573679447175, + "epoch": 0.6009374624414086, + "grad_norm": 2.90625, + "learning_rate": 7.98589969556161e-06, + "loss": 2.08689697265625, + "mean_token_accuracy": 0.5318664258718491, + "num_tokens": 22476438.0, + "step": 3750 + }, + { + "entropy": 2.122153177857399, + "epoch": 0.6089499619406273, + "grad_norm": 3.40625, + "learning_rate": 7.825668963307162e-06, + "loss": 2.138288879394531, + "mean_token_accuracy": 0.5232584895193577, + "num_tokens": 22780385.0, + "step": 3800 + }, + { + "entropy": 2.1190748488903046, + "epoch": 0.6169624614398461, + "grad_norm": 3.46875, + "learning_rate": 7.665438231052716e-06, + "loss": 2.139790344238281, + "mean_token_accuracy": 0.5243287613987923, + "num_tokens": 23080847.0, + "step": 3850 + }, + { + "entropy": 2.1312570214271545, + "epoch": 0.6249749609390649, + "grad_norm": 3.765625, + "learning_rate": 7.5052074987982706e-06, + "loss": 2.1471080017089843, + "mean_token_accuracy": 0.5232046774029732, + "num_tokens": 23384738.0, + "step": 3900 + }, + { + "entropy": 2.0779008239507677, + "epoch": 0.6329874604382837, + "grad_norm": 3.171875, + "learning_rate": 7.344976766543823e-06, + "loss": 2.0974729919433592, + "mean_token_accuracy": 0.5305448272824287, + "num_tokens": 23682750.0, + "step": 3950 + }, + { + "entropy": 2.1121455317735673, + "epoch": 0.6409999599375025, + "grad_norm": 3.921875, + "learning_rate": 7.184746034289377e-06, + "loss": 2.127069549560547, + "mean_token_accuracy": 0.5252936625480652, + "num_tokens": 23989029.0, + "step": 4000 + }, + { + "entropy": 2.0982888478040693, + "epoch": 0.6490124594367213, + "grad_norm": 4.0625, + "learning_rate": 7.024515302034931e-06, + "loss": 2.112042236328125, + "mean_token_accuracy": 0.5281323495507241, + "num_tokens": 24292063.0, + "step": 4050 + }, + { + "entropy": 2.098363963365555, + "epoch": 0.65702495893594, + "grad_norm": 3.046875, + "learning_rate": 6.864284569780485e-06, + "loss": 2.1156489562988283, + "mean_token_accuracy": 0.5275120075047016, + "num_tokens": 24590201.0, + "step": 4100 + }, + { + "entropy": 2.1065632277727127, + "epoch": 0.6650374584351588, + "grad_norm": 3.375, + "learning_rate": 6.704053837526038e-06, + "loss": 2.12911376953125, + "mean_token_accuracy": 0.5262684133648873, + "num_tokens": 24892122.0, + "step": 4150 + }, + { + "entropy": 2.1014072370529173, + "epoch": 0.6730499579343776, + "grad_norm": 2.84375, + "learning_rate": 6.543823105271592e-06, + "loss": 2.1130335998535155, + "mean_token_accuracy": 0.5262750512361527, + "num_tokens": 25194969.0, + "step": 4200 + }, + { + "entropy": 2.0905236744880678, + "epoch": 0.6810624574335964, + "grad_norm": 3.03125, + "learning_rate": 6.383592373017146e-06, + "loss": 2.1153533935546873, + "mean_token_accuracy": 0.527137616276741, + "num_tokens": 25499521.0, + "step": 4250 + }, + { + "entropy": 2.1032627063989637, + "epoch": 0.6890749569328152, + "grad_norm": 3.125, + "learning_rate": 6.223361640762698e-06, + "loss": 2.122831115722656, + "mean_token_accuracy": 0.5254572249948979, + "num_tokens": 25798933.0, + "step": 4300 + }, + { + "entropy": 2.118139310479164, + "epoch": 0.697087456432034, + "grad_norm": 3.28125, + "learning_rate": 6.063130908508252e-06, + "loss": 2.1376751708984374, + "mean_token_accuracy": 0.5243131402134895, + "num_tokens": 26096619.0, + "step": 4350 + }, + { + "entropy": 2.082554380297661, + "epoch": 0.7050999559312527, + "grad_norm": 3.390625, + "learning_rate": 5.902900176253806e-06, + "loss": 2.0949493408203126, + "mean_token_accuracy": 0.5316410692036152, + "num_tokens": 26394395.0, + "step": 4400 + }, + { + "entropy": 2.09870474755764, + "epoch": 0.7131124554304715, + "grad_norm": 3.421875, + "learning_rate": 5.7426694439993594e-06, + "loss": 2.1184346008300783, + "mean_token_accuracy": 0.5267573061585427, + "num_tokens": 26692333.0, + "step": 4450 + }, + { + "entropy": 2.0866885662078856, + "epoch": 0.7211249549296903, + "grad_norm": 3.71875, + "learning_rate": 5.5824387117449134e-06, + "loss": 2.102904968261719, + "mean_token_accuracy": 0.529726449996233, + "num_tokens": 26989384.0, + "step": 4500 + }, + { + "entropy": 2.114828346967697, + "epoch": 0.7291374544289091, + "grad_norm": 3.46875, + "learning_rate": 5.4222079794904666e-06, + "loss": 2.128611602783203, + "mean_token_accuracy": 0.5247649565339089, + "num_tokens": 27299927.0, + "step": 4550 + }, + { + "entropy": 2.09617226600647, + "epoch": 0.7371499539281279, + "grad_norm": 3.015625, + "learning_rate": 5.26197724723602e-06, + "loss": 2.1166041564941405, + "mean_token_accuracy": 0.5273530712723732, + "num_tokens": 27599011.0, + "step": 4600 + }, + { + "entropy": 2.0975282275676728, + "epoch": 0.7451624534273467, + "grad_norm": 3.3125, + "learning_rate": 5.101746514981574e-06, + "loss": 2.112121124267578, + "mean_token_accuracy": 0.5281847220659256, + "num_tokens": 27892355.0, + "step": 4650 + }, + { + "entropy": 2.095783769488335, + "epoch": 0.7531749529265654, + "grad_norm": 3.78125, + "learning_rate": 4.941515782727128e-06, + "loss": 2.110701904296875, + "mean_token_accuracy": 0.5272985179722309, + "num_tokens": 28187615.0, + "step": 4700 + }, + { + "entropy": 2.0964103603363036, + "epoch": 0.7611874524257842, + "grad_norm": 3.1875, + "learning_rate": 4.781285050472681e-06, + "loss": 2.11992919921875, + "mean_token_accuracy": 0.5271916574239731, + "num_tokens": 28485419.0, + "step": 4750 + }, + { + "entropy": 2.0820577853918074, + "epoch": 0.769199951925003, + "grad_norm": 3.5, + "learning_rate": 4.621054318218234e-06, + "loss": 2.095073394775391, + "mean_token_accuracy": 0.5297581899166107, + "num_tokens": 28780158.0, + "step": 4800 + }, + { + "entropy": 2.089591929912567, + "epoch": 0.7772124514242218, + "grad_norm": 3.125, + "learning_rate": 4.460823585963788e-06, + "loss": 2.109657897949219, + "mean_token_accuracy": 0.528961935043335, + "num_tokens": 29081052.0, + "step": 4850 + }, + { + "entropy": 2.092900555729866, + "epoch": 0.7852249509234406, + "grad_norm": 3.46875, + "learning_rate": 4.300592853709342e-06, + "loss": 2.106436920166016, + "mean_token_accuracy": 0.5284854583442211, + "num_tokens": 29376324.0, + "step": 4900 + }, + { + "entropy": 2.0630873823165894, + "epoch": 0.7932374504226594, + "grad_norm": 3.28125, + "learning_rate": 4.140362121454896e-06, + "loss": 2.088908233642578, + "mean_token_accuracy": 0.532561699450016, + "num_tokens": 29675269.0, + "step": 4950 + }, + { + "entropy": 2.0753238475322724, + "epoch": 0.8012499499218781, + "grad_norm": 3.21875, + "learning_rate": 3.980131389200449e-06, + "loss": 2.0961412048339843, + "mean_token_accuracy": 0.5304154875874519, + "num_tokens": 29974838.0, + "step": 5000 + }, + { + "entropy": 2.123110404610634, + "epoch": 0.8092624494210969, + "grad_norm": 3.28125, + "learning_rate": 3.819900656946002e-06, + "loss": 2.1538424682617188, + "mean_token_accuracy": 0.5222072719037533, + "num_tokens": 30286201.0, + "step": 5050 + }, + { + "entropy": 2.123967355489731, + "epoch": 0.8172749489203157, + "grad_norm": 3.046875, + "learning_rate": 3.6596699246915563e-06, + "loss": 2.1417193603515625, + "mean_token_accuracy": 0.5213662980496884, + "num_tokens": 30592837.0, + "step": 5100 + }, + { + "entropy": 2.080365374684334, + "epoch": 0.8252874484195345, + "grad_norm": 3.375, + "learning_rate": 3.4994391924371094e-06, + "loss": 2.0970494079589845, + "mean_token_accuracy": 0.5296585746109486, + "num_tokens": 30890674.0, + "step": 5150 + }, + { + "entropy": 2.1183922028541566, + "epoch": 0.8332999479187533, + "grad_norm": 3.359375, + "learning_rate": 3.3392084601826634e-06, + "loss": 2.132384948730469, + "mean_token_accuracy": 0.5249108146131038, + "num_tokens": 31195601.0, + "step": 5200 + }, + { + "entropy": 2.1079335528612138, + "epoch": 0.8413124474179721, + "grad_norm": 2.890625, + "learning_rate": 3.178977727928217e-06, + "loss": 2.1310113525390624, + "mean_token_accuracy": 0.5255681444704533, + "num_tokens": 31498768.0, + "step": 5250 + }, + { + "entropy": 2.1180070728063582, + "epoch": 0.8493249469171908, + "grad_norm": 3.78125, + "learning_rate": 3.01874699567377e-06, + "loss": 2.1365049743652342, + "mean_token_accuracy": 0.5235744562745094, + "num_tokens": 31801891.0, + "step": 5300 + }, + { + "entropy": 2.0909912359714506, + "epoch": 0.8573374464164096, + "grad_norm": 3.078125, + "learning_rate": 2.858516263419324e-06, + "loss": 2.108803405761719, + "mean_token_accuracy": 0.5267959542572498, + "num_tokens": 32102627.0, + "step": 5350 + }, + { + "entropy": 2.1209339940547944, + "epoch": 0.8653499459156284, + "grad_norm": 3.140625, + "learning_rate": 2.6982855311648777e-06, + "loss": 2.143946228027344, + "mean_token_accuracy": 0.5227122528851033, + "num_tokens": 32405500.0, + "step": 5400 + }, + { + "entropy": 2.1137486273050308, + "epoch": 0.8733624454148472, + "grad_norm": 3.03125, + "learning_rate": 2.5380547989104317e-06, + "loss": 2.127569122314453, + "mean_token_accuracy": 0.5245758745074273, + "num_tokens": 32712521.0, + "step": 5450 + }, + { + "entropy": 2.0872016245126725, + "epoch": 0.881374944914066, + "grad_norm": 3.171875, + "learning_rate": 2.377824066655985e-06, + "loss": 2.100517120361328, + "mean_token_accuracy": 0.5295753349363804, + "num_tokens": 33016210.0, + "step": 5500 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.898195132722463e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-5500/training_args.bin b/checkpoint-5500/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-5500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-6000/chat_template.jinja b/checkpoint-6000/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-6000/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-6000/config.json b/checkpoint-6000/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-6000/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-6000/generation_config.json b/checkpoint-6000/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-6000/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-6000/model.safetensors b/checkpoint-6000/model.safetensors new file mode 100644 index 0000000..9414bab --- /dev/null +++ b/checkpoint-6000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaa2f97519d1276e44439361d1c0c60b06a32d06573589fa8d33e65b2a46fd18 +size 3441185608 diff --git a/checkpoint-6000/optimizer.pt b/checkpoint-6000/optimizer.pt new file mode 100644 index 0000000..cde8e28 --- /dev/null +++ b/checkpoint-6000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70cc931a62d58dd3242a68c2eb093d0a8602f0100fe0c366a73d0430de508143 +size 6882572207 diff --git a/checkpoint-6000/rng_state.pth b/checkpoint-6000/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-6000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-6000/scheduler.pt b/checkpoint-6000/scheduler.pt new file mode 100644 index 0000000..1b79d05 --- /dev/null +++ b/checkpoint-6000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ebaa185fb474ec5ec68068be2b7cdb67749a5f183dd0899caddc37f33c89035 +size 1465 diff --git a/checkpoint-6000/tokenizer.json b/checkpoint-6000/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-6000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-6000/tokenizer_config.json b/checkpoint-6000/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-6000/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-6000/trainer_state.json b/checkpoint-6000/trainer_state.json new file mode 100644 index 0000000..cacd1b1 --- /dev/null +++ b/checkpoint-6000/trainer_state.json @@ -0,0 +1,1234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9614999399062537, + "eval_steps": 500, + "global_step": 6000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + }, + { + "entropy": 2.0951381689310073, + "epoch": 0.48876246945234564, + "grad_norm": 3.234375, + "learning_rate": 1.0229129947123859e-05, + "loss": 2.108144073486328, + "mean_token_accuracy": 0.5269971822202205, + "num_tokens": 18293136.0, + "step": 3050 + }, + { + "entropy": 2.116977259516716, + "epoch": 0.4967749689515644, + "grad_norm": 3.421875, + "learning_rate": 1.0068899214869413e-05, + "loss": 2.1327830505371095, + "mean_token_accuracy": 0.5238270902633667, + "num_tokens": 18591481.0, + "step": 3100 + }, + { + "entropy": 2.08598925113678, + "epoch": 0.5047874684507833, + "grad_norm": 2.96875, + "learning_rate": 9.908668482614967e-06, + "loss": 2.1032344055175782, + "mean_token_accuracy": 0.5289211909472943, + "num_tokens": 18885221.0, + "step": 3150 + }, + { + "entropy": 2.097712662220001, + "epoch": 0.512799967950002, + "grad_norm": 3.40625, + "learning_rate": 9.74843775036052e-06, + "loss": 2.1150633239746095, + "mean_token_accuracy": 0.5288272814452648, + "num_tokens": 19184784.0, + "step": 3200 + }, + { + "entropy": 2.1063760107755662, + "epoch": 0.5208124674492208, + "grad_norm": 3.28125, + "learning_rate": 9.588207018106075e-06, + "loss": 2.1217109680175783, + "mean_token_accuracy": 0.526290989369154, + "num_tokens": 19482538.0, + "step": 3250 + }, + { + "entropy": 2.090182801485062, + "epoch": 0.5288249669484396, + "grad_norm": 2.875, + "learning_rate": 9.427976285851627e-06, + "loss": 2.1130311584472654, + "mean_token_accuracy": 0.5285439921915531, + "num_tokens": 19782544.0, + "step": 3300 + }, + { + "entropy": 2.103260806798935, + "epoch": 0.5368374664476584, + "grad_norm": 3.59375, + "learning_rate": 9.267745553597181e-06, + "loss": 2.116189727783203, + "mean_token_accuracy": 0.5273195558786392, + "num_tokens": 20086408.0, + "step": 3350 + }, + { + "entropy": 2.0673747235536575, + "epoch": 0.5448499659468772, + "grad_norm": 3.125, + "learning_rate": 9.107514821342735e-06, + "loss": 2.087482147216797, + "mean_token_accuracy": 0.532562478929758, + "num_tokens": 20380429.0, + "step": 3400 + }, + { + "entropy": 2.1030789083242416, + "epoch": 0.552862465446096, + "grad_norm": 3.296875, + "learning_rate": 8.947284089088289e-06, + "loss": 2.122591247558594, + "mean_token_accuracy": 0.5256703786551953, + "num_tokens": 20682849.0, + "step": 3450 + }, + { + "entropy": 2.0988716661930082, + "epoch": 0.5608749649453146, + "grad_norm": 3.546875, + "learning_rate": 8.787053356833841e-06, + "loss": 2.115867156982422, + "mean_token_accuracy": 0.5266730943322182, + "num_tokens": 20979225.0, + "step": 3500 + }, + { + "entropy": 2.0954072701931, + "epoch": 0.5688874644445334, + "grad_norm": 3.859375, + "learning_rate": 8.626822624579395e-06, + "loss": 2.1116868591308595, + "mean_token_accuracy": 0.5267495161294937, + "num_tokens": 21284166.0, + "step": 3550 + }, + { + "entropy": 2.11238471031189, + "epoch": 0.5768999639437522, + "grad_norm": 3.484375, + "learning_rate": 8.466591892324949e-06, + "loss": 2.1292352294921875, + "mean_token_accuracy": 0.5249243360757828, + "num_tokens": 21584612.0, + "step": 3600 + }, + { + "entropy": 2.091464368700981, + "epoch": 0.584912463442971, + "grad_norm": 3.359375, + "learning_rate": 8.306361160070501e-06, + "loss": 2.1099319458007812, + "mean_token_accuracy": 0.5292035435140133, + "num_tokens": 21879345.0, + "step": 3650 + }, + { + "entropy": 2.095204224586487, + "epoch": 0.5929249629421898, + "grad_norm": 2.96875, + "learning_rate": 8.146130427816055e-06, + "loss": 2.1198716735839844, + "mean_token_accuracy": 0.5255991271138192, + "num_tokens": 22183486.0, + "step": 3700 + }, + { + "entropy": 2.0666573679447175, + "epoch": 0.6009374624414086, + "grad_norm": 2.90625, + "learning_rate": 7.98589969556161e-06, + "loss": 2.08689697265625, + "mean_token_accuracy": 0.5318664258718491, + "num_tokens": 22476438.0, + "step": 3750 + }, + { + "entropy": 2.122153177857399, + "epoch": 0.6089499619406273, + "grad_norm": 3.40625, + "learning_rate": 7.825668963307162e-06, + "loss": 2.138288879394531, + "mean_token_accuracy": 0.5232584895193577, + "num_tokens": 22780385.0, + "step": 3800 + }, + { + "entropy": 2.1190748488903046, + "epoch": 0.6169624614398461, + "grad_norm": 3.46875, + "learning_rate": 7.665438231052716e-06, + "loss": 2.139790344238281, + "mean_token_accuracy": 0.5243287613987923, + "num_tokens": 23080847.0, + "step": 3850 + }, + { + "entropy": 2.1312570214271545, + "epoch": 0.6249749609390649, + "grad_norm": 3.765625, + "learning_rate": 7.5052074987982706e-06, + "loss": 2.1471080017089843, + "mean_token_accuracy": 0.5232046774029732, + "num_tokens": 23384738.0, + "step": 3900 + }, + { + "entropy": 2.0779008239507677, + "epoch": 0.6329874604382837, + "grad_norm": 3.171875, + "learning_rate": 7.344976766543823e-06, + "loss": 2.0974729919433592, + "mean_token_accuracy": 0.5305448272824287, + "num_tokens": 23682750.0, + "step": 3950 + }, + { + "entropy": 2.1121455317735673, + "epoch": 0.6409999599375025, + "grad_norm": 3.921875, + "learning_rate": 7.184746034289377e-06, + "loss": 2.127069549560547, + "mean_token_accuracy": 0.5252936625480652, + "num_tokens": 23989029.0, + "step": 4000 + }, + { + "entropy": 2.0982888478040693, + "epoch": 0.6490124594367213, + "grad_norm": 4.0625, + "learning_rate": 7.024515302034931e-06, + "loss": 2.112042236328125, + "mean_token_accuracy": 0.5281323495507241, + "num_tokens": 24292063.0, + "step": 4050 + }, + { + "entropy": 2.098363963365555, + "epoch": 0.65702495893594, + "grad_norm": 3.046875, + "learning_rate": 6.864284569780485e-06, + "loss": 2.1156489562988283, + "mean_token_accuracy": 0.5275120075047016, + "num_tokens": 24590201.0, + "step": 4100 + }, + { + "entropy": 2.1065632277727127, + "epoch": 0.6650374584351588, + "grad_norm": 3.375, + "learning_rate": 6.704053837526038e-06, + "loss": 2.12911376953125, + "mean_token_accuracy": 0.5262684133648873, + "num_tokens": 24892122.0, + "step": 4150 + }, + { + "entropy": 2.1014072370529173, + "epoch": 0.6730499579343776, + "grad_norm": 2.84375, + "learning_rate": 6.543823105271592e-06, + "loss": 2.1130335998535155, + "mean_token_accuracy": 0.5262750512361527, + "num_tokens": 25194969.0, + "step": 4200 + }, + { + "entropy": 2.0905236744880678, + "epoch": 0.6810624574335964, + "grad_norm": 3.03125, + "learning_rate": 6.383592373017146e-06, + "loss": 2.1153533935546873, + "mean_token_accuracy": 0.527137616276741, + "num_tokens": 25499521.0, + "step": 4250 + }, + { + "entropy": 2.1032627063989637, + "epoch": 0.6890749569328152, + "grad_norm": 3.125, + "learning_rate": 6.223361640762698e-06, + "loss": 2.122831115722656, + "mean_token_accuracy": 0.5254572249948979, + "num_tokens": 25798933.0, + "step": 4300 + }, + { + "entropy": 2.118139310479164, + "epoch": 0.697087456432034, + "grad_norm": 3.28125, + "learning_rate": 6.063130908508252e-06, + "loss": 2.1376751708984374, + "mean_token_accuracy": 0.5243131402134895, + "num_tokens": 26096619.0, + "step": 4350 + }, + { + "entropy": 2.082554380297661, + "epoch": 0.7050999559312527, + "grad_norm": 3.390625, + "learning_rate": 5.902900176253806e-06, + "loss": 2.0949493408203126, + "mean_token_accuracy": 0.5316410692036152, + "num_tokens": 26394395.0, + "step": 4400 + }, + { + "entropy": 2.09870474755764, + "epoch": 0.7131124554304715, + "grad_norm": 3.421875, + "learning_rate": 5.7426694439993594e-06, + "loss": 2.1184346008300783, + "mean_token_accuracy": 0.5267573061585427, + "num_tokens": 26692333.0, + "step": 4450 + }, + { + "entropy": 2.0866885662078856, + "epoch": 0.7211249549296903, + "grad_norm": 3.71875, + "learning_rate": 5.5824387117449134e-06, + "loss": 2.102904968261719, + "mean_token_accuracy": 0.529726449996233, + "num_tokens": 26989384.0, + "step": 4500 + }, + { + "entropy": 2.114828346967697, + "epoch": 0.7291374544289091, + "grad_norm": 3.46875, + "learning_rate": 5.4222079794904666e-06, + "loss": 2.128611602783203, + "mean_token_accuracy": 0.5247649565339089, + "num_tokens": 27299927.0, + "step": 4550 + }, + { + "entropy": 2.09617226600647, + "epoch": 0.7371499539281279, + "grad_norm": 3.015625, + "learning_rate": 5.26197724723602e-06, + "loss": 2.1166041564941405, + "mean_token_accuracy": 0.5273530712723732, + "num_tokens": 27599011.0, + "step": 4600 + }, + { + "entropy": 2.0975282275676728, + "epoch": 0.7451624534273467, + "grad_norm": 3.3125, + "learning_rate": 5.101746514981574e-06, + "loss": 2.112121124267578, + "mean_token_accuracy": 0.5281847220659256, + "num_tokens": 27892355.0, + "step": 4650 + }, + { + "entropy": 2.095783769488335, + "epoch": 0.7531749529265654, + "grad_norm": 3.78125, + "learning_rate": 4.941515782727128e-06, + "loss": 2.110701904296875, + "mean_token_accuracy": 0.5272985179722309, + "num_tokens": 28187615.0, + "step": 4700 + }, + { + "entropy": 2.0964103603363036, + "epoch": 0.7611874524257842, + "grad_norm": 3.1875, + "learning_rate": 4.781285050472681e-06, + "loss": 2.11992919921875, + "mean_token_accuracy": 0.5271916574239731, + "num_tokens": 28485419.0, + "step": 4750 + }, + { + "entropy": 2.0820577853918074, + "epoch": 0.769199951925003, + "grad_norm": 3.5, + "learning_rate": 4.621054318218234e-06, + "loss": 2.095073394775391, + "mean_token_accuracy": 0.5297581899166107, + "num_tokens": 28780158.0, + "step": 4800 + }, + { + "entropy": 2.089591929912567, + "epoch": 0.7772124514242218, + "grad_norm": 3.125, + "learning_rate": 4.460823585963788e-06, + "loss": 2.109657897949219, + "mean_token_accuracy": 0.528961935043335, + "num_tokens": 29081052.0, + "step": 4850 + }, + { + "entropy": 2.092900555729866, + "epoch": 0.7852249509234406, + "grad_norm": 3.46875, + "learning_rate": 4.300592853709342e-06, + "loss": 2.106436920166016, + "mean_token_accuracy": 0.5284854583442211, + "num_tokens": 29376324.0, + "step": 4900 + }, + { + "entropy": 2.0630873823165894, + "epoch": 0.7932374504226594, + "grad_norm": 3.28125, + "learning_rate": 4.140362121454896e-06, + "loss": 2.088908233642578, + "mean_token_accuracy": 0.532561699450016, + "num_tokens": 29675269.0, + "step": 4950 + }, + { + "entropy": 2.0753238475322724, + "epoch": 0.8012499499218781, + "grad_norm": 3.21875, + "learning_rate": 3.980131389200449e-06, + "loss": 2.0961412048339843, + "mean_token_accuracy": 0.5304154875874519, + "num_tokens": 29974838.0, + "step": 5000 + }, + { + "entropy": 2.123110404610634, + "epoch": 0.8092624494210969, + "grad_norm": 3.28125, + "learning_rate": 3.819900656946002e-06, + "loss": 2.1538424682617188, + "mean_token_accuracy": 0.5222072719037533, + "num_tokens": 30286201.0, + "step": 5050 + }, + { + "entropy": 2.123967355489731, + "epoch": 0.8172749489203157, + "grad_norm": 3.046875, + "learning_rate": 3.6596699246915563e-06, + "loss": 2.1417193603515625, + "mean_token_accuracy": 0.5213662980496884, + "num_tokens": 30592837.0, + "step": 5100 + }, + { + "entropy": 2.080365374684334, + "epoch": 0.8252874484195345, + "grad_norm": 3.375, + "learning_rate": 3.4994391924371094e-06, + "loss": 2.0970494079589845, + "mean_token_accuracy": 0.5296585746109486, + "num_tokens": 30890674.0, + "step": 5150 + }, + { + "entropy": 2.1183922028541566, + "epoch": 0.8332999479187533, + "grad_norm": 3.359375, + "learning_rate": 3.3392084601826634e-06, + "loss": 2.132384948730469, + "mean_token_accuracy": 0.5249108146131038, + "num_tokens": 31195601.0, + "step": 5200 + }, + { + "entropy": 2.1079335528612138, + "epoch": 0.8413124474179721, + "grad_norm": 2.890625, + "learning_rate": 3.178977727928217e-06, + "loss": 2.1310113525390624, + "mean_token_accuracy": 0.5255681444704533, + "num_tokens": 31498768.0, + "step": 5250 + }, + { + "entropy": 2.1180070728063582, + "epoch": 0.8493249469171908, + "grad_norm": 3.78125, + "learning_rate": 3.01874699567377e-06, + "loss": 2.1365049743652342, + "mean_token_accuracy": 0.5235744562745094, + "num_tokens": 31801891.0, + "step": 5300 + }, + { + "entropy": 2.0909912359714506, + "epoch": 0.8573374464164096, + "grad_norm": 3.078125, + "learning_rate": 2.858516263419324e-06, + "loss": 2.108803405761719, + "mean_token_accuracy": 0.5267959542572498, + "num_tokens": 32102627.0, + "step": 5350 + }, + { + "entropy": 2.1209339940547944, + "epoch": 0.8653499459156284, + "grad_norm": 3.140625, + "learning_rate": 2.6982855311648777e-06, + "loss": 2.143946228027344, + "mean_token_accuracy": 0.5227122528851033, + "num_tokens": 32405500.0, + "step": 5400 + }, + { + "entropy": 2.1137486273050308, + "epoch": 0.8733624454148472, + "grad_norm": 3.03125, + "learning_rate": 2.5380547989104317e-06, + "loss": 2.127569122314453, + "mean_token_accuracy": 0.5245758745074273, + "num_tokens": 32712521.0, + "step": 5450 + }, + { + "entropy": 2.0872016245126725, + "epoch": 0.881374944914066, + "grad_norm": 3.171875, + "learning_rate": 2.377824066655985e-06, + "loss": 2.100517120361328, + "mean_token_accuracy": 0.5295753349363804, + "num_tokens": 33016210.0, + "step": 5500 + }, + { + "entropy": 2.0836504286527635, + "epoch": 0.8893874444132848, + "grad_norm": 3.328125, + "learning_rate": 2.2175933344015384e-06, + "loss": 2.1047386169433593, + "mean_token_accuracy": 0.5296267972886562, + "num_tokens": 33311705.0, + "step": 5550 + }, + { + "entropy": 2.0777365696430206, + "epoch": 0.8973999439125036, + "grad_norm": 3.59375, + "learning_rate": 2.057362602147092e-06, + "loss": 2.0985890197753907, + "mean_token_accuracy": 0.5303736463189125, + "num_tokens": 33603231.0, + "step": 5600 + }, + { + "entropy": 2.095007498860359, + "epoch": 0.9054124434117223, + "grad_norm": 3.59375, + "learning_rate": 1.8971318698926456e-06, + "loss": 2.106617431640625, + "mean_token_accuracy": 0.527913925498724, + "num_tokens": 33902978.0, + "step": 5650 + }, + { + "entropy": 2.106599664092064, + "epoch": 0.9134249429109411, + "grad_norm": 3.234375, + "learning_rate": 1.7369011376381991e-06, + "loss": 2.1318267822265624, + "mean_token_accuracy": 0.525322500616312, + "num_tokens": 34206935.0, + "step": 5700 + }, + { + "entropy": 2.094651755094528, + "epoch": 0.9214374424101599, + "grad_norm": 3.65625, + "learning_rate": 1.576670405383753e-06, + "loss": 2.110823211669922, + "mean_token_accuracy": 0.5286212807893753, + "num_tokens": 34499610.0, + "step": 5750 + }, + { + "entropy": 2.119523124098778, + "epoch": 0.9294499419093787, + "grad_norm": 3.375, + "learning_rate": 1.4164396731293063e-06, + "loss": 2.13890625, + "mean_token_accuracy": 0.5239527927339077, + "num_tokens": 34806213.0, + "step": 5800 + }, + { + "entropy": 2.0914100110530853, + "epoch": 0.9374624414085974, + "grad_norm": 3.46875, + "learning_rate": 1.2562089408748598e-06, + "loss": 2.113725280761719, + "mean_token_accuracy": 0.5276746636629105, + "num_tokens": 35106711.0, + "step": 5850 + }, + { + "entropy": 2.0877316004037856, + "epoch": 0.9454749409078161, + "grad_norm": 3.125, + "learning_rate": 1.0959782086204134e-06, + "loss": 2.097077178955078, + "mean_token_accuracy": 0.5296605123579502, + "num_tokens": 35398548.0, + "step": 5900 + }, + { + "entropy": 2.099357485175133, + "epoch": 0.9534874404070349, + "grad_norm": 3.234375, + "learning_rate": 9.357474763659671e-07, + "loss": 2.115228729248047, + "mean_token_accuracy": 0.5284087285399437, + "num_tokens": 35689259.0, + "step": 5950 + }, + { + "entropy": 2.0721785259246825, + "epoch": 0.9614999399062537, + "grad_norm": 3.84375, + "learning_rate": 7.755167441115207e-07, + "loss": 2.098111114501953, + "mean_token_accuracy": 0.5322318106889725, + "num_tokens": 35989481.0, + "step": 6000 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.250307255744676e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-6000/training_args.bin b/checkpoint-6000/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-6000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/checkpoint-6241/chat_template.jinja b/checkpoint-6241/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-6241/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-6241/config.json b/checkpoint-6241/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-6241/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-6241/generation_config.json b/checkpoint-6241/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-6241/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-6241/model.safetensors b/checkpoint-6241/model.safetensors new file mode 100644 index 0000000..c6cea83 --- /dev/null +++ b/checkpoint-6241/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3140e22b016936d08d1a9957161ddab89478b00c1cab50daaa2ad5dc5d16a4fc +size 3441185608 diff --git a/checkpoint-6241/optimizer.pt b/checkpoint-6241/optimizer.pt new file mode 100644 index 0000000..a94c452 --- /dev/null +++ b/checkpoint-6241/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87995b276edaded0776df2b192a978dbc6826df9aa65c79653b7296b87a4fac0 +size 6882572207 diff --git a/checkpoint-6241/rng_state.pth b/checkpoint-6241/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-6241/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-6241/scheduler.pt b/checkpoint-6241/scheduler.pt new file mode 100644 index 0000000..32660c0 --- /dev/null +++ b/checkpoint-6241/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed1f7c61e35590392ce30e26956732331fee27e786c2a4cf9b1aadf0ef625f40 +size 1465 diff --git a/checkpoint-6241/tokenizer.json b/checkpoint-6241/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-6241/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-6241/tokenizer_config.json b/checkpoint-6241/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-6241/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-6241/trainer_state.json b/checkpoint-6241/trainer_state.json new file mode 100644 index 0000000..99c534e --- /dev/null +++ b/checkpoint-6241/trainer_state.json @@ -0,0 +1,1274 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 6241, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.223189059495926, + "epoch": 0.008012499499218782, + "grad_norm": 2.875, + "learning_rate": 1.9842973882390643e-05, + "loss": 2.3428109741210936, + "mean_token_accuracy": 0.4963963569700718, + "num_tokens": 305366.0, + "step": 50 + }, + { + "entropy": 2.188130117058754, + "epoch": 0.016024998998437564, + "grad_norm": 3.328125, + "learning_rate": 1.96827431501362e-05, + "loss": 2.225358734130859, + "mean_token_accuracy": 0.5110921229422093, + "num_tokens": 605042.0, + "step": 100 + }, + { + "entropy": 2.197347580194473, + "epoch": 0.024037498497656344, + "grad_norm": 3.453125, + "learning_rate": 1.952251241788175e-05, + "loss": 2.2299420166015627, + "mean_token_accuracy": 0.5093490639328957, + "num_tokens": 906888.0, + "step": 150 + }, + { + "entropy": 2.1913950395584108, + "epoch": 0.03204999799687513, + "grad_norm": 2.90625, + "learning_rate": 1.9362281685627303e-05, + "loss": 2.210995941162109, + "mean_token_accuracy": 0.5097909501194954, + "num_tokens": 1213125.0, + "step": 200 + }, + { + "entropy": 2.1611998677253723, + "epoch": 0.040062497496093905, + "grad_norm": 3.25, + "learning_rate": 1.920205095337286e-05, + "loss": 2.1867474365234374, + "mean_token_accuracy": 0.5154410660266876, + "num_tokens": 1509026.0, + "step": 250 + }, + { + "entropy": 2.168051454424858, + "epoch": 0.04807499699531269, + "grad_norm": 2.9375, + "learning_rate": 1.904182022111841e-05, + "loss": 2.190071716308594, + "mean_token_accuracy": 0.5156884534657001, + "num_tokens": 1806850.0, + "step": 300 + }, + { + "entropy": 2.1737154543399813, + "epoch": 0.05608749649453147, + "grad_norm": 3.421875, + "learning_rate": 1.8881589488863967e-05, + "loss": 2.199147186279297, + "mean_token_accuracy": 0.514045044630766, + "num_tokens": 2104505.0, + "step": 350 + }, + { + "entropy": 2.148039159178734, + "epoch": 0.06409999599375026, + "grad_norm": 3.109375, + "learning_rate": 1.872135875660952e-05, + "loss": 2.176946105957031, + "mean_token_accuracy": 0.5177112428843975, + "num_tokens": 2391088.0, + "step": 400 + }, + { + "entropy": 2.194890711903572, + "epoch": 0.07211249549296903, + "grad_norm": 3.28125, + "learning_rate": 1.856112802435507e-05, + "loss": 2.216288757324219, + "mean_token_accuracy": 0.5110335703194141, + "num_tokens": 2691272.0, + "step": 450 + }, + { + "entropy": 2.1557786524295808, + "epoch": 0.08012499499218781, + "grad_norm": 4.15625, + "learning_rate": 1.8400897292100627e-05, + "loss": 2.1812680053710936, + "mean_token_accuracy": 0.5176697050035, + "num_tokens": 2991855.0, + "step": 500 + }, + { + "entropy": 2.1708167713880537, + "epoch": 0.08813749449140659, + "grad_norm": 4.34375, + "learning_rate": 1.824066655984618e-05, + "loss": 2.1921002197265627, + "mean_token_accuracy": 0.5156507430970669, + "num_tokens": 3295049.0, + "step": 550 + }, + { + "entropy": 2.1686314886808393, + "epoch": 0.09614999399062538, + "grad_norm": 2.875, + "learning_rate": 1.8080435827591735e-05, + "loss": 2.1914617919921877, + "mean_token_accuracy": 0.5139613692462445, + "num_tokens": 3597206.0, + "step": 600 + }, + { + "entropy": 2.15742906332016, + "epoch": 0.10416249348984416, + "grad_norm": 3.109375, + "learning_rate": 1.7920205095337287e-05, + "loss": 2.174853973388672, + "mean_token_accuracy": 0.5159518288075924, + "num_tokens": 3905268.0, + "step": 650 + }, + { + "entropy": 2.162363375425339, + "epoch": 0.11217499298906294, + "grad_norm": 3.171875, + "learning_rate": 1.7759974363082843e-05, + "loss": 2.184231872558594, + "mean_token_accuracy": 0.516631413847208, + "num_tokens": 4207934.0, + "step": 700 + }, + { + "entropy": 2.1516289287805557, + "epoch": 0.12018749248828171, + "grad_norm": 3.609375, + "learning_rate": 1.7599743630828395e-05, + "loss": 2.167020263671875, + "mean_token_accuracy": 0.516358617246151, + "num_tokens": 4515774.0, + "step": 750 + }, + { + "entropy": 2.1549544781446457, + "epoch": 0.1281999919875005, + "grad_norm": 3.28125, + "learning_rate": 1.7439512898573947e-05, + "loss": 2.181917877197266, + "mean_token_accuracy": 0.5182220858335495, + "num_tokens": 4814709.0, + "step": 800 + }, + { + "entropy": 2.1532615756988527, + "epoch": 0.1362124914867193, + "grad_norm": 3.078125, + "learning_rate": 1.7279282166319503e-05, + "loss": 2.1659007263183594, + "mean_token_accuracy": 0.5180425970256328, + "num_tokens": 5112585.0, + "step": 850 + }, + { + "entropy": 2.1704017293453215, + "epoch": 0.14422499098593805, + "grad_norm": 2.921875, + "learning_rate": 1.7119051434065055e-05, + "loss": 2.1934996032714844, + "mean_token_accuracy": 0.5154964691400528, + "num_tokens": 5411369.0, + "step": 900 + }, + { + "entropy": 2.1349107217788696, + "epoch": 0.15223749048515683, + "grad_norm": 2.984375, + "learning_rate": 1.6958820701810608e-05, + "loss": 2.1598545837402345, + "mean_token_accuracy": 0.5208920259773732, + "num_tokens": 5707060.0, + "step": 950 + }, + { + "entropy": 2.1541705918312073, + "epoch": 0.16024998998437562, + "grad_norm": 3.203125, + "learning_rate": 1.6798589969556163e-05, + "loss": 2.170956573486328, + "mean_token_accuracy": 0.5204938448965549, + "num_tokens": 6004079.0, + "step": 1000 + }, + { + "entropy": 2.1523867017030716, + "epoch": 0.1682624894835944, + "grad_norm": 3.09375, + "learning_rate": 1.6638359237301716e-05, + "loss": 2.1695118713378907, + "mean_token_accuracy": 0.5177376234531402, + "num_tokens": 6315462.0, + "step": 1050 + }, + { + "entropy": 2.1356022965908052, + "epoch": 0.17627498898281319, + "grad_norm": 3.21875, + "learning_rate": 1.6478128505047268e-05, + "loss": 2.1513107299804686, + "mean_token_accuracy": 0.5217180585861206, + "num_tokens": 6613060.0, + "step": 1100 + }, + { + "entropy": 2.1483460652828215, + "epoch": 0.18428748848203197, + "grad_norm": 3.21875, + "learning_rate": 1.6317897772792824e-05, + "loss": 2.1671595764160156, + "mean_token_accuracy": 0.5199222292006016, + "num_tokens": 6909655.0, + "step": 1150 + }, + { + "entropy": 2.1074040442705155, + "epoch": 0.19229998798125075, + "grad_norm": 3.828125, + "learning_rate": 1.6157667040538376e-05, + "loss": 2.12770751953125, + "mean_token_accuracy": 0.5252153894305229, + "num_tokens": 7203263.0, + "step": 1200 + }, + { + "entropy": 2.1354027593135836, + "epoch": 0.20031248748046954, + "grad_norm": 2.921875, + "learning_rate": 1.5997436308283932e-05, + "loss": 2.156183166503906, + "mean_token_accuracy": 0.518907299786806, + "num_tokens": 7505411.0, + "step": 1250 + }, + { + "entropy": 2.142924894094467, + "epoch": 0.20832498697968832, + "grad_norm": 3.5625, + "learning_rate": 1.5837205576029484e-05, + "loss": 2.158486633300781, + "mean_token_accuracy": 0.5205909128487111, + "num_tokens": 7803562.0, + "step": 1300 + }, + { + "entropy": 2.1305291467905043, + "epoch": 0.2163374864789071, + "grad_norm": 3.34375, + "learning_rate": 1.5676974843775036e-05, + "loss": 2.151043701171875, + "mean_token_accuracy": 0.5211617235839366, + "num_tokens": 8108037.0, + "step": 1350 + }, + { + "entropy": 2.160947166085243, + "epoch": 0.2243499859781259, + "grad_norm": 3.234375, + "learning_rate": 1.5516744111520592e-05, + "loss": 2.177792663574219, + "mean_token_accuracy": 0.5167004990577698, + "num_tokens": 8408674.0, + "step": 1400 + }, + { + "entropy": 2.120403967499733, + "epoch": 0.23236248547734467, + "grad_norm": 3.375, + "learning_rate": 1.5356513379266144e-05, + "loss": 2.1462562561035154, + "mean_token_accuracy": 0.5225985661149025, + "num_tokens": 8707650.0, + "step": 1450 + }, + { + "entropy": 2.138882914185524, + "epoch": 0.24037498497656343, + "grad_norm": 3.171875, + "learning_rate": 1.5196282647011698e-05, + "loss": 2.1499905395507812, + "mean_token_accuracy": 0.5204668046534061, + "num_tokens": 9006296.0, + "step": 1500 + }, + { + "entropy": 2.131927245259285, + "epoch": 0.2483874844757822, + "grad_norm": 3.546875, + "learning_rate": 1.5036051914757252e-05, + "loss": 2.157076416015625, + "mean_token_accuracy": 0.5220408822596073, + "num_tokens": 9307203.0, + "step": 1550 + }, + { + "entropy": 2.1308567851781843, + "epoch": 0.256399983975001, + "grad_norm": 3.546875, + "learning_rate": 1.4875821182502805e-05, + "loss": 2.146601104736328, + "mean_token_accuracy": 0.5219539681077003, + "num_tokens": 9610070.0, + "step": 1600 + }, + { + "entropy": 2.1097783839702604, + "epoch": 0.2644124834742198, + "grad_norm": 3.84375, + "learning_rate": 1.4715590450248359e-05, + "loss": 2.1268209838867187, + "mean_token_accuracy": 0.5236652019619942, + "num_tokens": 9909117.0, + "step": 1650 + }, + { + "entropy": 2.123499180674553, + "epoch": 0.2724249829734386, + "grad_norm": 3.46875, + "learning_rate": 1.4555359717993913e-05, + "loss": 2.1409934997558593, + "mean_token_accuracy": 0.5225465354323388, + "num_tokens": 10209461.0, + "step": 1700 + }, + { + "entropy": 2.13052960395813, + "epoch": 0.2804374824726573, + "grad_norm": 2.71875, + "learning_rate": 1.4395128985739467e-05, + "loss": 2.151373291015625, + "mean_token_accuracy": 0.5227928113937378, + "num_tokens": 10509282.0, + "step": 1750 + }, + { + "entropy": 2.1226114094257356, + "epoch": 0.2884499819718761, + "grad_norm": 3.65625, + "learning_rate": 1.4234898253485019e-05, + "loss": 2.144451141357422, + "mean_token_accuracy": 0.5233827362954616, + "num_tokens": 10811056.0, + "step": 1800 + }, + { + "entropy": 2.126143154501915, + "epoch": 0.2964624814710949, + "grad_norm": 3.359375, + "learning_rate": 1.4074667521230575e-05, + "loss": 2.1457598876953123, + "mean_token_accuracy": 0.5221967925131321, + "num_tokens": 11111153.0, + "step": 1850 + }, + { + "entropy": 2.1048326581716537, + "epoch": 0.30447498097031367, + "grad_norm": 3.59375, + "learning_rate": 1.3914436788976127e-05, + "loss": 2.1260116577148436, + "mean_token_accuracy": 0.5272719909250736, + "num_tokens": 11405606.0, + "step": 1900 + }, + { + "entropy": 2.1236617982387545, + "epoch": 0.31248748046953245, + "grad_norm": 3.0, + "learning_rate": 1.375420605672168e-05, + "loss": 2.151782684326172, + "mean_token_accuracy": 0.5227546173334122, + "num_tokens": 11703943.0, + "step": 1950 + }, + { + "entropy": 2.121563429236412, + "epoch": 0.32049997996875124, + "grad_norm": 3.34375, + "learning_rate": 1.3593975324467235e-05, + "loss": 2.1370144653320313, + "mean_token_accuracy": 0.5235123547911644, + "num_tokens": 12002522.0, + "step": 2000 + }, + { + "entropy": 2.1155461168289182, + "epoch": 0.32851247946797, + "grad_norm": 3.46875, + "learning_rate": 1.3433744592212787e-05, + "loss": 2.1364064025878906, + "mean_token_accuracy": 0.5237135806679726, + "num_tokens": 12303098.0, + "step": 2050 + }, + { + "entropy": 2.1386052322387696, + "epoch": 0.3365249789671888, + "grad_norm": 3.484375, + "learning_rate": 1.327351385995834e-05, + "loss": 2.149888458251953, + "mean_token_accuracy": 0.522669050693512, + "num_tokens": 12605534.0, + "step": 2100 + }, + { + "entropy": 2.102600795030594, + "epoch": 0.3445374784664076, + "grad_norm": 3.796875, + "learning_rate": 1.3113283127703895e-05, + "loss": 2.1236566162109374, + "mean_token_accuracy": 0.5279092155396938, + "num_tokens": 12901218.0, + "step": 2150 + }, + { + "entropy": 2.132645187973976, + "epoch": 0.35254997796562637, + "grad_norm": 3.53125, + "learning_rate": 1.2953052395449447e-05, + "loss": 2.1523503112792968, + "mean_token_accuracy": 0.5198129612207413, + "num_tokens": 13213018.0, + "step": 2200 + }, + { + "entropy": 2.1204321217536926, + "epoch": 0.36056247746484515, + "grad_norm": 3.25, + "learning_rate": 1.2792821663195001e-05, + "loss": 2.1325447082519533, + "mean_token_accuracy": 0.5243636940419674, + "num_tokens": 13514408.0, + "step": 2250 + }, + { + "entropy": 2.102410731315613, + "epoch": 0.36857497696406394, + "grad_norm": 3.1875, + "learning_rate": 1.2632590930940555e-05, + "loss": 2.123565368652344, + "mean_token_accuracy": 0.5257614709436893, + "num_tokens": 13818272.0, + "step": 2300 + }, + { + "entropy": 2.0876205080747603, + "epoch": 0.3765874764632827, + "grad_norm": 3.328125, + "learning_rate": 1.247236019868611e-05, + "loss": 2.116696472167969, + "mean_token_accuracy": 0.5275124031305313, + "num_tokens": 14110702.0, + "step": 2350 + }, + { + "entropy": 2.1078518223762512, + "epoch": 0.3845999759625015, + "grad_norm": 3.453125, + "learning_rate": 1.2312129466431662e-05, + "loss": 2.1291993713378905, + "mean_token_accuracy": 0.524967464953661, + "num_tokens": 14407015.0, + "step": 2400 + }, + { + "entropy": 2.0930898267030718, + "epoch": 0.3926124754617203, + "grad_norm": 3.5, + "learning_rate": 1.2151898734177216e-05, + "loss": 2.1094123840332033, + "mean_token_accuracy": 0.5290587748587131, + "num_tokens": 14696565.0, + "step": 2450 + }, + { + "entropy": 2.1109757220745085, + "epoch": 0.4006249749609391, + "grad_norm": 3.5, + "learning_rate": 1.199166800192277e-05, + "loss": 2.1246888732910154, + "mean_token_accuracy": 0.5258957357704639, + "num_tokens": 14994401.0, + "step": 2500 + }, + { + "entropy": 2.1087811160087586, + "epoch": 0.40863747446015786, + "grad_norm": 3.1875, + "learning_rate": 1.1831437269668324e-05, + "loss": 2.12890625, + "mean_token_accuracy": 0.5269917643070221, + "num_tokens": 15292837.0, + "step": 2550 + }, + { + "entropy": 2.097252429127693, + "epoch": 0.41664997395937664, + "grad_norm": 3.0, + "learning_rate": 1.1671206537413878e-05, + "loss": 2.1145870971679686, + "mean_token_accuracy": 0.5274137698113919, + "num_tokens": 15590647.0, + "step": 2600 + }, + { + "entropy": 2.111199595332146, + "epoch": 0.4246624734585954, + "grad_norm": 3.25, + "learning_rate": 1.151097580515943e-05, + "loss": 2.1274327087402343, + "mean_token_accuracy": 0.5250249288976192, + "num_tokens": 15892772.0, + "step": 2650 + }, + { + "entropy": 2.1051502913236617, + "epoch": 0.4326749729578142, + "grad_norm": 3.203125, + "learning_rate": 1.1350745072904986e-05, + "loss": 2.127935333251953, + "mean_token_accuracy": 0.5263640700280666, + "num_tokens": 16191611.0, + "step": 2700 + }, + { + "entropy": 2.1127885603904724, + "epoch": 0.440687472457033, + "grad_norm": 3.515625, + "learning_rate": 1.1190514340650538e-05, + "loss": 2.129211883544922, + "mean_token_accuracy": 0.5250939278304577, + "num_tokens": 16483582.0, + "step": 2750 + }, + { + "entropy": 2.110930626988411, + "epoch": 0.4486999719562518, + "grad_norm": 3.1875, + "learning_rate": 1.103028360839609e-05, + "loss": 2.1220079040527344, + "mean_token_accuracy": 0.5250919266045093, + "num_tokens": 16787253.0, + "step": 2800 + }, + { + "entropy": 2.1182780677080153, + "epoch": 0.45671247145547056, + "grad_norm": 3.8125, + "learning_rate": 1.0870052876141646e-05, + "loss": 2.1358837890625, + "mean_token_accuracy": 0.5217854312062263, + "num_tokens": 17088741.0, + "step": 2850 + }, + { + "entropy": 2.1117833071947096, + "epoch": 0.46472497095468934, + "grad_norm": 3.171875, + "learning_rate": 1.0709822143887198e-05, + "loss": 2.125662841796875, + "mean_token_accuracy": 0.524649152457714, + "num_tokens": 17389236.0, + "step": 2900 + }, + { + "entropy": 2.1262730568647386, + "epoch": 0.47273747045390807, + "grad_norm": 3.625, + "learning_rate": 1.054959141163275e-05, + "loss": 2.142789611816406, + "mean_token_accuracy": 0.5236384868621826, + "num_tokens": 17693413.0, + "step": 2950 + }, + { + "entropy": 2.115410739183426, + "epoch": 0.48074996995312685, + "grad_norm": 3.140625, + "learning_rate": 1.0389360679378306e-05, + "loss": 2.1337106323242185, + "mean_token_accuracy": 0.5249222332239151, + "num_tokens": 17990326.0, + "step": 3000 + }, + { + "entropy": 2.0951381689310073, + "epoch": 0.48876246945234564, + "grad_norm": 3.234375, + "learning_rate": 1.0229129947123859e-05, + "loss": 2.108144073486328, + "mean_token_accuracy": 0.5269971822202205, + "num_tokens": 18293136.0, + "step": 3050 + }, + { + "entropy": 2.116977259516716, + "epoch": 0.4967749689515644, + "grad_norm": 3.421875, + "learning_rate": 1.0068899214869413e-05, + "loss": 2.1327830505371095, + "mean_token_accuracy": 0.5238270902633667, + "num_tokens": 18591481.0, + "step": 3100 + }, + { + "entropy": 2.08598925113678, + "epoch": 0.5047874684507833, + "grad_norm": 2.96875, + "learning_rate": 9.908668482614967e-06, + "loss": 2.1032344055175782, + "mean_token_accuracy": 0.5289211909472943, + "num_tokens": 18885221.0, + "step": 3150 + }, + { + "entropy": 2.097712662220001, + "epoch": 0.512799967950002, + "grad_norm": 3.40625, + "learning_rate": 9.74843775036052e-06, + "loss": 2.1150633239746095, + "mean_token_accuracy": 0.5288272814452648, + "num_tokens": 19184784.0, + "step": 3200 + }, + { + "entropy": 2.1063760107755662, + "epoch": 0.5208124674492208, + "grad_norm": 3.28125, + "learning_rate": 9.588207018106075e-06, + "loss": 2.1217109680175783, + "mean_token_accuracy": 0.526290989369154, + "num_tokens": 19482538.0, + "step": 3250 + }, + { + "entropy": 2.090182801485062, + "epoch": 0.5288249669484396, + "grad_norm": 2.875, + "learning_rate": 9.427976285851627e-06, + "loss": 2.1130311584472654, + "mean_token_accuracy": 0.5285439921915531, + "num_tokens": 19782544.0, + "step": 3300 + }, + { + "entropy": 2.103260806798935, + "epoch": 0.5368374664476584, + "grad_norm": 3.59375, + "learning_rate": 9.267745553597181e-06, + "loss": 2.116189727783203, + "mean_token_accuracy": 0.5273195558786392, + "num_tokens": 20086408.0, + "step": 3350 + }, + { + "entropy": 2.0673747235536575, + "epoch": 0.5448499659468772, + "grad_norm": 3.125, + "learning_rate": 9.107514821342735e-06, + "loss": 2.087482147216797, + "mean_token_accuracy": 0.532562478929758, + "num_tokens": 20380429.0, + "step": 3400 + }, + { + "entropy": 2.1030789083242416, + "epoch": 0.552862465446096, + "grad_norm": 3.296875, + "learning_rate": 8.947284089088289e-06, + "loss": 2.122591247558594, + "mean_token_accuracy": 0.5256703786551953, + "num_tokens": 20682849.0, + "step": 3450 + }, + { + "entropy": 2.0988716661930082, + "epoch": 0.5608749649453146, + "grad_norm": 3.546875, + "learning_rate": 8.787053356833841e-06, + "loss": 2.115867156982422, + "mean_token_accuracy": 0.5266730943322182, + "num_tokens": 20979225.0, + "step": 3500 + }, + { + "entropy": 2.0954072701931, + "epoch": 0.5688874644445334, + "grad_norm": 3.859375, + "learning_rate": 8.626822624579395e-06, + "loss": 2.1116868591308595, + "mean_token_accuracy": 0.5267495161294937, + "num_tokens": 21284166.0, + "step": 3550 + }, + { + "entropy": 2.11238471031189, + "epoch": 0.5768999639437522, + "grad_norm": 3.484375, + "learning_rate": 8.466591892324949e-06, + "loss": 2.1292352294921875, + "mean_token_accuracy": 0.5249243360757828, + "num_tokens": 21584612.0, + "step": 3600 + }, + { + "entropy": 2.091464368700981, + "epoch": 0.584912463442971, + "grad_norm": 3.359375, + "learning_rate": 8.306361160070501e-06, + "loss": 2.1099319458007812, + "mean_token_accuracy": 0.5292035435140133, + "num_tokens": 21879345.0, + "step": 3650 + }, + { + "entropy": 2.095204224586487, + "epoch": 0.5929249629421898, + "grad_norm": 2.96875, + "learning_rate": 8.146130427816055e-06, + "loss": 2.1198716735839844, + "mean_token_accuracy": 0.5255991271138192, + "num_tokens": 22183486.0, + "step": 3700 + }, + { + "entropy": 2.0666573679447175, + "epoch": 0.6009374624414086, + "grad_norm": 2.90625, + "learning_rate": 7.98589969556161e-06, + "loss": 2.08689697265625, + "mean_token_accuracy": 0.5318664258718491, + "num_tokens": 22476438.0, + "step": 3750 + }, + { + "entropy": 2.122153177857399, + "epoch": 0.6089499619406273, + "grad_norm": 3.40625, + "learning_rate": 7.825668963307162e-06, + "loss": 2.138288879394531, + "mean_token_accuracy": 0.5232584895193577, + "num_tokens": 22780385.0, + "step": 3800 + }, + { + "entropy": 2.1190748488903046, + "epoch": 0.6169624614398461, + "grad_norm": 3.46875, + "learning_rate": 7.665438231052716e-06, + "loss": 2.139790344238281, + "mean_token_accuracy": 0.5243287613987923, + "num_tokens": 23080847.0, + "step": 3850 + }, + { + "entropy": 2.1312570214271545, + "epoch": 0.6249749609390649, + "grad_norm": 3.765625, + "learning_rate": 7.5052074987982706e-06, + "loss": 2.1471080017089843, + "mean_token_accuracy": 0.5232046774029732, + "num_tokens": 23384738.0, + "step": 3900 + }, + { + "entropy": 2.0779008239507677, + "epoch": 0.6329874604382837, + "grad_norm": 3.171875, + "learning_rate": 7.344976766543823e-06, + "loss": 2.0974729919433592, + "mean_token_accuracy": 0.5305448272824287, + "num_tokens": 23682750.0, + "step": 3950 + }, + { + "entropy": 2.1121455317735673, + "epoch": 0.6409999599375025, + "grad_norm": 3.921875, + "learning_rate": 7.184746034289377e-06, + "loss": 2.127069549560547, + "mean_token_accuracy": 0.5252936625480652, + "num_tokens": 23989029.0, + "step": 4000 + }, + { + "entropy": 2.0982888478040693, + "epoch": 0.6490124594367213, + "grad_norm": 4.0625, + "learning_rate": 7.024515302034931e-06, + "loss": 2.112042236328125, + "mean_token_accuracy": 0.5281323495507241, + "num_tokens": 24292063.0, + "step": 4050 + }, + { + "entropy": 2.098363963365555, + "epoch": 0.65702495893594, + "grad_norm": 3.046875, + "learning_rate": 6.864284569780485e-06, + "loss": 2.1156489562988283, + "mean_token_accuracy": 0.5275120075047016, + "num_tokens": 24590201.0, + "step": 4100 + }, + { + "entropy": 2.1065632277727127, + "epoch": 0.6650374584351588, + "grad_norm": 3.375, + "learning_rate": 6.704053837526038e-06, + "loss": 2.12911376953125, + "mean_token_accuracy": 0.5262684133648873, + "num_tokens": 24892122.0, + "step": 4150 + }, + { + "entropy": 2.1014072370529173, + "epoch": 0.6730499579343776, + "grad_norm": 2.84375, + "learning_rate": 6.543823105271592e-06, + "loss": 2.1130335998535155, + "mean_token_accuracy": 0.5262750512361527, + "num_tokens": 25194969.0, + "step": 4200 + }, + { + "entropy": 2.0905236744880678, + "epoch": 0.6810624574335964, + "grad_norm": 3.03125, + "learning_rate": 6.383592373017146e-06, + "loss": 2.1153533935546873, + "mean_token_accuracy": 0.527137616276741, + "num_tokens": 25499521.0, + "step": 4250 + }, + { + "entropy": 2.1032627063989637, + "epoch": 0.6890749569328152, + "grad_norm": 3.125, + "learning_rate": 6.223361640762698e-06, + "loss": 2.122831115722656, + "mean_token_accuracy": 0.5254572249948979, + "num_tokens": 25798933.0, + "step": 4300 + }, + { + "entropy": 2.118139310479164, + "epoch": 0.697087456432034, + "grad_norm": 3.28125, + "learning_rate": 6.063130908508252e-06, + "loss": 2.1376751708984374, + "mean_token_accuracy": 0.5243131402134895, + "num_tokens": 26096619.0, + "step": 4350 + }, + { + "entropy": 2.082554380297661, + "epoch": 0.7050999559312527, + "grad_norm": 3.390625, + "learning_rate": 5.902900176253806e-06, + "loss": 2.0949493408203126, + "mean_token_accuracy": 0.5316410692036152, + "num_tokens": 26394395.0, + "step": 4400 + }, + { + "entropy": 2.09870474755764, + "epoch": 0.7131124554304715, + "grad_norm": 3.421875, + "learning_rate": 5.7426694439993594e-06, + "loss": 2.1184346008300783, + "mean_token_accuracy": 0.5267573061585427, + "num_tokens": 26692333.0, + "step": 4450 + }, + { + "entropy": 2.0866885662078856, + "epoch": 0.7211249549296903, + "grad_norm": 3.71875, + "learning_rate": 5.5824387117449134e-06, + "loss": 2.102904968261719, + "mean_token_accuracy": 0.529726449996233, + "num_tokens": 26989384.0, + "step": 4500 + }, + { + "entropy": 2.114828346967697, + "epoch": 0.7291374544289091, + "grad_norm": 3.46875, + "learning_rate": 5.4222079794904666e-06, + "loss": 2.128611602783203, + "mean_token_accuracy": 0.5247649565339089, + "num_tokens": 27299927.0, + "step": 4550 + }, + { + "entropy": 2.09617226600647, + "epoch": 0.7371499539281279, + "grad_norm": 3.015625, + "learning_rate": 5.26197724723602e-06, + "loss": 2.1166041564941405, + "mean_token_accuracy": 0.5273530712723732, + "num_tokens": 27599011.0, + "step": 4600 + }, + { + "entropy": 2.0975282275676728, + "epoch": 0.7451624534273467, + "grad_norm": 3.3125, + "learning_rate": 5.101746514981574e-06, + "loss": 2.112121124267578, + "mean_token_accuracy": 0.5281847220659256, + "num_tokens": 27892355.0, + "step": 4650 + }, + { + "entropy": 2.095783769488335, + "epoch": 0.7531749529265654, + "grad_norm": 3.78125, + "learning_rate": 4.941515782727128e-06, + "loss": 2.110701904296875, + "mean_token_accuracy": 0.5272985179722309, + "num_tokens": 28187615.0, + "step": 4700 + }, + { + "entropy": 2.0964103603363036, + "epoch": 0.7611874524257842, + "grad_norm": 3.1875, + "learning_rate": 4.781285050472681e-06, + "loss": 2.11992919921875, + "mean_token_accuracy": 0.5271916574239731, + "num_tokens": 28485419.0, + "step": 4750 + }, + { + "entropy": 2.0820577853918074, + "epoch": 0.769199951925003, + "grad_norm": 3.5, + "learning_rate": 4.621054318218234e-06, + "loss": 2.095073394775391, + "mean_token_accuracy": 0.5297581899166107, + "num_tokens": 28780158.0, + "step": 4800 + }, + { + "entropy": 2.089591929912567, + "epoch": 0.7772124514242218, + "grad_norm": 3.125, + "learning_rate": 4.460823585963788e-06, + "loss": 2.109657897949219, + "mean_token_accuracy": 0.528961935043335, + "num_tokens": 29081052.0, + "step": 4850 + }, + { + "entropy": 2.092900555729866, + "epoch": 0.7852249509234406, + "grad_norm": 3.46875, + "learning_rate": 4.300592853709342e-06, + "loss": 2.106436920166016, + "mean_token_accuracy": 0.5284854583442211, + "num_tokens": 29376324.0, + "step": 4900 + }, + { + "entropy": 2.0630873823165894, + "epoch": 0.7932374504226594, + "grad_norm": 3.28125, + "learning_rate": 4.140362121454896e-06, + "loss": 2.088908233642578, + "mean_token_accuracy": 0.532561699450016, + "num_tokens": 29675269.0, + "step": 4950 + }, + { + "entropy": 2.0753238475322724, + "epoch": 0.8012499499218781, + "grad_norm": 3.21875, + "learning_rate": 3.980131389200449e-06, + "loss": 2.0961412048339843, + "mean_token_accuracy": 0.5304154875874519, + "num_tokens": 29974838.0, + "step": 5000 + }, + { + "entropy": 2.123110404610634, + "epoch": 0.8092624494210969, + "grad_norm": 3.28125, + "learning_rate": 3.819900656946002e-06, + "loss": 2.1538424682617188, + "mean_token_accuracy": 0.5222072719037533, + "num_tokens": 30286201.0, + "step": 5050 + }, + { + "entropy": 2.123967355489731, + "epoch": 0.8172749489203157, + "grad_norm": 3.046875, + "learning_rate": 3.6596699246915563e-06, + "loss": 2.1417193603515625, + "mean_token_accuracy": 0.5213662980496884, + "num_tokens": 30592837.0, + "step": 5100 + }, + { + "entropy": 2.080365374684334, + "epoch": 0.8252874484195345, + "grad_norm": 3.375, + "learning_rate": 3.4994391924371094e-06, + "loss": 2.0970494079589845, + "mean_token_accuracy": 0.5296585746109486, + "num_tokens": 30890674.0, + "step": 5150 + }, + { + "entropy": 2.1183922028541566, + "epoch": 0.8332999479187533, + "grad_norm": 3.359375, + "learning_rate": 3.3392084601826634e-06, + "loss": 2.132384948730469, + "mean_token_accuracy": 0.5249108146131038, + "num_tokens": 31195601.0, + "step": 5200 + }, + { + "entropy": 2.1079335528612138, + "epoch": 0.8413124474179721, + "grad_norm": 2.890625, + "learning_rate": 3.178977727928217e-06, + "loss": 2.1310113525390624, + "mean_token_accuracy": 0.5255681444704533, + "num_tokens": 31498768.0, + "step": 5250 + }, + { + "entropy": 2.1180070728063582, + "epoch": 0.8493249469171908, + "grad_norm": 3.78125, + "learning_rate": 3.01874699567377e-06, + "loss": 2.1365049743652342, + "mean_token_accuracy": 0.5235744562745094, + "num_tokens": 31801891.0, + "step": 5300 + }, + { + "entropy": 2.0909912359714506, + "epoch": 0.8573374464164096, + "grad_norm": 3.078125, + "learning_rate": 2.858516263419324e-06, + "loss": 2.108803405761719, + "mean_token_accuracy": 0.5267959542572498, + "num_tokens": 32102627.0, + "step": 5350 + }, + { + "entropy": 2.1209339940547944, + "epoch": 0.8653499459156284, + "grad_norm": 3.140625, + "learning_rate": 2.6982855311648777e-06, + "loss": 2.143946228027344, + "mean_token_accuracy": 0.5227122528851033, + "num_tokens": 32405500.0, + "step": 5400 + }, + { + "entropy": 2.1137486273050308, + "epoch": 0.8733624454148472, + "grad_norm": 3.03125, + "learning_rate": 2.5380547989104317e-06, + "loss": 2.127569122314453, + "mean_token_accuracy": 0.5245758745074273, + "num_tokens": 32712521.0, + "step": 5450 + }, + { + "entropy": 2.0872016245126725, + "epoch": 0.881374944914066, + "grad_norm": 3.171875, + "learning_rate": 2.377824066655985e-06, + "loss": 2.100517120361328, + "mean_token_accuracy": 0.5295753349363804, + "num_tokens": 33016210.0, + "step": 5500 + }, + { + "entropy": 2.0836504286527635, + "epoch": 0.8893874444132848, + "grad_norm": 3.328125, + "learning_rate": 2.2175933344015384e-06, + "loss": 2.1047386169433593, + "mean_token_accuracy": 0.5296267972886562, + "num_tokens": 33311705.0, + "step": 5550 + }, + { + "entropy": 2.0777365696430206, + "epoch": 0.8973999439125036, + "grad_norm": 3.59375, + "learning_rate": 2.057362602147092e-06, + "loss": 2.0985890197753907, + "mean_token_accuracy": 0.5303736463189125, + "num_tokens": 33603231.0, + "step": 5600 + }, + { + "entropy": 2.095007498860359, + "epoch": 0.9054124434117223, + "grad_norm": 3.59375, + "learning_rate": 1.8971318698926456e-06, + "loss": 2.106617431640625, + "mean_token_accuracy": 0.527913925498724, + "num_tokens": 33902978.0, + "step": 5650 + }, + { + "entropy": 2.106599664092064, + "epoch": 0.9134249429109411, + "grad_norm": 3.234375, + "learning_rate": 1.7369011376381991e-06, + "loss": 2.1318267822265624, + "mean_token_accuracy": 0.525322500616312, + "num_tokens": 34206935.0, + "step": 5700 + }, + { + "entropy": 2.094651755094528, + "epoch": 0.9214374424101599, + "grad_norm": 3.65625, + "learning_rate": 1.576670405383753e-06, + "loss": 2.110823211669922, + "mean_token_accuracy": 0.5286212807893753, + "num_tokens": 34499610.0, + "step": 5750 + }, + { + "entropy": 2.119523124098778, + "epoch": 0.9294499419093787, + "grad_norm": 3.375, + "learning_rate": 1.4164396731293063e-06, + "loss": 2.13890625, + "mean_token_accuracy": 0.5239527927339077, + "num_tokens": 34806213.0, + "step": 5800 + }, + { + "entropy": 2.0914100110530853, + "epoch": 0.9374624414085974, + "grad_norm": 3.46875, + "learning_rate": 1.2562089408748598e-06, + "loss": 2.113725280761719, + "mean_token_accuracy": 0.5276746636629105, + "num_tokens": 35106711.0, + "step": 5850 + }, + { + "entropy": 2.0877316004037856, + "epoch": 0.9454749409078161, + "grad_norm": 3.125, + "learning_rate": 1.0959782086204134e-06, + "loss": 2.097077178955078, + "mean_token_accuracy": 0.5296605123579502, + "num_tokens": 35398548.0, + "step": 5900 + }, + { + "entropy": 2.099357485175133, + "epoch": 0.9534874404070349, + "grad_norm": 3.234375, + "learning_rate": 9.357474763659671e-07, + "loss": 2.115228729248047, + "mean_token_accuracy": 0.5284087285399437, + "num_tokens": 35689259.0, + "step": 5950 + }, + { + "entropy": 2.0721785259246825, + "epoch": 0.9614999399062537, + "grad_norm": 3.84375, + "learning_rate": 7.755167441115207e-07, + "loss": 2.098111114501953, + "mean_token_accuracy": 0.5322318106889725, + "num_tokens": 35989481.0, + "step": 6000 + }, + { + "entropy": 2.0931629753112793, + "epoch": 0.9695124394054725, + "grad_norm": 3.15625, + "learning_rate": 6.152860118570742e-07, + "loss": 2.1118145751953126, + "mean_token_accuracy": 0.5284056887030602, + "num_tokens": 36287566.0, + "step": 6050 + }, + { + "entropy": 2.1098696780204773, + "epoch": 0.9775249389046913, + "grad_norm": 3.421875, + "learning_rate": 4.550552796026278e-07, + "loss": 2.135518341064453, + "mean_token_accuracy": 0.5236021775007248, + "num_tokens": 36586868.0, + "step": 6100 + }, + { + "entropy": 2.074869705438614, + "epoch": 0.9855374384039101, + "grad_norm": 3.625, + "learning_rate": 2.9482454734818144e-07, + "loss": 2.0890965270996094, + "mean_token_accuracy": 0.5310191959142685, + "num_tokens": 36882175.0, + "step": 6150 + }, + { + "entropy": 2.1156716483831404, + "epoch": 0.9935499379031288, + "grad_norm": 3.75, + "learning_rate": 1.3459381509373498e-07, + "loss": 2.136824951171875, + "mean_token_accuracy": 0.5256458315253257, + "num_tokens": 37180928.0, + "step": 6200 + } + ], + "logging_steps": 50, + "max_steps": 6241, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.419576669616251e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-6241/training_args.bin b/checkpoint-6241/training_args.bin new file mode 100644 index 0000000..4b409d5 --- /dev/null +++ b/checkpoint-6241/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671 +size 5713 diff --git a/config.json b/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..49a3060 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0", + "enable_thinking": true +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..2ff556a --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1769e30abebb7a7f0f067fab0a814a39f17689469b14573ae2e8dc79229c63d4 +size 3441185608 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..be524ba --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad278028e81c012e7c0ce968b1675e4c1bba648363b7965ce3b1dcbbf63cd2d4 +size 7185