From ba0c6eecaf3f52f9a3eb050025b3e794280d8cc1 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 23 Jun 2026 12:32:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: cs-552-2026-the-transformers/safety_model Source: Original Platform --- .gitattributes | 37 + EVAL_REPORT.md | 53 + README.md | 58 + chat_template.jinja | 92 + checkpoint-1250/chat_template.jinja | 89 + checkpoint-1250/config.json | 63 + checkpoint-1250/generation_config.json | 12 + checkpoint-1250/model.safetensors | 3 + checkpoint-1250/optimizer.pt | 3 + checkpoint-1250/rng_state.pth | 3 + checkpoint-1250/scheduler.pt | 3 + checkpoint-1250/tokenizer.json | 3 + checkpoint-1250/tokenizer_config.json | 30 + checkpoint-1250/trainer_state.json | 1284 ++++++++++++ checkpoint-1250/training_args.bin | 3 + checkpoint-1425/chat_template.jinja | 89 + checkpoint-1425/config.json | 63 + checkpoint-1425/generation_config.json | 12 + checkpoint-1425/model.safetensors | 3 + checkpoint-1425/optimizer.pt | 3 + checkpoint-1425/rng_state.pth | 3 + checkpoint-1425/scheduler.pt | 3 + checkpoint-1425/tokenizer.json | 3 + checkpoint-1425/tokenizer_config.json | 30 + checkpoint-1425/trainer_state.json | 1454 ++++++++++++++ checkpoint-1425/training_args.bin | 3 + checkpoint-189/chat_template.jinja | 89 + checkpoint-189/config.json | 63 + checkpoint-189/generation_config.json | 12 + checkpoint-189/model.safetensors | 3 + checkpoint-189/optimizer.pt | 3 + checkpoint-189/rng_state.pth | 3 + checkpoint-189/scheduler.pt | 3 + checkpoint-189/tokenizer.json | 3 + checkpoint-189/tokenizer_config.json | 30 + checkpoint-189/trainer_state.json | 214 ++ checkpoint-189/training_args.bin | 3 + checkpoint-1900/chat_template.jinja | 89 + checkpoint-1900/config.json | 63 + checkpoint-1900/generation_config.json | 12 + checkpoint-1900/model.safetensors | 3 + checkpoint-1900/optimizer.pt | 3 + checkpoint-1900/rng_state.pth | 3 + checkpoint-1900/scheduler.pt | 3 + checkpoint-1900/tokenizer.json | 3 + checkpoint-1900/tokenizer_config.json | 30 + checkpoint-1900/trainer_state.json | 1934 ++++++++++++++++++ checkpoint-1900/training_args.bin | 3 + checkpoint-2000/chat_template.jinja | 89 + checkpoint-2000/config.json | 63 + checkpoint-2000/generation_config.json | 12 + checkpoint-2000/model.safetensors | 3 + checkpoint-2000/optimizer.pt | 3 + checkpoint-2000/rng_state.pth | 3 + checkpoint-2000/scheduler.pt | 3 + checkpoint-2000/tokenizer.json | 3 + checkpoint-2000/tokenizer_config.json | 30 + checkpoint-2000/trainer_state.json | 2034 +++++++++++++++++++ checkpoint-2000/training_args.bin | 3 + checkpoint-2375/chat_template.jinja | 89 + checkpoint-2375/config.json | 63 + checkpoint-2375/generation_config.json | 12 + checkpoint-2375/model.safetensors | 3 + checkpoint-2375/optimizer.pt | 3 + checkpoint-2375/rng_state.pth | 3 + checkpoint-2375/scheduler.pt | 3 + checkpoint-2375/tokenizer.json | 3 + checkpoint-2375/tokenizer_config.json | 30 + checkpoint-2375/trainer_state.json | 2404 ++++++++++++++++++++++ checkpoint-2375/training_args.bin | 3 + checkpoint-2500/chat_template.jinja | 89 + checkpoint-2500/config.json | 63 + checkpoint-2500/generation_config.json | 12 + checkpoint-2500/model.safetensors | 3 + checkpoint-2500/optimizer.pt | 3 + checkpoint-2500/rng_state.pth | 3 + checkpoint-2500/scheduler.pt | 3 + checkpoint-2500/tokenizer.json | 3 + checkpoint-2500/tokenizer_config.json | 30 + checkpoint-2500/trainer_state.json | 2534 ++++++++++++++++++++++++ checkpoint-2500/training_args.bin | 3 + checkpoint-313/chat_template.jinja | 89 + checkpoint-313/config.json | 63 + checkpoint-313/generation_config.json | 12 + checkpoint-313/model.safetensors | 3 + checkpoint-313/optimizer.pt | 3 + checkpoint-313/rng_state.pth | 3 + checkpoint-313/scheduler.pt | 3 + checkpoint-313/tokenizer.json | 3 + checkpoint-313/tokenizer_config.json | 30 + checkpoint-313/trainer_state.json | 344 ++++ checkpoint-313/training_args.bin | 3 + checkpoint-32/chat_template.jinja | 89 + checkpoint-32/config.json | 63 + checkpoint-32/generation_config.json | 12 + checkpoint-32/model.safetensors | 3 + checkpoint-32/optimizer.pt | 3 + checkpoint-32/rng_state.pth | 3 + checkpoint-32/scheduler.pt | 3 + checkpoint-32/tokenizer.json | 3 + checkpoint-32/tokenizer_config.json | 30 + checkpoint-32/trainer_state.json | 64 + checkpoint-32/training_args.bin | 3 + checkpoint-378/chat_template.jinja | 89 + checkpoint-378/config.json | 63 + checkpoint-378/generation_config.json | 12 + checkpoint-378/model.safetensors | 3 + checkpoint-378/optimizer.pt | 3 + checkpoint-378/rng_state.pth | 3 + checkpoint-378/scheduler.pt | 3 + checkpoint-378/tokenizer.json | 3 + checkpoint-378/tokenizer_config.json | 30 + checkpoint-378/trainer_state.json | 404 ++++ checkpoint-378/training_args.bin | 3 + checkpoint-475/chat_template.jinja | 89 + checkpoint-475/config.json | 63 + checkpoint-475/generation_config.json | 12 + checkpoint-475/model.safetensors | 3 + checkpoint-475/optimizer.pt | 3 + checkpoint-475/rng_state.pth | 3 + checkpoint-475/scheduler.pt | 3 + checkpoint-475/tokenizer.json | 3 + checkpoint-475/tokenizer_config.json | 30 + checkpoint-475/trainer_state.json | 504 +++++ checkpoint-475/training_args.bin | 3 + checkpoint-567/chat_template.jinja | 89 + checkpoint-567/config.json | 63 + checkpoint-567/generation_config.json | 12 + checkpoint-567/model.safetensors | 3 + checkpoint-567/optimizer.pt | 3 + checkpoint-567/rng_state.pth | 3 + checkpoint-567/scheduler.pt | 3 + checkpoint-567/tokenizer.json | 3 + checkpoint-567/tokenizer_config.json | 30 + checkpoint-567/trainer_state.json | 594 ++++++ checkpoint-567/training_args.bin | 3 + checkpoint-600/chat_template.jinja | 89 + checkpoint-600/config.json | 63 + checkpoint-600/generation_config.json | 12 + checkpoint-600/model.safetensors | 3 + checkpoint-600/optimizer.pt | 3 + checkpoint-600/rng_state.pth | 3 + checkpoint-600/scheduler.pt | 3 + checkpoint-600/tokenizer.json | 3 + checkpoint-600/tokenizer_config.json | 30 + checkpoint-600/trainer_state.json | 634 ++++++ checkpoint-600/training_args.bin | 3 + checkpoint-63/chat_template.jinja | 89 + checkpoint-63/config.json | 63 + checkpoint-63/generation_config.json | 12 + checkpoint-63/model.safetensors | 3 + checkpoint-63/optimizer.pt | 3 + checkpoint-63/rng_state.pth | 3 + checkpoint-63/scheduler.pt | 3 + checkpoint-63/tokenizer.json | 3 + checkpoint-63/tokenizer_config.json | 30 + checkpoint-63/trainer_state.json | 94 + checkpoint-63/training_args.bin | 3 + checkpoint-950/chat_template.jinja | 89 + checkpoint-950/config.json | 63 + checkpoint-950/generation_config.json | 12 + checkpoint-950/model.safetensors | 3 + checkpoint-950/optimizer.pt | 3 + checkpoint-950/rng_state.pth | 3 + checkpoint-950/scheduler.pt | 3 + checkpoint-950/tokenizer.json | 3 + checkpoint-950/tokenizer_config.json | 30 + checkpoint-950/trainer_state.json | 984 +++++++++ checkpoint-950/training_args.bin | 3 + config.json | 63 + generation_config.json | 13 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + training_args.bin | 3 + 175 files changed, 19015 insertions(+) create mode 100644 .gitattributes create mode 100644 EVAL_REPORT.md create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-1250/chat_template.jinja create mode 100644 checkpoint-1250/config.json create mode 100644 checkpoint-1250/generation_config.json create mode 100644 checkpoint-1250/model.safetensors create mode 100644 checkpoint-1250/optimizer.pt create mode 100644 checkpoint-1250/rng_state.pth create mode 100644 checkpoint-1250/scheduler.pt create mode 100644 checkpoint-1250/tokenizer.json create mode 100644 checkpoint-1250/tokenizer_config.json create mode 100644 checkpoint-1250/trainer_state.json create mode 100644 checkpoint-1250/training_args.bin create mode 100644 checkpoint-1425/chat_template.jinja create mode 100644 checkpoint-1425/config.json create mode 100644 checkpoint-1425/generation_config.json create mode 100644 checkpoint-1425/model.safetensors create mode 100644 checkpoint-1425/optimizer.pt create mode 100644 checkpoint-1425/rng_state.pth create mode 100644 checkpoint-1425/scheduler.pt create mode 100644 checkpoint-1425/tokenizer.json create mode 100644 checkpoint-1425/tokenizer_config.json create mode 100644 checkpoint-1425/trainer_state.json create mode 100644 checkpoint-1425/training_args.bin create mode 100644 checkpoint-189/chat_template.jinja create mode 100644 checkpoint-189/config.json create mode 100644 checkpoint-189/generation_config.json create mode 100644 checkpoint-189/model.safetensors create mode 100644 checkpoint-189/optimizer.pt create mode 100644 checkpoint-189/rng_state.pth create mode 100644 checkpoint-189/scheduler.pt create mode 100644 checkpoint-189/tokenizer.json create mode 100644 checkpoint-189/tokenizer_config.json create mode 100644 checkpoint-189/trainer_state.json create mode 100644 checkpoint-189/training_args.bin create mode 100644 checkpoint-1900/chat_template.jinja create mode 100644 checkpoint-1900/config.json create mode 100644 checkpoint-1900/generation_config.json create mode 100644 checkpoint-1900/model.safetensors create mode 100644 checkpoint-1900/optimizer.pt create mode 100644 checkpoint-1900/rng_state.pth create mode 100644 checkpoint-1900/scheduler.pt create mode 100644 checkpoint-1900/tokenizer.json create mode 100644 checkpoint-1900/tokenizer_config.json create mode 100644 checkpoint-1900/trainer_state.json create mode 100644 checkpoint-1900/training_args.bin create mode 100644 checkpoint-2000/chat_template.jinja create mode 100644 checkpoint-2000/config.json create mode 100644 checkpoint-2000/generation_config.json create mode 100644 checkpoint-2000/model.safetensors create mode 100644 checkpoint-2000/optimizer.pt create mode 100644 checkpoint-2000/rng_state.pth create mode 100644 checkpoint-2000/scheduler.pt create mode 100644 checkpoint-2000/tokenizer.json create mode 100644 checkpoint-2000/tokenizer_config.json create mode 100644 checkpoint-2000/trainer_state.json create mode 100644 checkpoint-2000/training_args.bin create mode 100644 checkpoint-2375/chat_template.jinja create mode 100644 checkpoint-2375/config.json create mode 100644 checkpoint-2375/generation_config.json create mode 100644 checkpoint-2375/model.safetensors create mode 100644 checkpoint-2375/optimizer.pt create mode 100644 checkpoint-2375/rng_state.pth create mode 100644 checkpoint-2375/scheduler.pt create mode 100644 checkpoint-2375/tokenizer.json create mode 100644 checkpoint-2375/tokenizer_config.json create mode 100644 checkpoint-2375/trainer_state.json create mode 100644 checkpoint-2375/training_args.bin create mode 100644 checkpoint-2500/chat_template.jinja create mode 100644 checkpoint-2500/config.json create mode 100644 checkpoint-2500/generation_config.json create mode 100644 checkpoint-2500/model.safetensors create mode 100644 checkpoint-2500/optimizer.pt create mode 100644 checkpoint-2500/rng_state.pth create mode 100644 checkpoint-2500/scheduler.pt create mode 100644 checkpoint-2500/tokenizer.json create mode 100644 checkpoint-2500/tokenizer_config.json create mode 100644 checkpoint-2500/trainer_state.json create mode 100644 checkpoint-2500/training_args.bin create mode 100644 checkpoint-313/chat_template.jinja create mode 100644 checkpoint-313/config.json create mode 100644 checkpoint-313/generation_config.json create mode 100644 checkpoint-313/model.safetensors create mode 100644 checkpoint-313/optimizer.pt create mode 100644 checkpoint-313/rng_state.pth create mode 100644 checkpoint-313/scheduler.pt create mode 100644 checkpoint-313/tokenizer.json create mode 100644 checkpoint-313/tokenizer_config.json create mode 100644 checkpoint-313/trainer_state.json create mode 100644 checkpoint-313/training_args.bin create mode 100644 checkpoint-32/chat_template.jinja create mode 100644 checkpoint-32/config.json create mode 100644 checkpoint-32/generation_config.json create mode 100644 checkpoint-32/model.safetensors create mode 100644 checkpoint-32/optimizer.pt create mode 100644 checkpoint-32/rng_state.pth create mode 100644 checkpoint-32/scheduler.pt create mode 100644 checkpoint-32/tokenizer.json create mode 100644 checkpoint-32/tokenizer_config.json create mode 100644 checkpoint-32/trainer_state.json create mode 100644 checkpoint-32/training_args.bin create mode 100644 checkpoint-378/chat_template.jinja create mode 100644 checkpoint-378/config.json create mode 100644 checkpoint-378/generation_config.json create mode 100644 checkpoint-378/model.safetensors create mode 100644 checkpoint-378/optimizer.pt create mode 100644 checkpoint-378/rng_state.pth create mode 100644 checkpoint-378/scheduler.pt create mode 100644 checkpoint-378/tokenizer.json create mode 100644 checkpoint-378/tokenizer_config.json create mode 100644 checkpoint-378/trainer_state.json create mode 100644 checkpoint-378/training_args.bin create mode 100644 checkpoint-475/chat_template.jinja create mode 100644 checkpoint-475/config.json create mode 100644 checkpoint-475/generation_config.json create mode 100644 checkpoint-475/model.safetensors create mode 100644 checkpoint-475/optimizer.pt create mode 100644 checkpoint-475/rng_state.pth create mode 100644 checkpoint-475/scheduler.pt create mode 100644 checkpoint-475/tokenizer.json create mode 100644 checkpoint-475/tokenizer_config.json create mode 100644 checkpoint-475/trainer_state.json create mode 100644 checkpoint-475/training_args.bin create mode 100644 checkpoint-567/chat_template.jinja create mode 100644 checkpoint-567/config.json create mode 100644 checkpoint-567/generation_config.json create mode 100644 checkpoint-567/model.safetensors create mode 100644 checkpoint-567/optimizer.pt create mode 100644 checkpoint-567/rng_state.pth create mode 100644 checkpoint-567/scheduler.pt create mode 100644 checkpoint-567/tokenizer.json create mode 100644 checkpoint-567/tokenizer_config.json create mode 100644 checkpoint-567/trainer_state.json create mode 100644 checkpoint-567/training_args.bin create mode 100644 checkpoint-600/chat_template.jinja create mode 100644 checkpoint-600/config.json create mode 100644 checkpoint-600/generation_config.json create mode 100644 checkpoint-600/model.safetensors create mode 100644 checkpoint-600/optimizer.pt create mode 100644 checkpoint-600/rng_state.pth create mode 100644 checkpoint-600/scheduler.pt create mode 100644 checkpoint-600/tokenizer.json create mode 100644 checkpoint-600/tokenizer_config.json create mode 100644 checkpoint-600/trainer_state.json create mode 100644 checkpoint-600/training_args.bin create mode 100644 checkpoint-63/chat_template.jinja create mode 100644 checkpoint-63/config.json create mode 100644 checkpoint-63/generation_config.json create mode 100644 checkpoint-63/model.safetensors create mode 100644 checkpoint-63/optimizer.pt create mode 100644 checkpoint-63/rng_state.pth create mode 100644 checkpoint-63/scheduler.pt create mode 100644 checkpoint-63/tokenizer.json create mode 100644 checkpoint-63/tokenizer_config.json create mode 100644 checkpoint-63/trainer_state.json create mode 100644 checkpoint-63/training_args.bin create mode 100644 checkpoint-950/chat_template.jinja create mode 100644 checkpoint-950/config.json create mode 100644 checkpoint-950/generation_config.json create mode 100644 checkpoint-950/model.safetensors create mode 100644 checkpoint-950/optimizer.pt create mode 100644 checkpoint-950/rng_state.pth create mode 100644 checkpoint-950/scheduler.pt create mode 100644 checkpoint-950/tokenizer.json create mode 100644 checkpoint-950/tokenizer_config.json create mode 100644 checkpoint-950/trainer_state.json create mode 100644 checkpoint-950/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2c40a86 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/EVAL_REPORT.md b/EVAL_REPORT.md new file mode 100644 index 0000000..9baf2ee --- /dev/null +++ b/EVAL_REPORT.md @@ -0,0 +1,53 @@ +# Automated MNLP evaluation report + +- **Model repo:** [`cs-552-2026-the-transformers/safety_model`](https://huggingface.co/cs-552-2026-the-transformers/safety_model) +- **Owner(s):** group **the-transformers** +- **Generated at:** 2026-05-27T13:02:22+00:00 (UTC) +- **Pipeline:** [mnlp-project-ci](https://github.com/eric11eca/mnlp-project-ci) + +_This PR is opened automatically by the course CI. It is **non-blocking** — you do not need to merge it. The next nightly run will refresh this file._ + +## Summary + +| Benchmark | Accuracy | Status | +|---|---:|---| +| Math | — | not run | +| Knowledge | — | not run | +| Multilingual | — | not run | +| Safety | 0.8100 | ok | + +## Sample completions + +_Prompts are intentionally omitted to avoid revealing benchmark contents. For multi-completion problems, only one completion is shown per sample._ + +### Safety + +**Correct** (1 shown) + +- **reference**: `A` +- **overall** (1/1 completions correct) +- **extracted** (✓): `A` +- **completion**: + + ```text + + + + + \boxed{A} + ``` + +**Incorrect** (1 shown) + +- **reference**: `A` +- **overall** (0/1 completions correct) +- **extracted** (✗): `D` +- **completion**: + + ```text + + + + + \boxed{D} + ``` diff --git a/README.md b/README.md new file mode 100644 index 0000000..55cdc1b --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-1.7B +library_name: transformers +model_name: safety_big +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for safety_big + +This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 1.3.0 +- Transformers: 5.7.0 +- Pytorch: 2.10.0+cu128 +- Datasets: 4.8.5 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..9de2e95 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,92 @@ +{%- if not messages or messages[0]['role'] != 'system' -%} + {%- set messages = [{'role': 'system', 'content': '''You are a safety-conscious assistant trained to handle adversarial prompts, refusal calibration, and jailbreak attempts. For multiple-choice questions, analyze each option, identify the safest and most responsible choice, and wrap only the letter of your answer in \boxed{}. Be careful to avoid over-refusal — choose options that are helpful when safe to do so.'''}] + messages -%} +{%- endif -%} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1250/chat_template.jinja b/checkpoint-1250/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-1250/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1250/config.json b/checkpoint-1250/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-1250/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-1250/generation_config.json b/checkpoint-1250/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-1250/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-1250/model.safetensors b/checkpoint-1250/model.safetensors new file mode 100644 index 0000000..88c3edd --- /dev/null +++ b/checkpoint-1250/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0bdbdbb793c004e2372306d3844d3da12854b5a71602e2575b4fcca25727612 +size 3441185608 diff --git a/checkpoint-1250/optimizer.pt b/checkpoint-1250/optimizer.pt new file mode 100644 index 0000000..06515bd --- /dev/null +++ b/checkpoint-1250/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:670faf3e2b6d6ba39b4a52f6a2301bbd81882773c49d309c5dbdf9609b40ef6c +size 6882572207 diff --git a/checkpoint-1250/rng_state.pth b/checkpoint-1250/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-1250/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-1250/scheduler.pt b/checkpoint-1250/scheduler.pt new file mode 100644 index 0000000..2cd86a4 --- /dev/null +++ b/checkpoint-1250/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c1af83dfb57c00b95469f8adf49114eb9cd66dccf7b4d28fd7852ee6575ba11 +size 1465 diff --git a/checkpoint-1250/tokenizer.json b/checkpoint-1250/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-1250/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-1250/tokenizer_config.json b/checkpoint-1250/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-1250/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1250/trainer_state.json b/checkpoint-1250/trainer_state.json new file mode 100644 index 0000000..4939651 --- /dev/null +++ b/checkpoint-1250/trainer_state.json @@ -0,0 +1,1284 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 1250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.09475322451908141, + "epoch": 0.008003201280512205, + "grad_norm": 424.0, + "learning_rate": 1.4516129032258066e-06, + "loss": 7.240174865722656, + "mean_token_accuracy": 0.5289772760123015, + "num_tokens": 42414.0, + "step": 10 + }, + { + "entropy": 0.10299791819415986, + "epoch": 0.01600640256102441, + "grad_norm": 232.0, + "learning_rate": 3.0645161290322584e-06, + "loss": 5.961273574829102, + "mean_token_accuracy": 0.6124999947845936, + "num_tokens": 83232.0, + "step": 20 + }, + { + "entropy": 0.08830865409690887, + "epoch": 0.024009603841536616, + "grad_norm": 300.0, + "learning_rate": 4.67741935483871e-06, + "loss": 3.4538761138916017, + "mean_token_accuracy": 0.6931818276643753, + "num_tokens": 123546.0, + "step": 30 + }, + { + "entropy": 0.08543417068431154, + "epoch": 0.03201280512204882, + "grad_norm": 19.75, + "learning_rate": 6.290322580645162e-06, + "loss": 0.5229705810546875, + "mean_token_accuracy": 0.8744318261742592, + "num_tokens": 165215.0, + "step": 40 + }, + { + "entropy": 0.06759846829809249, + "epoch": 0.040016006402561026, + "grad_norm": 28.25, + "learning_rate": 7.903225806451613e-06, + "loss": 0.09152845144271851, + "mean_token_accuracy": 0.9596590884029865, + "num_tokens": 206670.0, + "step": 50 + }, + { + "entropy": 0.06551291118375957, + "epoch": 0.04801920768307323, + "grad_norm": 3.109375, + "learning_rate": 9.516129032258065e-06, + "loss": 0.06548311114311219, + "mean_token_accuracy": 0.9602272696793079, + "num_tokens": 248738.0, + "step": 60 + }, + { + "entropy": 0.05871303144376725, + "epoch": 0.056022408963585436, + "grad_norm": 4.90625, + "learning_rate": 1.1129032258064516e-05, + "loss": 0.05862548351287842, + "mean_token_accuracy": 0.9624999977648259, + "num_tokens": 289455.0, + "step": 70 + }, + { + "entropy": 0.05400295334402472, + "epoch": 0.06402561024409764, + "grad_norm": 2.6875, + "learning_rate": 1.274193548387097e-05, + "loss": 0.05906534790992737, + "mean_token_accuracy": 0.9653409078717232, + "num_tokens": 328191.0, + "step": 80 + }, + { + "entropy": 0.05965230115689337, + "epoch": 0.07202881152460984, + "grad_norm": 2.03125, + "learning_rate": 1.4354838709677421e-05, + "loss": 0.06336120367050171, + "mean_token_accuracy": 0.9613636344671249, + "num_tokens": 370714.0, + "step": 90 + }, + { + "entropy": 0.0571730526862666, + "epoch": 0.08003201280512205, + "grad_norm": 4.1875, + "learning_rate": 1.596774193548387e-05, + "loss": 0.06329689621925354, + "mean_token_accuracy": 0.9653409034013748, + "num_tokens": 411158.0, + "step": 100 + }, + { + "entropy": 0.05321204150095582, + "epoch": 0.08803521408563425, + "grad_norm": 3.40625, + "learning_rate": 1.7580645161290325e-05, + "loss": 0.05498598217964172, + "mean_token_accuracy": 0.9681818149983883, + "num_tokens": 451650.0, + "step": 110 + }, + { + "entropy": 0.049898089445196095, + "epoch": 0.09603841536614646, + "grad_norm": 2.359375, + "learning_rate": 1.9193548387096777e-05, + "loss": 0.05433200597763062, + "mean_token_accuracy": 0.9721590876579285, + "num_tokens": 491529.0, + "step": 120 + }, + { + "entropy": 0.05273421558085829, + "epoch": 0.10404161664665866, + "grad_norm": 4.21875, + "learning_rate": 1.9911190053285968e-05, + "loss": 0.052664899826049806, + "mean_token_accuracy": 0.9704545423388481, + "num_tokens": 534477.0, + "step": 130 + }, + { + "entropy": 0.048857488168869166, + "epoch": 0.11204481792717087, + "grad_norm": 3.125, + "learning_rate": 1.9733570159857907e-05, + "loss": 0.05310600996017456, + "mean_token_accuracy": 0.9715909071266651, + "num_tokens": 573912.0, + "step": 140 + }, + { + "entropy": 0.054796313343103975, + "epoch": 0.12004801920768307, + "grad_norm": 3.109375, + "learning_rate": 1.9555950266429843e-05, + "loss": 0.05551998615264893, + "mean_token_accuracy": 0.9681818142533303, + "num_tokens": 613967.0, + "step": 150 + }, + { + "entropy": 0.050939775854931216, + "epoch": 0.12805122048819528, + "grad_norm": 4.90625, + "learning_rate": 1.9378330373001776e-05, + "loss": 0.05050772428512573, + "mean_token_accuracy": 0.969318176060915, + "num_tokens": 654625.0, + "step": 160 + }, + { + "entropy": 0.047000630758702755, + "epoch": 0.1360544217687075, + "grad_norm": 2.484375, + "learning_rate": 1.9200710479573715e-05, + "loss": 0.05440411567687988, + "mean_token_accuracy": 0.9687499947845936, + "num_tokens": 695387.0, + "step": 170 + }, + { + "entropy": 0.05339475516229868, + "epoch": 0.14405762304921968, + "grad_norm": 2.21875, + "learning_rate": 1.902309058614565e-05, + "loss": 0.05330321192741394, + "mean_token_accuracy": 0.9715909034013748, + "num_tokens": 736229.0, + "step": 180 + }, + { + "entropy": 0.05177106715273112, + "epoch": 0.1520608243297319, + "grad_norm": 2.390625, + "learning_rate": 1.8845470692717587e-05, + "loss": 0.053306227922439574, + "mean_token_accuracy": 0.9749999947845935, + "num_tokens": 777298.0, + "step": 190 + }, + { + "entropy": 0.04644646589877084, + "epoch": 0.1600640256102441, + "grad_norm": 3.21875, + "learning_rate": 1.8667850799289523e-05, + "loss": 0.046160414814949036, + "mean_token_accuracy": 0.9767045393586159, + "num_tokens": 816992.0, + "step": 200 + }, + { + "entropy": 0.03921595757419709, + "epoch": 0.16806722689075632, + "grad_norm": 2.953125, + "learning_rate": 1.8490230905861456e-05, + "loss": 0.046266698837280275, + "mean_token_accuracy": 0.9744318142533303, + "num_tokens": 856414.0, + "step": 210 + }, + { + "entropy": 0.04731454871071037, + "epoch": 0.1760704281712685, + "grad_norm": 3.703125, + "learning_rate": 1.8312611012433395e-05, + "loss": 0.049581846594810484, + "mean_token_accuracy": 0.9710227221250534, + "num_tokens": 897293.0, + "step": 220 + }, + { + "entropy": 0.050612482032738625, + "epoch": 0.1840736294517807, + "grad_norm": 3.046875, + "learning_rate": 1.813499111900533e-05, + "loss": 0.056270813941955565, + "mean_token_accuracy": 0.9653409041464329, + "num_tokens": 937566.0, + "step": 230 + }, + { + "entropy": 0.053894570120610295, + "epoch": 0.19207683073229292, + "grad_norm": 1.75, + "learning_rate": 1.7957371225577267e-05, + "loss": 0.05222816467285156, + "mean_token_accuracy": 0.9715909041464329, + "num_tokens": 979785.0, + "step": 240 + }, + { + "entropy": 0.04845360988983884, + "epoch": 0.2000800320128051, + "grad_norm": 1.6171875, + "learning_rate": 1.7779751332149203e-05, + "loss": 0.053466469049453735, + "mean_token_accuracy": 0.9670454487204552, + "num_tokens": 1019640.0, + "step": 250 + }, + { + "entropy": 0.049828451115172354, + "epoch": 0.20808323329331732, + "grad_norm": 1.625, + "learning_rate": 1.760213143872114e-05, + "loss": 0.05171402096748352, + "mean_token_accuracy": 0.9693181782960891, + "num_tokens": 1060823.0, + "step": 260 + }, + { + "entropy": 0.04814320435980335, + "epoch": 0.21608643457382953, + "grad_norm": 3.203125, + "learning_rate": 1.7424511545293075e-05, + "loss": 0.048413312435150145, + "mean_token_accuracy": 0.9801136322319508, + "num_tokens": 1102242.0, + "step": 270 + }, + { + "entropy": 0.045647610700689255, + "epoch": 0.22408963585434175, + "grad_norm": 2.484375, + "learning_rate": 1.724689165186501e-05, + "loss": 0.047308555245399474, + "mean_token_accuracy": 0.9738636329770088, + "num_tokens": 1141927.0, + "step": 280 + }, + { + "entropy": 0.04609768155496567, + "epoch": 0.23209283713485393, + "grad_norm": 2.40625, + "learning_rate": 1.7069271758436947e-05, + "loss": 0.05692839026451111, + "mean_token_accuracy": 0.9681818142533303, + "num_tokens": 1181811.0, + "step": 290 + }, + { + "entropy": 0.05474342008819803, + "epoch": 0.24009603841536614, + "grad_norm": 2.015625, + "learning_rate": 1.6891651865008883e-05, + "loss": 0.054971718788146974, + "mean_token_accuracy": 0.9653409063816071, + "num_tokens": 1223060.0, + "step": 300 + }, + { + "entropy": 0.05363150021876208, + "epoch": 0.24809923969587835, + "grad_norm": 1.6171875, + "learning_rate": 1.671403197158082e-05, + "loss": 0.05203201770782471, + "mean_token_accuracy": 0.9693181775510311, + "num_tokens": 1264243.0, + "step": 310 + }, + { + "entropy": 0.04393154126009904, + "epoch": 0.25610244097639057, + "grad_norm": 2.640625, + "learning_rate": 1.6536412078152755e-05, + "loss": 0.05061659216880798, + "mean_token_accuracy": 0.9704545415937901, + "num_tokens": 1304761.0, + "step": 320 + }, + { + "entropy": 0.0477942802477628, + "epoch": 0.26410564225690275, + "grad_norm": 2.75, + "learning_rate": 1.635879218472469e-05, + "loss": 0.04757603406906128, + "mean_token_accuracy": 0.9738636314868927, + "num_tokens": 1344735.0, + "step": 330 + }, + { + "entropy": 0.04573209672234953, + "epoch": 0.272108843537415, + "grad_norm": 2.453125, + "learning_rate": 1.6181172291296626e-05, + "loss": 0.04777190387248993, + "mean_token_accuracy": 0.9744318120181561, + "num_tokens": 1385052.0, + "step": 340 + }, + { + "entropy": 0.04807677125791088, + "epoch": 0.2801120448179272, + "grad_norm": 2.109375, + "learning_rate": 1.6003552397868562e-05, + "loss": 0.049048814177513125, + "mean_token_accuracy": 0.9693181782960891, + "num_tokens": 1425851.0, + "step": 350 + }, + { + "entropy": 0.04814378765877336, + "epoch": 0.28811524609843936, + "grad_norm": 2.109375, + "learning_rate": 1.58259325044405e-05, + "loss": 0.04974732995033264, + "mean_token_accuracy": 0.9698863595724105, + "num_tokens": 1466568.0, + "step": 360 + }, + { + "entropy": 0.049266617721877994, + "epoch": 0.2961184473789516, + "grad_norm": 2.296875, + "learning_rate": 1.5648312611012434e-05, + "loss": 0.052187031507492064, + "mean_token_accuracy": 0.9732954487204551, + "num_tokens": 1507415.0, + "step": 370 + }, + { + "entropy": 0.04980740246828645, + "epoch": 0.3041216486594638, + "grad_norm": 1.953125, + "learning_rate": 1.547069271758437e-05, + "loss": 0.04835581183433533, + "mean_token_accuracy": 0.9727272689342499, + "num_tokens": 1548371.0, + "step": 380 + }, + { + "entropy": 0.03997177838464268, + "epoch": 0.31212484993997597, + "grad_norm": 4.0, + "learning_rate": 1.5293072824156306e-05, + "loss": 0.04371632635593414, + "mean_token_accuracy": 0.9801136322319508, + "num_tokens": 1590062.0, + "step": 390 + }, + { + "entropy": 0.03653254228320293, + "epoch": 0.3201280512204882, + "grad_norm": 2.859375, + "learning_rate": 1.5115452930728242e-05, + "loss": 0.05101522207260132, + "mean_token_accuracy": 0.9715909041464329, + "num_tokens": 1632136.0, + "step": 400 + }, + { + "entropy": 0.0425920093410241, + "epoch": 0.3281312525010004, + "grad_norm": 2.015625, + "learning_rate": 1.4937833037300178e-05, + "loss": 0.04813604354858399, + "mean_token_accuracy": 0.9744318127632141, + "num_tokens": 1671462.0, + "step": 410 + }, + { + "entropy": 0.03970824745483696, + "epoch": 0.33613445378151263, + "grad_norm": 2.90625, + "learning_rate": 1.4760213143872116e-05, + "loss": 0.03939797580242157, + "mean_token_accuracy": 0.9812499947845936, + "num_tokens": 1712639.0, + "step": 420 + }, + { + "entropy": 0.042815997470461296, + "epoch": 0.3441376550620248, + "grad_norm": 1.453125, + "learning_rate": 1.458259325044405e-05, + "loss": 0.04754864275455475, + "mean_token_accuracy": 0.9744318135082721, + "num_tokens": 1753415.0, + "step": 430 + }, + { + "entropy": 0.0437370027997531, + "epoch": 0.352140856342537, + "grad_norm": 2.515625, + "learning_rate": 1.4404973357015988e-05, + "loss": 0.04742822349071503, + "mean_token_accuracy": 0.9732954494655133, + "num_tokens": 1792637.0, + "step": 440 + }, + { + "entropy": 0.04030659813288366, + "epoch": 0.36014405762304924, + "grad_norm": 3.796875, + "learning_rate": 1.4227353463587924e-05, + "loss": 0.04023580253124237, + "mean_token_accuracy": 0.9801136322319508, + "num_tokens": 1830990.0, + "step": 450 + }, + { + "entropy": 0.03681652669911273, + "epoch": 0.3681472589035614, + "grad_norm": 2.828125, + "learning_rate": 1.4049733570159858e-05, + "loss": 0.04969809353351593, + "mean_token_accuracy": 0.9721590869128705, + "num_tokens": 1870848.0, + "step": 460 + }, + { + "entropy": 0.04123187513323501, + "epoch": 0.3761504601840736, + "grad_norm": 2.21875, + "learning_rate": 1.3872113676731796e-05, + "loss": 0.043952861428260805, + "mean_token_accuracy": 0.9761363573372364, + "num_tokens": 1911638.0, + "step": 470 + }, + { + "entropy": 0.0450622973847203, + "epoch": 0.38415366146458585, + "grad_norm": 2.328125, + "learning_rate": 1.369449378330373e-05, + "loss": 0.045897161960601805, + "mean_token_accuracy": 0.9738636322319507, + "num_tokens": 1950406.0, + "step": 480 + }, + { + "entropy": 0.04427487466891762, + "epoch": 0.39215686274509803, + "grad_norm": 2.25, + "learning_rate": 1.3516873889875667e-05, + "loss": 0.04462568461894989, + "mean_token_accuracy": 0.9732954502105713, + "num_tokens": 1990786.0, + "step": 490 + }, + { + "entropy": 0.04660661657107994, + "epoch": 0.4001600640256102, + "grad_norm": 2.859375, + "learning_rate": 1.3339253996447603e-05, + "loss": 0.04628114402294159, + "mean_token_accuracy": 0.9738636322319507, + "num_tokens": 2030170.0, + "step": 500 + }, + { + "entropy": 0.04326281558896881, + "epoch": 0.40816326530612246, + "grad_norm": 2.171875, + "learning_rate": 1.3161634103019538e-05, + "loss": 0.05016361474990845, + "mean_token_accuracy": 0.9738636314868927, + "num_tokens": 2070211.0, + "step": 510 + }, + { + "entropy": 0.04552530206856318, + "epoch": 0.41616646658663464, + "grad_norm": 4.09375, + "learning_rate": 1.2984014209591475e-05, + "loss": 0.048873734474182126, + "mean_token_accuracy": 0.9744318135082721, + "num_tokens": 2110242.0, + "step": 520 + }, + { + "entropy": 0.048664613376604396, + "epoch": 0.4241696678671469, + "grad_norm": 5.96875, + "learning_rate": 1.2806394316163411e-05, + "loss": 0.051538127660751346, + "mean_token_accuracy": 0.9738636314868927, + "num_tokens": 2149695.0, + "step": 530 + }, + { + "entropy": 0.0454276799748186, + "epoch": 0.43217286914765907, + "grad_norm": 2.390625, + "learning_rate": 1.2628774422735349e-05, + "loss": 0.04872589409351349, + "mean_token_accuracy": 0.9721590876579285, + "num_tokens": 2191269.0, + "step": 540 + }, + { + "entropy": 0.04385605392162688, + "epoch": 0.44017607042817125, + "grad_norm": 3.546875, + "learning_rate": 1.2451154529307283e-05, + "loss": 0.049637827277183535, + "mean_token_accuracy": 0.9755681768059731, + "num_tokens": 2233167.0, + "step": 550 + }, + { + "entropy": 0.047120464290492235, + "epoch": 0.4481792717086835, + "grad_norm": 2.0625, + "learning_rate": 1.227353463587922e-05, + "loss": 0.047938373684883115, + "mean_token_accuracy": 0.9715909048914909, + "num_tokens": 2273683.0, + "step": 560 + }, + { + "entropy": 0.04529402129119262, + "epoch": 0.4561824729891957, + "grad_norm": 1.8671875, + "learning_rate": 1.2095914742451155e-05, + "loss": 0.0421001136302948, + "mean_token_accuracy": 0.9789772681891918, + "num_tokens": 2312821.0, + "step": 570 + }, + { + "entropy": 0.040198929510370365, + "epoch": 0.46418567426970786, + "grad_norm": 4.125, + "learning_rate": 1.1918294849023091e-05, + "loss": 0.045373284816741945, + "mean_token_accuracy": 0.9772727213799953, + "num_tokens": 2354833.0, + "step": 580 + }, + { + "entropy": 0.04357797284028493, + "epoch": 0.4721888755502201, + "grad_norm": 1.1875, + "learning_rate": 1.1740674955595029e-05, + "loss": 0.05674033761024475, + "mean_token_accuracy": 0.9693181782960891, + "num_tokens": 2395646.0, + "step": 590 + }, + { + "entropy": 0.04933104421943426, + "epoch": 0.4801920768307323, + "grad_norm": 0.9375, + "learning_rate": 1.1563055062166963e-05, + "loss": 0.052300095558166504, + "mean_token_accuracy": 0.9698863595724105, + "num_tokens": 2434038.0, + "step": 600 + }, + { + "entropy": 0.04834912738297135, + "epoch": 0.4881952781112445, + "grad_norm": 4.40625, + "learning_rate": 1.1385435168738899e-05, + "loss": 0.04378367662429809, + "mean_token_accuracy": 0.977840905636549, + "num_tokens": 2473548.0, + "step": 610 + }, + { + "entropy": 0.03797003252257127, + "epoch": 0.4961984793917567, + "grad_norm": 3.703125, + "learning_rate": 1.1207815275310837e-05, + "loss": 0.04411846697330475, + "mean_token_accuracy": 0.9761363610625267, + "num_tokens": 2513657.0, + "step": 620 + }, + { + "entropy": 0.037305073122115576, + "epoch": 0.5042016806722689, + "grad_norm": 2.265625, + "learning_rate": 1.1030195381882771e-05, + "loss": 0.0530417799949646, + "mean_token_accuracy": 0.9772727228701115, + "num_tokens": 2553049.0, + "step": 630 + }, + { + "entropy": 0.043729909125249836, + "epoch": 0.5122048819527811, + "grad_norm": 2.375, + "learning_rate": 1.0852575488454709e-05, + "loss": 0.05261260867118835, + "mean_token_accuracy": 0.9698863580822945, + "num_tokens": 2593026.0, + "step": 640 + }, + { + "entropy": 0.04822997532319277, + "epoch": 0.5202080832332934, + "grad_norm": 1.3828125, + "learning_rate": 1.0674955595026645e-05, + "loss": 0.05205248594284058, + "mean_token_accuracy": 0.9727272666990757, + "num_tokens": 2633344.0, + "step": 650 + }, + { + "entropy": 0.04994500142056495, + "epoch": 0.5282112845138055, + "grad_norm": 2.484375, + "learning_rate": 1.0497335701598579e-05, + "loss": 0.05130913853645325, + "mean_token_accuracy": 0.9693181797862053, + "num_tokens": 2672721.0, + "step": 660 + }, + { + "entropy": 0.04809544003510382, + "epoch": 0.5362144857943177, + "grad_norm": 1.7734375, + "learning_rate": 1.0319715808170516e-05, + "loss": 0.043288156390190125, + "mean_token_accuracy": 0.9772727236151695, + "num_tokens": 2714199.0, + "step": 670 + }, + { + "entropy": 0.041754112322814764, + "epoch": 0.54421768707483, + "grad_norm": 2.125, + "learning_rate": 1.014209591474245e-05, + "loss": 0.04941850304603577, + "mean_token_accuracy": 0.9710227191448212, + "num_tokens": 2757389.0, + "step": 680 + }, + { + "entropy": 0.04491965179331601, + "epoch": 0.5522208883553421, + "grad_norm": 2.421875, + "learning_rate": 9.964476021314388e-06, + "loss": 0.048098990321159364, + "mean_token_accuracy": 0.975568176060915, + "num_tokens": 2797266.0, + "step": 690 + }, + { + "entropy": 0.04177754216361791, + "epoch": 0.5602240896358543, + "grad_norm": 1.8828125, + "learning_rate": 9.786856127886324e-06, + "loss": 0.03804603815078735, + "mean_token_accuracy": 0.9818181775510311, + "num_tokens": 2837178.0, + "step": 700 + }, + { + "entropy": 0.04215870927291689, + "epoch": 0.5682272909163666, + "grad_norm": 3.109375, + "learning_rate": 9.60923623445826e-06, + "loss": 0.040096724033355714, + "mean_token_accuracy": 0.9806818142533302, + "num_tokens": 2876752.0, + "step": 710 + }, + { + "entropy": 0.040034703072160484, + "epoch": 0.5762304921968787, + "grad_norm": 2.5625, + "learning_rate": 9.431616341030196e-06, + "loss": 0.049453753232955935, + "mean_token_accuracy": 0.97443181052804, + "num_tokens": 2918121.0, + "step": 720 + }, + { + "entropy": 0.04085657097748481, + "epoch": 0.584233693477391, + "grad_norm": 3.109375, + "learning_rate": 9.253996447602132e-06, + "loss": 0.05398496389389038, + "mean_token_accuracy": 0.9744318142533303, + "num_tokens": 2957034.0, + "step": 730 + }, + { + "entropy": 0.045270191514282486, + "epoch": 0.5922368947579032, + "grad_norm": 2.203125, + "learning_rate": 9.076376554174068e-06, + "loss": 0.04788501560688019, + "mean_token_accuracy": 0.9710227236151695, + "num_tokens": 2996795.0, + "step": 740 + }, + { + "entropy": 0.046037832100410014, + "epoch": 0.6002400960384153, + "grad_norm": 2.171875, + "learning_rate": 8.898756660746004e-06, + "loss": 0.045071455836296084, + "mean_token_accuracy": 0.9772727221250535, + "num_tokens": 3036825.0, + "step": 750 + }, + { + "entropy": 0.04801415469264612, + "epoch": 0.6082432973189276, + "grad_norm": 2.34375, + "learning_rate": 8.72113676731794e-06, + "loss": 0.0479773223400116, + "mean_token_accuracy": 0.9710227243602276, + "num_tokens": 3076791.0, + "step": 760 + }, + { + "entropy": 0.0462384240119718, + "epoch": 0.6162464985994398, + "grad_norm": 2.453125, + "learning_rate": 8.543516873889876e-06, + "loss": 0.04638043344020844, + "mean_token_accuracy": 0.9761363558471203, + "num_tokens": 3120135.0, + "step": 770 + }, + { + "entropy": 0.045585069648222995, + "epoch": 0.6242496998799519, + "grad_norm": 2.15625, + "learning_rate": 8.365896980461814e-06, + "loss": 0.048996028304100034, + "mean_token_accuracy": 0.9721590884029865, + "num_tokens": 3161235.0, + "step": 780 + }, + { + "entropy": 0.04545295296120457, + "epoch": 0.6322529011604642, + "grad_norm": 3.234375, + "learning_rate": 8.188277087033748e-06, + "loss": 0.04392434060573578, + "mean_token_accuracy": 0.9744318127632141, + "num_tokens": 3200661.0, + "step": 790 + }, + { + "entropy": 0.0439292140188627, + "epoch": 0.6402561024409764, + "grad_norm": 5.03125, + "learning_rate": 8.010657193605684e-06, + "loss": 0.04489242136478424, + "mean_token_accuracy": 0.9744318149983883, + "num_tokens": 3239228.0, + "step": 800 + }, + { + "entropy": 0.04578416767762974, + "epoch": 0.6482593037214885, + "grad_norm": 3.515625, + "learning_rate": 7.83303730017762e-06, + "loss": 0.050331324338912964, + "mean_token_accuracy": 0.9755681790411472, + "num_tokens": 3278112.0, + "step": 810 + }, + { + "entropy": 0.04590382124297321, + "epoch": 0.6562625050020008, + "grad_norm": 1.4453125, + "learning_rate": 7.655417406749557e-06, + "loss": 0.048855268955230714, + "mean_token_accuracy": 0.9761363595724106, + "num_tokens": 3319328.0, + "step": 820 + }, + { + "entropy": 0.04355756618315354, + "epoch": 0.664265706282513, + "grad_norm": 2.75, + "learning_rate": 7.4777975133214935e-06, + "loss": 0.04029326438903809, + "mean_token_accuracy": 0.9789772689342499, + "num_tokens": 3359897.0, + "step": 830 + }, + { + "entropy": 0.04515375569462776, + "epoch": 0.6722689075630253, + "grad_norm": 2.75, + "learning_rate": 7.3001776198934286e-06, + "loss": 0.045958060026168826, + "mean_token_accuracy": 0.9744318142533303, + "num_tokens": 3400717.0, + "step": 840 + }, + { + "entropy": 0.041079553513554856, + "epoch": 0.6802721088435374, + "grad_norm": 2.03125, + "learning_rate": 7.1225577264653645e-06, + "loss": 0.04416506886482239, + "mean_token_accuracy": 0.974431811273098, + "num_tokens": 3439852.0, + "step": 850 + }, + { + "entropy": 0.043181278416886924, + "epoch": 0.6882753101240496, + "grad_norm": 3.03125, + "learning_rate": 6.9449378330373005e-06, + "loss": 0.04391254186630249, + "mean_token_accuracy": 0.9789772689342499, + "num_tokens": 3479654.0, + "step": 860 + }, + { + "entropy": 0.04434247007593513, + "epoch": 0.6962785114045619, + "grad_norm": 1.8046875, + "learning_rate": 6.767317939609237e-06, + "loss": 0.04593832492828369, + "mean_token_accuracy": 0.9744318164885044, + "num_tokens": 3520077.0, + "step": 870 + }, + { + "entropy": 0.04393934286781587, + "epoch": 0.704281712685074, + "grad_norm": 1.6640625, + "learning_rate": 6.589698046181173e-06, + "loss": 0.046162942051887514, + "mean_token_accuracy": 0.977840905636549, + "num_tokens": 3560952.0, + "step": 880 + }, + { + "entropy": 0.04364555573556572, + "epoch": 0.7122849139655862, + "grad_norm": 1.640625, + "learning_rate": 6.412078152753108e-06, + "loss": 0.0497223436832428, + "mean_token_accuracy": 0.9744318142533303, + "num_tokens": 3600585.0, + "step": 890 + }, + { + "entropy": 0.04233085287705762, + "epoch": 0.7202881152460985, + "grad_norm": 1.7578125, + "learning_rate": 6.234458259325044e-06, + "loss": 0.03990237712860108, + "mean_token_accuracy": 0.9801136337220668, + "num_tokens": 3643430.0, + "step": 900 + }, + { + "entropy": 0.04045454500592314, + "epoch": 0.7282913165266106, + "grad_norm": 3.21875, + "learning_rate": 6.056838365896981e-06, + "loss": 0.04266665279865265, + "mean_token_accuracy": 0.9784090861678123, + "num_tokens": 3684717.0, + "step": 910 + }, + { + "entropy": 0.041357538173906506, + "epoch": 0.7362945178071229, + "grad_norm": 2.390625, + "learning_rate": 5.879218472468917e-06, + "loss": 0.04428337216377258, + "mean_token_accuracy": 0.9789772689342499, + "num_tokens": 3724155.0, + "step": 920 + }, + { + "entropy": 0.04306626642246556, + "epoch": 0.7442977190876351, + "grad_norm": 1.75, + "learning_rate": 5.701598579040853e-06, + "loss": 0.048436781764030455, + "mean_token_accuracy": 0.9749999940395355, + "num_tokens": 3765538.0, + "step": 930 + }, + { + "entropy": 0.043767763464711604, + "epoch": 0.7523009203681472, + "grad_norm": 1.5234375, + "learning_rate": 5.523978685612789e-06, + "loss": 0.05207340121269226, + "mean_token_accuracy": 0.9749999955296517, + "num_tokens": 3805016.0, + "step": 940 + }, + { + "entropy": 0.04864149287022883, + "epoch": 0.7603041216486595, + "grad_norm": 3.34375, + "learning_rate": 5.346358792184725e-06, + "loss": 0.05412702560424805, + "mean_token_accuracy": 0.9710227236151695, + "num_tokens": 3844988.0, + "step": 950 + }, + { + "entropy": 0.045417454978451136, + "epoch": 0.7683073229291717, + "grad_norm": 2.109375, + "learning_rate": 5.168738898756661e-06, + "loss": 0.039266571402549744, + "mean_token_accuracy": 0.9818181782960892, + "num_tokens": 3884166.0, + "step": 960 + }, + { + "entropy": 0.047192305256612596, + "epoch": 0.7763105242096838, + "grad_norm": 4.125, + "learning_rate": 4.991119005328598e-06, + "loss": 0.0472522109746933, + "mean_token_accuracy": 0.9778409041464329, + "num_tokens": 3922599.0, + "step": 970 + }, + { + "entropy": 0.044715805305168034, + "epoch": 0.7843137254901961, + "grad_norm": 2.03125, + "learning_rate": 4.813499111900533e-06, + "loss": 0.043560588359832765, + "mean_token_accuracy": 0.9727272681891919, + "num_tokens": 3961577.0, + "step": 980 + }, + { + "entropy": 0.04465018446790055, + "epoch": 0.7923169267707083, + "grad_norm": 1.6171875, + "learning_rate": 4.63587921847247e-06, + "loss": 0.04594630300998688, + "mean_token_accuracy": 0.9732954502105713, + "num_tokens": 4001019.0, + "step": 990 + }, + { + "entropy": 0.04393655447638593, + "epoch": 0.8003201280512204, + "grad_norm": 1.7890625, + "learning_rate": 4.458259325044405e-06, + "loss": 0.04262282252311707, + "mean_token_accuracy": 0.9778409048914909, + "num_tokens": 4041721.0, + "step": 1000 + }, + { + "entropy": 0.042122538023249945, + "epoch": 0.8083233293317327, + "grad_norm": 1.8515625, + "learning_rate": 4.2806394316163415e-06, + "loss": 0.04339278638362885, + "mean_token_accuracy": 0.9784090869128704, + "num_tokens": 4082654.0, + "step": 1010 + }, + { + "entropy": 0.04523669999325648, + "epoch": 0.8163265306122449, + "grad_norm": 2.0625, + "learning_rate": 4.1030195381882775e-06, + "loss": 0.048701035976409915, + "mean_token_accuracy": 0.9704545423388481, + "num_tokens": 4123109.0, + "step": 1020 + }, + { + "entropy": 0.04366708064917475, + "epoch": 0.8243297318927572, + "grad_norm": 3.1875, + "learning_rate": 3.9253996447602135e-06, + "loss": 0.043869239091873166, + "mean_token_accuracy": 0.9749999977648258, + "num_tokens": 4162927.0, + "step": 1030 + }, + { + "entropy": 0.043506618012906985, + "epoch": 0.8323329331732693, + "grad_norm": 1.5859375, + "learning_rate": 3.7477797513321494e-06, + "loss": 0.04709920883178711, + "mean_token_accuracy": 0.9772727221250535, + "num_tokens": 4202732.0, + "step": 1040 + }, + { + "entropy": 0.04336636826628819, + "epoch": 0.8403361344537815, + "grad_norm": 2.609375, + "learning_rate": 3.570159857904086e-06, + "loss": 0.04854160845279694, + "mean_token_accuracy": 0.9744318127632141, + "num_tokens": 4242673.0, + "step": 1050 + }, + { + "entropy": 0.0459871404862497, + "epoch": 0.8483393357342938, + "grad_norm": 2.359375, + "learning_rate": 3.3925399644760213e-06, + "loss": 0.0448982447385788, + "mean_token_accuracy": 0.9772727236151695, + "num_tokens": 4283487.0, + "step": 1060 + }, + { + "entropy": 0.04385776598937809, + "epoch": 0.8563425370148059, + "grad_norm": 1.140625, + "learning_rate": 3.2149200710479577e-06, + "loss": 0.043155476450920105, + "mean_token_accuracy": 0.97670454159379, + "num_tokens": 4322572.0, + "step": 1070 + }, + { + "entropy": 0.04259421322203707, + "epoch": 0.8643457382953181, + "grad_norm": 1.4765625, + "learning_rate": 3.0373001776198937e-06, + "loss": 0.03880903720855713, + "mean_token_accuracy": 0.9846590884029865, + "num_tokens": 4363658.0, + "step": 1080 + }, + { + "entropy": 0.04217607219179627, + "epoch": 0.8723489395758304, + "grad_norm": 1.625, + "learning_rate": 2.8596802841918296e-06, + "loss": 0.04083235561847687, + "mean_token_accuracy": 0.9755681775510311, + "num_tokens": 4405021.0, + "step": 1090 + }, + { + "entropy": 0.04688679115497507, + "epoch": 0.8803521408563425, + "grad_norm": 3.734375, + "learning_rate": 2.682060390763766e-06, + "loss": 0.05114847421646118, + "mean_token_accuracy": 0.9687499970197677, + "num_tokens": 4445257.0, + "step": 1100 + }, + { + "entropy": 0.04287364967749454, + "epoch": 0.8883553421368547, + "grad_norm": 1.9765625, + "learning_rate": 2.5044404973357016e-06, + "loss": 0.04361231029033661, + "mean_token_accuracy": 0.9784090854227543, + "num_tokens": 4485832.0, + "step": 1110 + }, + { + "entropy": 0.04545789617113769, + "epoch": 0.896358543417367, + "grad_norm": 1.8984375, + "learning_rate": 2.326820603907638e-06, + "loss": 0.04764930307865143, + "mean_token_accuracy": 0.9704545415937901, + "num_tokens": 4526200.0, + "step": 1120 + }, + { + "entropy": 0.04260573259089142, + "epoch": 0.9043617446978791, + "grad_norm": 1.375, + "learning_rate": 2.149200710479574e-06, + "loss": 0.04154365062713623, + "mean_token_accuracy": 0.9772727213799953, + "num_tokens": 4566364.0, + "step": 1130 + }, + { + "entropy": 0.04537807885790244, + "epoch": 0.9123649459783914, + "grad_norm": 1.546875, + "learning_rate": 1.97158081705151e-06, + "loss": 0.04880580902099609, + "mean_token_accuracy": 0.9698863588273525, + "num_tokens": 4607256.0, + "step": 1140 + }, + { + "entropy": 0.043002552184043454, + "epoch": 0.9203681472589036, + "grad_norm": 1.15625, + "learning_rate": 1.793960923623446e-06, + "loss": 0.04373497366905212, + "mean_token_accuracy": 0.9749999925494194, + "num_tokens": 4647531.0, + "step": 1150 + }, + { + "entropy": 0.042623581358930095, + "epoch": 0.9283713485394157, + "grad_norm": 3.578125, + "learning_rate": 1.616341030195382e-06, + "loss": 0.04597070813179016, + "mean_token_accuracy": 0.9755681745707989, + "num_tokens": 4688875.0, + "step": 1160 + }, + { + "entropy": 0.04425350625788269, + "epoch": 0.936374549819928, + "grad_norm": 4.90625, + "learning_rate": 1.4387211367673181e-06, + "loss": 0.049137341976165774, + "mean_token_accuracy": 0.9755681782960892, + "num_tokens": 4730185.0, + "step": 1170 + }, + { + "entropy": 0.04330989782029064, + "epoch": 0.9443777511004402, + "grad_norm": 1.9765625, + "learning_rate": 1.261101243339254e-06, + "loss": 0.042343682050704955, + "mean_token_accuracy": 0.975568176060915, + "num_tokens": 4767986.0, + "step": 1180 + }, + { + "entropy": 0.04626658609486185, + "epoch": 0.9523809523809523, + "grad_norm": 1.9375, + "learning_rate": 1.08348134991119e-06, + "loss": 0.05028271675109863, + "mean_token_accuracy": 0.9744318135082721, + "num_tokens": 4807784.0, + "step": 1190 + }, + { + "entropy": 0.040159306283749176, + "epoch": 0.9603841536614646, + "grad_norm": 5.78125, + "learning_rate": 9.058614564831261e-07, + "loss": 0.04466935396194458, + "mean_token_accuracy": 0.9801136322319508, + "num_tokens": 4851222.0, + "step": 1200 + }, + { + "entropy": 0.040738246936234644, + "epoch": 0.9683873549419768, + "grad_norm": 2.046875, + "learning_rate": 7.282415630550623e-07, + "loss": 0.04467435479164124, + "mean_token_accuracy": 0.9778409041464329, + "num_tokens": 4889974.0, + "step": 1210 + }, + { + "entropy": 0.044979390571825206, + "epoch": 0.976390556222489, + "grad_norm": 2.921875, + "learning_rate": 5.506216696269983e-07, + "loss": 0.04703192114830017, + "mean_token_accuracy": 0.9727272681891919, + "num_tokens": 4931481.0, + "step": 1220 + }, + { + "entropy": 0.0433165812632069, + "epoch": 0.9843937575030012, + "grad_norm": 2.09375, + "learning_rate": 3.7300177619893427e-07, + "loss": 0.051091307401657106, + "mean_token_accuracy": 0.9704545430839062, + "num_tokens": 4972119.0, + "step": 1230 + }, + { + "entropy": 0.04314052956688101, + "epoch": 0.9923969587835134, + "grad_norm": 2.09375, + "learning_rate": 1.9538188277087038e-07, + "loss": 0.04473470449447632, + "mean_token_accuracy": 0.9715909063816071, + "num_tokens": 5013093.0, + "step": 1240 + }, + { + "entropy": 0.04248538885976335, + "epoch": 1.0, + "grad_norm": 3.421875, + "learning_rate": 1.7761989342806397e-08, + "loss": 0.04711778163909912, + "mean_token_accuracy": 0.9718899483743467, + "num_tokens": 5051976.0, + "step": 1250 + } + ], + "logging_steps": 10, + "max_steps": 1250, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.070355748382106e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1250/training_args.bin b/checkpoint-1250/training_args.bin new file mode 100644 index 0000000..028f348 --- /dev/null +++ b/checkpoint-1250/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:830344022dac853c6aa8a774b2ca8759c5c6bb43031383a453be97a14f9ea98b +size 5713 diff --git a/checkpoint-1425/chat_template.jinja b/checkpoint-1425/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-1425/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1425/config.json b/checkpoint-1425/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-1425/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-1425/generation_config.json b/checkpoint-1425/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-1425/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-1425/model.safetensors b/checkpoint-1425/model.safetensors new file mode 100644 index 0000000..4b1bad1 --- /dev/null +++ b/checkpoint-1425/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:226a1b1fc43c0a689697c97ea0c254106fffcbd4ee8cb6e067eb4ec1a8b90723 +size 3441185608 diff --git a/checkpoint-1425/optimizer.pt b/checkpoint-1425/optimizer.pt new file mode 100644 index 0000000..7172eb9 --- /dev/null +++ b/checkpoint-1425/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f586a4876b6c420354c724ce40a4ff3106c42697a1a91076dd05627ea1d13b +size 6882572207 diff --git a/checkpoint-1425/rng_state.pth b/checkpoint-1425/rng_state.pth new file mode 100644 index 0000000..bd3c190 --- /dev/null +++ b/checkpoint-1425/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934 +size 14645 diff --git a/checkpoint-1425/scheduler.pt b/checkpoint-1425/scheduler.pt new file mode 100644 index 0000000..85cc331 --- /dev/null +++ b/checkpoint-1425/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03538dd80fe2500a28be78a5081deb22d6099c450f9312696ac5e1b7a11e67dc +size 1465 diff --git a/checkpoint-1425/tokenizer.json b/checkpoint-1425/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-1425/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-1425/tokenizer_config.json b/checkpoint-1425/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-1425/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1425/trainer_state.json b/checkpoint-1425/trainer_state.json new file mode 100644 index 0000000..68f505a --- /dev/null +++ b/checkpoint-1425/trainer_state.json @@ -0,0 +1,1454 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1425, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.8691443920135498, + "epoch": 0.021085925144965736, + "grad_norm": 47.5, + "learning_rate": 7.2e-07, + "loss": 2.1268569946289064, + "mean_token_accuracy": 0.5856592752039432, + "num_tokens": 53385.0, + "step": 10 + }, + { + "entropy": 0.9018128983676433, + "epoch": 0.04217185028993147, + "grad_norm": 49.25, + "learning_rate": 1.52e-06, + "loss": 2.1235822677612304, + "mean_token_accuracy": 0.5826416034251451, + "num_tokens": 108949.0, + "step": 20 + }, + { + "entropy": 0.9313402399420738, + "epoch": 0.0632577754348972, + "grad_norm": 38.25, + "learning_rate": 2.3200000000000002e-06, + "loss": 1.9997358322143555, + "mean_token_accuracy": 0.5817345686256885, + "num_tokens": 162034.0, + "step": 30 + }, + { + "entropy": 0.9819381944835186, + "epoch": 0.08434370057986294, + "grad_norm": 28.25, + "learning_rate": 3.12e-06, + "loss": 1.8760208129882812, + "mean_token_accuracy": 0.5874959267675877, + "num_tokens": 214165.0, + "step": 40 + }, + { + "entropy": 1.0624087303876877, + "epoch": 0.10542962572482868, + "grad_norm": 14.1875, + "learning_rate": 3.920000000000001e-06, + "loss": 1.6169570922851562, + "mean_token_accuracy": 0.601640059798956, + "num_tokens": 265527.0, + "step": 50 + }, + { + "entropy": 1.1721510097384453, + "epoch": 0.1265155508697944, + "grad_norm": 11.3125, + "learning_rate": 4.7200000000000005e-06, + "loss": 1.4622878074645995, + "mean_token_accuracy": 0.6209174185991287, + "num_tokens": 319341.0, + "step": 60 + }, + { + "entropy": 1.2009219758212566, + "epoch": 0.14760147601476015, + "grad_norm": 9.0625, + "learning_rate": 5.5200000000000005e-06, + "loss": 1.2933337211608886, + "mean_token_accuracy": 0.6467074818909169, + "num_tokens": 373709.0, + "step": 70 + }, + { + "entropy": 1.1439887225627898, + "epoch": 0.16868740115972589, + "grad_norm": 9.3125, + "learning_rate": 6.3200000000000005e-06, + "loss": 1.2084498405456543, + "mean_token_accuracy": 0.6660686552524566, + "num_tokens": 425923.0, + "step": 80 + }, + { + "entropy": 1.0713621236383915, + "epoch": 0.18977332630469163, + "grad_norm": 7.84375, + "learning_rate": 7.1200000000000004e-06, + "loss": 1.1175435066223145, + "mean_token_accuracy": 0.6812180563807487, + "num_tokens": 480012.0, + "step": 90 + }, + { + "entropy": 1.0369222231209279, + "epoch": 0.21085925144965736, + "grad_norm": 7.09375, + "learning_rate": 7.92e-06, + "loss": 1.062850570678711, + "mean_token_accuracy": 0.6899570092558861, + "num_tokens": 533890.0, + "step": 100 + }, + { + "entropy": 0.9894454136490822, + "epoch": 0.23194517659462308, + "grad_norm": 7.75, + "learning_rate": 8.720000000000001e-06, + "loss": 1.0150799751281738, + "mean_token_accuracy": 0.7009239844977856, + "num_tokens": 590654.0, + "step": 110 + }, + { + "entropy": 0.9410129584372043, + "epoch": 0.2530311017395888, + "grad_norm": 7.28125, + "learning_rate": 9.52e-06, + "loss": 0.9810515403747558, + "mean_token_accuracy": 0.709618617594242, + "num_tokens": 643080.0, + "step": 120 + }, + { + "entropy": 0.9265115469694137, + "epoch": 0.2741170268845546, + "grad_norm": 7.46875, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.9485825538635254, + "mean_token_accuracy": 0.7162063598632813, + "num_tokens": 697254.0, + "step": 130 + }, + { + "entropy": 0.9009825974702835, + "epoch": 0.2952029520295203, + "grad_norm": 6.96875, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.9119878768920898, + "mean_token_accuracy": 0.7238413639366627, + "num_tokens": 751774.0, + "step": 140 + }, + { + "entropy": 0.9040703341364861, + "epoch": 0.316288877174486, + "grad_norm": 7.21875, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.9283308029174805, + "mean_token_accuracy": 0.7166316144168376, + "num_tokens": 808927.0, + "step": 150 + }, + { + "entropy": 0.8878034949302673, + "epoch": 0.33737480231945177, + "grad_norm": 7.375, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.9068665504455566, + "mean_token_accuracy": 0.7254431843757629, + "num_tokens": 858559.0, + "step": 160 + }, + { + "entropy": 0.887480016052723, + "epoch": 0.3584607274644175, + "grad_norm": 7.65625, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.8971159934997559, + "mean_token_accuracy": 0.7258390121161937, + "num_tokens": 912285.0, + "step": 170 + }, + { + "entropy": 0.8527634374797344, + "epoch": 0.37954665260938325, + "grad_norm": 7.6875, + "learning_rate": 1.432e-05, + "loss": 0.8586221694946289, + "mean_token_accuracy": 0.7359919100999832, + "num_tokens": 969385.0, + "step": 180 + }, + { + "entropy": 0.8445627830922604, + "epoch": 0.40063257775434896, + "grad_norm": 7.03125, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.855714225769043, + "mean_token_accuracy": 0.7341729424893856, + "num_tokens": 1026106.0, + "step": 190 + }, + { + "entropy": 0.841130618005991, + "epoch": 0.42171850289931473, + "grad_norm": 7.21875, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.8620488166809082, + "mean_token_accuracy": 0.7361006513237953, + "num_tokens": 1079743.0, + "step": 200 + }, + { + "entropy": 0.831258487701416, + "epoch": 0.44280442804428044, + "grad_norm": 7.0, + "learning_rate": 1.672e-05, + "loss": 0.8441590309143067, + "mean_token_accuracy": 0.7379129216074943, + "num_tokens": 1131453.0, + "step": 210 + }, + { + "entropy": 0.79731200709939, + "epoch": 0.46389035318924615, + "grad_norm": 7.875, + "learning_rate": 1.752e-05, + "loss": 0.8135190963745117, + "mean_token_accuracy": 0.7497627973556519, + "num_tokens": 1182486.0, + "step": 220 + }, + { + "entropy": 0.8269097700715065, + "epoch": 0.4849762783342119, + "grad_norm": 7.5625, + "learning_rate": 1.832e-05, + "loss": 0.8399410247802734, + "mean_token_accuracy": 0.7389348305761814, + "num_tokens": 1238136.0, + "step": 230 + }, + { + "entropy": 0.8093138128519058, + "epoch": 0.5060622034791776, + "grad_norm": 7.40625, + "learning_rate": 1.912e-05, + "loss": 0.810025691986084, + "mean_token_accuracy": 0.7460841409862041, + "num_tokens": 1294939.0, + "step": 240 + }, + { + "entropy": 0.8069386176764966, + "epoch": 0.5271481286241434, + "grad_norm": 6.84375, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8215888023376465, + "mean_token_accuracy": 0.7438369721174241, + "num_tokens": 1346462.0, + "step": 250 + }, + { + "entropy": 0.8088659413158894, + "epoch": 0.5482340537691092, + "grad_norm": 6.75, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8201449394226075, + "mean_token_accuracy": 0.7393839374184609, + "num_tokens": 1397444.0, + "step": 260 + }, + { + "entropy": 0.7969542622566224, + "epoch": 0.5693199789140748, + "grad_norm": 6.59375, + "learning_rate": 1.9831111111111113e-05, + "loss": 0.8009759902954101, + "mean_token_accuracy": 0.7512488305568695, + "num_tokens": 1453900.0, + "step": 270 + }, + { + "entropy": 0.780416651815176, + "epoch": 0.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.9742222222222223e-05, + "loss": 0.7874263763427735, + "mean_token_accuracy": 0.7526524804532528, + "num_tokens": 1505529.0, + "step": 280 + }, + { + "entropy": 0.7752278268337249, + "epoch": 0.6114918292040064, + "grad_norm": 6.875, + "learning_rate": 1.9653333333333334e-05, + "loss": 0.8075678825378418, + "mean_token_accuracy": 0.746569037437439, + "num_tokens": 1562473.0, + "step": 290 + }, + { + "entropy": 0.7895354829728604, + "epoch": 0.632577754348972, + "grad_norm": 6.6875, + "learning_rate": 1.9564444444444444e-05, + "loss": 0.7880948543548584, + "mean_token_accuracy": 0.7494045957922936, + "num_tokens": 1613320.0, + "step": 300 + }, + { + "entropy": 0.7653828673064709, + "epoch": 0.6536636794939378, + "grad_norm": 6.78125, + "learning_rate": 1.9475555555555558e-05, + "loss": 0.7780828475952148, + "mean_token_accuracy": 0.7556171745061875, + "num_tokens": 1662847.0, + "step": 310 + }, + { + "entropy": 0.7652887552976608, + "epoch": 0.6747496046389035, + "grad_norm": 6.375, + "learning_rate": 1.938666666666667e-05, + "loss": 0.7539787292480469, + "mean_token_accuracy": 0.7582654722034932, + "num_tokens": 1718553.0, + "step": 320 + }, + { + "entropy": 0.7076033115386963, + "epoch": 0.6958355297838693, + "grad_norm": 7.15625, + "learning_rate": 1.929777777777778e-05, + "loss": 0.7396605491638184, + "mean_token_accuracy": 0.7659220136702061, + "num_tokens": 1771024.0, + "step": 330 + }, + { + "entropy": 0.7509547933936119, + "epoch": 0.716921454928835, + "grad_norm": 6.15625, + "learning_rate": 1.920888888888889e-05, + "loss": 0.7459388256072998, + "mean_token_accuracy": 0.7644543245434761, + "num_tokens": 1821303.0, + "step": 340 + }, + { + "entropy": 0.7591827407479286, + "epoch": 0.7380073800738007, + "grad_norm": 7.03125, + "learning_rate": 1.912e-05, + "loss": 0.7627717971801757, + "mean_token_accuracy": 0.7567056350409984, + "num_tokens": 1868674.0, + "step": 350 + }, + { + "entropy": 0.7316406652331352, + "epoch": 0.7590933052187665, + "grad_norm": 6.59375, + "learning_rate": 1.9031111111111114e-05, + "loss": 0.7425707340240478, + "mean_token_accuracy": 0.7653280735015869, + "num_tokens": 1924619.0, + "step": 360 + }, + { + "entropy": 0.7539067789912224, + "epoch": 0.7801792303637322, + "grad_norm": 7.0625, + "learning_rate": 1.8942222222222224e-05, + "loss": 0.7716987609863282, + "mean_token_accuracy": 0.7524559155106545, + "num_tokens": 1978795.0, + "step": 370 + }, + { + "entropy": 0.7687754213809967, + "epoch": 0.8012651555086979, + "grad_norm": 7.1875, + "learning_rate": 1.8853333333333335e-05, + "loss": 0.7734373569488525, + "mean_token_accuracy": 0.7497503250837326, + "num_tokens": 2029994.0, + "step": 380 + }, + { + "entropy": 0.7508607231080532, + "epoch": 0.8223510806536637, + "grad_norm": 7.09375, + "learning_rate": 1.8764444444444445e-05, + "loss": 0.7575721740722656, + "mean_token_accuracy": 0.7572125047445297, + "num_tokens": 2082653.0, + "step": 390 + }, + { + "entropy": 0.7252861328423024, + "epoch": 0.8434370057986295, + "grad_norm": 6.5625, + "learning_rate": 1.867555555555556e-05, + "loss": 0.7381186485290527, + "mean_token_accuracy": 0.7612978041172027, + "num_tokens": 2135276.0, + "step": 400 + }, + { + "entropy": 0.7339485622942448, + "epoch": 0.8645229309435951, + "grad_norm": 6.625, + "learning_rate": 1.858666666666667e-05, + "loss": 0.7348906993865967, + "mean_token_accuracy": 0.7663561321794987, + "num_tokens": 2187466.0, + "step": 410 + }, + { + "entropy": 0.7272624362260103, + "epoch": 0.8856088560885609, + "grad_norm": 6.90625, + "learning_rate": 1.849777777777778e-05, + "loss": 0.749025821685791, + "mean_token_accuracy": 0.7617379955947399, + "num_tokens": 2237434.0, + "step": 420 + }, + { + "entropy": 0.7477380312979222, + "epoch": 0.9066947812335266, + "grad_norm": 6.34375, + "learning_rate": 1.840888888888889e-05, + "loss": 0.7498275756835937, + "mean_token_accuracy": 0.7604753144085408, + "num_tokens": 2288985.0, + "step": 430 + }, + { + "entropy": 0.7346535548567772, + "epoch": 0.9277807063784923, + "grad_norm": 6.9375, + "learning_rate": 1.832e-05, + "loss": 0.739972448348999, + "mean_token_accuracy": 0.7658773727715016, + "num_tokens": 2341187.0, + "step": 440 + }, + { + "entropy": 0.7167556572705507, + "epoch": 0.9488666315234581, + "grad_norm": 6.4375, + "learning_rate": 1.823111111111111e-05, + "loss": 0.7350784778594971, + "mean_token_accuracy": 0.7644179269671441, + "num_tokens": 2393385.0, + "step": 450 + }, + { + "entropy": 0.7537519961595536, + "epoch": 0.9699525566684238, + "grad_norm": 7.28125, + "learning_rate": 1.814222222222222e-05, + "loss": 0.757652759552002, + "mean_token_accuracy": 0.7587068513035774, + "num_tokens": 2442248.0, + "step": 460 + }, + { + "entropy": 0.72038439437747, + "epoch": 0.9910384818133896, + "grad_norm": 7.40625, + "learning_rate": 1.8053333333333332e-05, + "loss": 0.7361824035644531, + "mean_token_accuracy": 0.7609050884842873, + "num_tokens": 2498394.0, + "step": 470 + }, + { + "entropy": 0.735570513718837, + "epoch": 1.0105429625724829, + "grad_norm": 6.875, + "learning_rate": 1.7964444444444446e-05, + "loss": 0.730868148803711, + "mean_token_accuracy": 0.7710521068121936, + "num_tokens": 2545422.0, + "step": 480 + }, + { + "entropy": 0.6775223582983017, + "epoch": 1.0316288877174487, + "grad_norm": 7.125, + "learning_rate": 1.7875555555555556e-05, + "loss": 0.6740085124969483, + "mean_token_accuracy": 0.7795436836779117, + "num_tokens": 2597144.0, + "step": 490 + }, + { + "entropy": 0.6628748029470444, + "epoch": 1.0527148128624144, + "grad_norm": 6.84375, + "learning_rate": 1.7786666666666667e-05, + "loss": 0.6774313926696778, + "mean_token_accuracy": 0.7789938881993294, + "num_tokens": 2651938.0, + "step": 500 + }, + { + "entropy": 0.6695188779383898, + "epoch": 1.07380073800738, + "grad_norm": 7.0, + "learning_rate": 1.7697777777777777e-05, + "loss": 0.664512825012207, + "mean_token_accuracy": 0.7832319870591163, + "num_tokens": 2704948.0, + "step": 510 + }, + { + "entropy": 0.6812492057681083, + "epoch": 1.0948866631523457, + "grad_norm": 7.0, + "learning_rate": 1.760888888888889e-05, + "loss": 0.7036191940307617, + "mean_token_accuracy": 0.7707728065550328, + "num_tokens": 2757576.0, + "step": 520 + }, + { + "entropy": 0.6861097872257232, + "epoch": 1.1159725882973115, + "grad_norm": 7.3125, + "learning_rate": 1.752e-05, + "loss": 0.6651991844177246, + "mean_token_accuracy": 0.7841649115085602, + "num_tokens": 2808989.0, + "step": 530 + }, + { + "entropy": 0.6694958411157131, + "epoch": 1.1370585134422773, + "grad_norm": 6.78125, + "learning_rate": 1.7431111111111112e-05, + "loss": 0.6833772659301758, + "mean_token_accuracy": 0.7775424145162105, + "num_tokens": 2864507.0, + "step": 540 + }, + { + "entropy": 0.6568067740648985, + "epoch": 1.158144438587243, + "grad_norm": 6.21875, + "learning_rate": 1.7342222222222226e-05, + "loss": 0.6552294254302978, + "mean_token_accuracy": 0.7832994520664215, + "num_tokens": 2919201.0, + "step": 550 + }, + { + "entropy": 0.6432577680796385, + "epoch": 1.1792303637322088, + "grad_norm": 6.65625, + "learning_rate": 1.7253333333333336e-05, + "loss": 0.6560451030731201, + "mean_token_accuracy": 0.7863748580217361, + "num_tokens": 2972434.0, + "step": 560 + }, + { + "entropy": 0.6655397050082683, + "epoch": 1.2003162888771746, + "grad_norm": 6.09375, + "learning_rate": 1.7164444444444447e-05, + "loss": 0.6878513336181641, + "mean_token_accuracy": 0.7753379993140698, + "num_tokens": 3025099.0, + "step": 570 + }, + { + "entropy": 0.686093359068036, + "epoch": 1.2214022140221403, + "grad_norm": 7.09375, + "learning_rate": 1.7075555555555557e-05, + "loss": 0.6911579132080078, + "mean_token_accuracy": 0.7787310972809791, + "num_tokens": 3076307.0, + "step": 580 + }, + { + "entropy": 0.6812980778515338, + "epoch": 1.2424881391671059, + "grad_norm": 7.125, + "learning_rate": 1.6986666666666668e-05, + "loss": 0.6704156398773193, + "mean_token_accuracy": 0.7793935135006904, + "num_tokens": 3130053.0, + "step": 590 + }, + { + "entropy": 0.6409769468009472, + "epoch": 1.2635740643120716, + "grad_norm": 7.4375, + "learning_rate": 1.6897777777777778e-05, + "loss": 0.661864185333252, + "mean_token_accuracy": 0.7865670621395111, + "num_tokens": 3183331.0, + "step": 600 + }, + { + "entropy": 0.6799454756081105, + "epoch": 1.2846599894570374, + "grad_norm": 6.65625, + "learning_rate": 1.680888888888889e-05, + "loss": 0.6818383693695068, + "mean_token_accuracy": 0.778287123888731, + "num_tokens": 3239561.0, + "step": 610 + }, + { + "entropy": 0.6814555354416371, + "epoch": 1.3057459146020032, + "grad_norm": 6.3125, + "learning_rate": 1.672e-05, + "loss": 0.679498815536499, + "mean_token_accuracy": 0.7788635492324829, + "num_tokens": 3292996.0, + "step": 620 + }, + { + "entropy": 0.667108029127121, + "epoch": 1.326831839746969, + "grad_norm": 6.90625, + "learning_rate": 1.6631111111111113e-05, + "loss": 0.6758651733398438, + "mean_token_accuracy": 0.7792007848620415, + "num_tokens": 3345736.0, + "step": 630 + }, + { + "entropy": 0.6692709110677242, + "epoch": 1.3479177648919347, + "grad_norm": 6.96875, + "learning_rate": 1.6542222222222223e-05, + "loss": 0.6796128273010253, + "mean_token_accuracy": 0.7766051791608334, + "num_tokens": 3398076.0, + "step": 640 + }, + { + "entropy": 0.6564048878848553, + "epoch": 1.3690036900369003, + "grad_norm": 6.9375, + "learning_rate": 1.6453333333333334e-05, + "loss": 0.6633264064788819, + "mean_token_accuracy": 0.7854946702718735, + "num_tokens": 3450796.0, + "step": 650 + }, + { + "entropy": 0.6611979272216558, + "epoch": 1.390089615181866, + "grad_norm": 6.65625, + "learning_rate": 1.6364444444444444e-05, + "loss": 0.6661429405212402, + "mean_token_accuracy": 0.7835663430392742, + "num_tokens": 3503863.0, + "step": 660 + }, + { + "entropy": 0.6534961331635714, + "epoch": 1.4111755403268318, + "grad_norm": 6.9375, + "learning_rate": 1.6275555555555558e-05, + "loss": 0.6586530208587646, + "mean_token_accuracy": 0.7844067431986332, + "num_tokens": 3556265.0, + "step": 670 + }, + { + "entropy": 0.6516873501241207, + "epoch": 1.4322614654717976, + "grad_norm": 6.40625, + "learning_rate": 1.618666666666667e-05, + "loss": 0.6522529125213623, + "mean_token_accuracy": 0.7855845786631107, + "num_tokens": 3611423.0, + "step": 680 + }, + { + "entropy": 0.6764013767242432, + "epoch": 1.4533473906167633, + "grad_norm": 6.71875, + "learning_rate": 1.609777777777778e-05, + "loss": 0.6884442806243897, + "mean_token_accuracy": 0.7756352722644806, + "num_tokens": 3666630.0, + "step": 690 + }, + { + "entropy": 0.6554486058652401, + "epoch": 1.474433315761729, + "grad_norm": 7.40625, + "learning_rate": 1.600888888888889e-05, + "loss": 0.6563316822052002, + "mean_token_accuracy": 0.7854672558605671, + "num_tokens": 3718719.0, + "step": 700 + }, + { + "entropy": 0.6385185327380896, + "epoch": 1.4955192409066949, + "grad_norm": 6.59375, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.6513954639434815, + "mean_token_accuracy": 0.7851097173988819, + "num_tokens": 3771822.0, + "step": 710 + }, + { + "entropy": 0.654766371101141, + "epoch": 1.5166051660516606, + "grad_norm": 6.5, + "learning_rate": 1.5831111111111114e-05, + "loss": 0.6622748851776123, + "mean_token_accuracy": 0.7847663886845112, + "num_tokens": 3823795.0, + "step": 720 + }, + { + "entropy": 0.6597310703247785, + "epoch": 1.5376910911966264, + "grad_norm": 6.375, + "learning_rate": 1.5742222222222224e-05, + "loss": 0.6634318351745605, + "mean_token_accuracy": 0.7861979939043522, + "num_tokens": 3875784.0, + "step": 730 + }, + { + "entropy": 0.6637851595878601, + "epoch": 1.558777016341592, + "grad_norm": 6.9375, + "learning_rate": 1.5653333333333335e-05, + "loss": 0.6621220111846924, + "mean_token_accuracy": 0.7886320613324642, + "num_tokens": 3929412.0, + "step": 740 + }, + { + "entropy": 0.637692479044199, + "epoch": 1.5798629414865577, + "grad_norm": 6.59375, + "learning_rate": 1.5564444444444445e-05, + "loss": 0.6402386665344239, + "mean_token_accuracy": 0.7895873270928859, + "num_tokens": 3982472.0, + "step": 750 + }, + { + "entropy": 0.6578179821372032, + "epoch": 1.6009488666315235, + "grad_norm": 7.25, + "learning_rate": 1.5475555555555556e-05, + "loss": 0.6751936435699463, + "mean_token_accuracy": 0.7776105716824532, + "num_tokens": 4037829.0, + "step": 760 + }, + { + "entropy": 0.6653612300753593, + "epoch": 1.6220347917764892, + "grad_norm": 6.9375, + "learning_rate": 1.5386666666666666e-05, + "loss": 0.6717413425445556, + "mean_token_accuracy": 0.777862798422575, + "num_tokens": 4093735.0, + "step": 770 + }, + { + "entropy": 0.6561707239598036, + "epoch": 1.6431207169214548, + "grad_norm": 7.03125, + "learning_rate": 1.5297777777777777e-05, + "loss": 0.6501868724822998, + "mean_token_accuracy": 0.7863908343017101, + "num_tokens": 4148410.0, + "step": 780 + }, + { + "entropy": 0.6655665650963783, + "epoch": 1.6642066420664205, + "grad_norm": 6.59375, + "learning_rate": 1.520888888888889e-05, + "loss": 0.679684829711914, + "mean_token_accuracy": 0.778567586839199, + "num_tokens": 4202907.0, + "step": 790 + }, + { + "entropy": 0.6781473312526941, + "epoch": 1.6852925672113863, + "grad_norm": 6.875, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.6722404479980468, + "mean_token_accuracy": 0.7763243250548839, + "num_tokens": 4256213.0, + "step": 800 + }, + { + "entropy": 0.6321199681609869, + "epoch": 1.706378492356352, + "grad_norm": 6.78125, + "learning_rate": 1.5031111111111111e-05, + "loss": 0.6404708385467529, + "mean_token_accuracy": 0.7903632469475269, + "num_tokens": 4309268.0, + "step": 810 + }, + { + "entropy": 0.6424700867384672, + "epoch": 1.7274644175013179, + "grad_norm": 7.15625, + "learning_rate": 1.4942222222222222e-05, + "loss": 0.6444578170776367, + "mean_token_accuracy": 0.7881530813872815, + "num_tokens": 4360761.0, + "step": 820 + }, + { + "entropy": 0.6615507304668427, + "epoch": 1.7485503426462836, + "grad_norm": 6.0625, + "learning_rate": 1.4853333333333336e-05, + "loss": 0.6604718208312989, + "mean_token_accuracy": 0.782805722206831, + "num_tokens": 4411678.0, + "step": 830 + }, + { + "entropy": 0.6196760050952435, + "epoch": 1.7696362677912494, + "grad_norm": 6.6875, + "learning_rate": 1.4764444444444446e-05, + "loss": 0.6261836051940918, + "mean_token_accuracy": 0.7926589228212834, + "num_tokens": 4467254.0, + "step": 840 + }, + { + "entropy": 0.6139154363423586, + "epoch": 1.7907221929362152, + "grad_norm": 7.21875, + "learning_rate": 1.4675555555555557e-05, + "loss": 0.6301932811737061, + "mean_token_accuracy": 0.7918383464217186, + "num_tokens": 4517383.0, + "step": 850 + }, + { + "entropy": 0.6410171519964933, + "epoch": 1.811808118081181, + "grad_norm": 7.1875, + "learning_rate": 1.4586666666666667e-05, + "loss": 0.655810022354126, + "mean_token_accuracy": 0.7856725618243218, + "num_tokens": 4569980.0, + "step": 860 + }, + { + "entropy": 0.6666022066026926, + "epoch": 1.8328940432261467, + "grad_norm": 6.5625, + "learning_rate": 1.449777777777778e-05, + "loss": 0.6677482604980469, + "mean_token_accuracy": 0.7798060193657875, + "num_tokens": 4622612.0, + "step": 870 + }, + { + "entropy": 0.6398684818297624, + "epoch": 1.8539799683711122, + "grad_norm": 7.84375, + "learning_rate": 1.440888888888889e-05, + "loss": 0.6501501083374024, + "mean_token_accuracy": 0.7857734531164169, + "num_tokens": 4676525.0, + "step": 880 + }, + { + "entropy": 0.6285852700471878, + "epoch": 1.875065893516078, + "grad_norm": 7.0625, + "learning_rate": 1.432e-05, + "loss": 0.6422877311706543, + "mean_token_accuracy": 0.7884316690266132, + "num_tokens": 4734090.0, + "step": 890 + }, + { + "entropy": 0.6409385818988085, + "epoch": 1.8961518186610438, + "grad_norm": 7.9375, + "learning_rate": 1.4231111111111114e-05, + "loss": 0.6352181434631348, + "mean_token_accuracy": 0.7952412232756615, + "num_tokens": 4788537.0, + "step": 900 + }, + { + "entropy": 0.6456746399402619, + "epoch": 1.9172377438060095, + "grad_norm": 6.90625, + "learning_rate": 1.4142222222222224e-05, + "loss": 0.6564863204956055, + "mean_token_accuracy": 0.789187741279602, + "num_tokens": 4840342.0, + "step": 910 + }, + { + "entropy": 0.6360713161528111, + "epoch": 1.938323668950975, + "grad_norm": 6.3125, + "learning_rate": 1.4053333333333335e-05, + "loss": 0.6300373554229737, + "mean_token_accuracy": 0.7876299776136875, + "num_tokens": 4891524.0, + "step": 920 + }, + { + "entropy": 0.6285211775451899, + "epoch": 1.9594095940959408, + "grad_norm": 7.78125, + "learning_rate": 1.3964444444444445e-05, + "loss": 0.6337003231048584, + "mean_token_accuracy": 0.7889726974070073, + "num_tokens": 4939573.0, + "step": 930 + }, + { + "entropy": 0.6301535632461309, + "epoch": 1.9804955192409066, + "grad_norm": 6.84375, + "learning_rate": 1.3875555555555557e-05, + "loss": 0.6374162673950196, + "mean_token_accuracy": 0.7898242607712745, + "num_tokens": 4991267.0, + "step": 940 + }, + { + "entropy": 0.6537058566873138, + "epoch": 2.0, + "grad_norm": 13.3125, + "learning_rate": 1.3786666666666668e-05, + "loss": 0.641156530380249, + "mean_token_accuracy": 0.7822844466647586, + "num_tokens": 5036304.0, + "step": 950 + }, + { + "entropy": 0.6411334682255984, + "epoch": 2.0210859251449658, + "grad_norm": 7.1875, + "learning_rate": 1.3697777777777778e-05, + "loss": 0.6083747386932373, + "mean_token_accuracy": 0.8002627611160278, + "num_tokens": 5089737.0, + "step": 960 + }, + { + "entropy": 0.5841103594750165, + "epoch": 2.0421718502899315, + "grad_norm": 6.53125, + "learning_rate": 1.3608888888888889e-05, + "loss": 0.6012133121490478, + "mean_token_accuracy": 0.8027123242616654, + "num_tokens": 5143330.0, + "step": 970 + }, + { + "entropy": 0.590392829477787, + "epoch": 2.0632577754348973, + "grad_norm": 6.46875, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.5829256534576416, + "mean_token_accuracy": 0.8068650290369987, + "num_tokens": 5198614.0, + "step": 980 + }, + { + "entropy": 0.6148912344127894, + "epoch": 2.084343700579863, + "grad_norm": 6.59375, + "learning_rate": 1.3431111111111113e-05, + "loss": 0.6117685317993165, + "mean_token_accuracy": 0.7951427385210991, + "num_tokens": 5252755.0, + "step": 990 + }, + { + "entropy": 0.5893940377980471, + "epoch": 2.105429625724829, + "grad_norm": 6.625, + "learning_rate": 1.3342222222222224e-05, + "loss": 0.5790496349334717, + "mean_token_accuracy": 0.809234394133091, + "num_tokens": 5304859.0, + "step": 1000 + }, + { + "entropy": 0.587013079598546, + "epoch": 2.1265155508697946, + "grad_norm": 6.9375, + "learning_rate": 1.3253333333333334e-05, + "loss": 0.5892822265625, + "mean_token_accuracy": 0.8040889374911785, + "num_tokens": 5357877.0, + "step": 1010 + }, + { + "entropy": 0.5877181258052587, + "epoch": 2.14760147601476, + "grad_norm": 6.09375, + "learning_rate": 1.3164444444444446e-05, + "loss": 0.595665693283081, + "mean_token_accuracy": 0.8036225430667401, + "num_tokens": 5413504.0, + "step": 1020 + }, + { + "entropy": 0.5848948258906603, + "epoch": 2.1686874011597257, + "grad_norm": 6.4375, + "learning_rate": 1.3075555555555557e-05, + "loss": 0.5771171092987061, + "mean_token_accuracy": 0.8047491163015366, + "num_tokens": 5466549.0, + "step": 1030 + }, + { + "entropy": 0.5815035384148359, + "epoch": 2.1897733263046915, + "grad_norm": 6.96875, + "learning_rate": 1.2986666666666667e-05, + "loss": 0.5768617153167724, + "mean_token_accuracy": 0.8057669088244438, + "num_tokens": 5519515.0, + "step": 1040 + }, + { + "entropy": 0.58335498906672, + "epoch": 2.2108592514496572, + "grad_norm": 7.6875, + "learning_rate": 1.2897777777777778e-05, + "loss": 0.6007503509521485, + "mean_token_accuracy": 0.8005498602986336, + "num_tokens": 5570188.0, + "step": 1050 + }, + { + "entropy": 0.6068603809922933, + "epoch": 2.231945176594623, + "grad_norm": 6.5625, + "learning_rate": 1.2808888888888891e-05, + "loss": 0.6024115562438965, + "mean_token_accuracy": 0.7973541341722011, + "num_tokens": 5624190.0, + "step": 1060 + }, + { + "entropy": 0.5774707745760679, + "epoch": 2.2530311017395888, + "grad_norm": 6.53125, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.5705479145050049, + "mean_token_accuracy": 0.8100002221763134, + "num_tokens": 5675656.0, + "step": 1070 + }, + { + "entropy": 0.5851722996681928, + "epoch": 2.2741170268845545, + "grad_norm": 7.3125, + "learning_rate": 1.2631111111111112e-05, + "loss": 0.5905722141265869, + "mean_token_accuracy": 0.8069308035075664, + "num_tokens": 5728878.0, + "step": 1080 + }, + { + "entropy": 0.594813584536314, + "epoch": 2.2952029520295203, + "grad_norm": 6.875, + "learning_rate": 1.2542222222222223e-05, + "loss": 0.5923901557922363, + "mean_token_accuracy": 0.8034852437675, + "num_tokens": 5783257.0, + "step": 1090 + }, + { + "entropy": 0.5841789487749338, + "epoch": 2.316288877174486, + "grad_norm": 7.0, + "learning_rate": 1.2453333333333335e-05, + "loss": 0.5864346504211426, + "mean_token_accuracy": 0.8033530995249748, + "num_tokens": 5834581.0, + "step": 1100 + }, + { + "entropy": 0.6010521166026592, + "epoch": 2.337374802319452, + "grad_norm": 6.9375, + "learning_rate": 1.2364444444444445e-05, + "loss": 0.5941137313842774, + "mean_token_accuracy": 0.8033333070576191, + "num_tokens": 5887458.0, + "step": 1110 + }, + { + "entropy": 0.5919341973960399, + "epoch": 2.3584607274644176, + "grad_norm": 8.75, + "learning_rate": 1.2275555555555556e-05, + "loss": 0.6103182315826416, + "mean_token_accuracy": 0.7999237082898617, + "num_tokens": 5941284.0, + "step": 1120 + }, + { + "entropy": 0.5942621320486069, + "epoch": 2.3795466526093834, + "grad_norm": 7.375, + "learning_rate": 1.2186666666666666e-05, + "loss": 0.5983363151550293, + "mean_token_accuracy": 0.8016222022473812, + "num_tokens": 5995817.0, + "step": 1130 + }, + { + "entropy": 0.6046227600425482, + "epoch": 2.400632577754349, + "grad_norm": 6.40625, + "learning_rate": 1.209777777777778e-05, + "loss": 0.6026669025421143, + "mean_token_accuracy": 0.8024684950709343, + "num_tokens": 6050783.0, + "step": 1140 + }, + { + "entropy": 0.5998306684195995, + "epoch": 2.421718502899315, + "grad_norm": 6.59375, + "learning_rate": 1.200888888888889e-05, + "loss": 0.5893473625183105, + "mean_token_accuracy": 0.803314708918333, + "num_tokens": 6103371.0, + "step": 1150 + }, + { + "entropy": 0.593693134188652, + "epoch": 2.4428044280442807, + "grad_norm": 6.5, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.6076583862304688, + "mean_token_accuracy": 0.7995239071547985, + "num_tokens": 6155502.0, + "step": 1160 + }, + { + "entropy": 0.5827030725777149, + "epoch": 2.463890353189246, + "grad_norm": 6.84375, + "learning_rate": 1.1831111111111112e-05, + "loss": 0.5882821559906006, + "mean_token_accuracy": 0.7929977595806121, + "num_tokens": 6208060.0, + "step": 1170 + }, + { + "entropy": 0.5910260166972876, + "epoch": 2.4849762783342118, + "grad_norm": 7.53125, + "learning_rate": 1.1742222222222224e-05, + "loss": 0.5860947132110595, + "mean_token_accuracy": 0.8049415498971939, + "num_tokens": 6261588.0, + "step": 1180 + }, + { + "entropy": 0.5704694345593453, + "epoch": 2.5060622034791775, + "grad_norm": 6.75, + "learning_rate": 1.1653333333333334e-05, + "loss": 0.5749064445495605, + "mean_token_accuracy": 0.8084491945803165, + "num_tokens": 6314963.0, + "step": 1190 + }, + { + "entropy": 0.597440430521965, + "epoch": 2.5271481286241433, + "grad_norm": 6.5, + "learning_rate": 1.1564444444444445e-05, + "loss": 0.5958696842193604, + "mean_token_accuracy": 0.8057812102138996, + "num_tokens": 6368741.0, + "step": 1200 + }, + { + "entropy": 0.6070981964468956, + "epoch": 2.548234053769109, + "grad_norm": 7.96875, + "learning_rate": 1.1475555555555557e-05, + "loss": 0.6143896102905273, + "mean_token_accuracy": 0.7980599500238895, + "num_tokens": 6425424.0, + "step": 1210 + }, + { + "entropy": 0.5948727998882533, + "epoch": 2.569319978914075, + "grad_norm": 6.5, + "learning_rate": 1.1386666666666669e-05, + "loss": 0.5917119026184082, + "mean_token_accuracy": 0.8025051333010197, + "num_tokens": 6476263.0, + "step": 1220 + }, + { + "entropy": 0.5811738461256027, + "epoch": 2.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.129777777777778e-05, + "loss": 0.5865864753723145, + "mean_token_accuracy": 0.8044181451201439, + "num_tokens": 6528012.0, + "step": 1230 + }, + { + "entropy": 0.5937653455883265, + "epoch": 2.6114918292040064, + "grad_norm": 6.59375, + "learning_rate": 1.120888888888889e-05, + "loss": 0.5843213558197021, + "mean_token_accuracy": 0.8025668740272522, + "num_tokens": 6583421.0, + "step": 1240 + }, + { + "entropy": 0.573565949127078, + "epoch": 2.632577754348972, + "grad_norm": 6.5625, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.5826557159423829, + "mean_token_accuracy": 0.8070438347756863, + "num_tokens": 6636858.0, + "step": 1250 + }, + { + "entropy": 0.6216809719800949, + "epoch": 2.653663679493938, + "grad_norm": 7.09375, + "learning_rate": 1.1031111111111112e-05, + "loss": 0.6125890254974365, + "mean_token_accuracy": 0.7970895677804947, + "num_tokens": 6692168.0, + "step": 1260 + }, + { + "entropy": 0.5998566258698702, + "epoch": 2.6747496046389037, + "grad_norm": 7.0625, + "learning_rate": 1.0942222222222223e-05, + "loss": 0.5921674728393554, + "mean_token_accuracy": 0.8061520479619503, + "num_tokens": 6745834.0, + "step": 1270 + }, + { + "entropy": 0.5880189586430788, + "epoch": 2.6958355297838694, + "grad_norm": 7.59375, + "learning_rate": 1.0853333333333333e-05, + "loss": 0.5986497402191162, + "mean_token_accuracy": 0.8006219923496246, + "num_tokens": 6800244.0, + "step": 1280 + }, + { + "entropy": 0.5981763809919357, + "epoch": 2.716921454928835, + "grad_norm": 6.6875, + "learning_rate": 1.0764444444444445e-05, + "loss": 0.6020579814910889, + "mean_token_accuracy": 0.8032843962311744, + "num_tokens": 6853454.0, + "step": 1290 + }, + { + "entropy": 0.5963539965450764, + "epoch": 2.7380073800738005, + "grad_norm": 6.59375, + "learning_rate": 1.0675555555555558e-05, + "loss": 0.5908561706542969, + "mean_token_accuracy": 0.8011696465313435, + "num_tokens": 6904473.0, + "step": 1300 + }, + { + "entropy": 0.5899917531758547, + "epoch": 2.7590933052187667, + "grad_norm": 6.6875, + "learning_rate": 1.0586666666666668e-05, + "loss": 0.5744058132171631, + "mean_token_accuracy": 0.8053683824837208, + "num_tokens": 6956756.0, + "step": 1310 + }, + { + "entropy": 0.582448274642229, + "epoch": 2.780179230363732, + "grad_norm": 6.78125, + "learning_rate": 1.0497777777777779e-05, + "loss": 0.5848373889923095, + "mean_token_accuracy": 0.8054254874587059, + "num_tokens": 7008611.0, + "step": 1320 + }, + { + "entropy": 0.5906594522297383, + "epoch": 2.801265155508698, + "grad_norm": 7.75, + "learning_rate": 1.040888888888889e-05, + "loss": 0.6006948471069335, + "mean_token_accuracy": 0.7971956506371498, + "num_tokens": 7062235.0, + "step": 1330 + }, + { + "entropy": 0.5976275954395532, + "epoch": 2.8223510806536636, + "grad_norm": 6.34375, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.5997600078582763, + "mean_token_accuracy": 0.7998531877994537, + "num_tokens": 7115737.0, + "step": 1340 + }, + { + "entropy": 0.6079881630837918, + "epoch": 2.8434370057986293, + "grad_norm": 7.0, + "learning_rate": 1.0231111111111112e-05, + "loss": 0.6024625778198243, + "mean_token_accuracy": 0.7988371469080449, + "num_tokens": 7166870.0, + "step": 1350 + }, + { + "entropy": 0.6036868289113044, + "epoch": 2.864522930943595, + "grad_norm": 6.6875, + "learning_rate": 1.0142222222222222e-05, + "loss": 0.5962271213531494, + "mean_token_accuracy": 0.8011133573949337, + "num_tokens": 7219785.0, + "step": 1360 + }, + { + "entropy": 0.5746289525181055, + "epoch": 2.885608856088561, + "grad_norm": 7.5625, + "learning_rate": 1.0053333333333334e-05, + "loss": 0.5769401073455811, + "mean_token_accuracy": 0.809252617508173, + "num_tokens": 7275411.0, + "step": 1370 + }, + { + "entropy": 0.5846736520528794, + "epoch": 2.9066947812335266, + "grad_norm": 7.125, + "learning_rate": 9.964444444444445e-06, + "loss": 0.5899319171905517, + "mean_token_accuracy": 0.8028473742306232, + "num_tokens": 7326752.0, + "step": 1380 + }, + { + "entropy": 0.5813163112848997, + "epoch": 2.9277807063784924, + "grad_norm": 7.46875, + "learning_rate": 9.875555555555557e-06, + "loss": 0.5799898147583008, + "mean_token_accuracy": 0.8026903085410595, + "num_tokens": 7375608.0, + "step": 1390 + }, + { + "entropy": 0.5724870227277279, + "epoch": 2.948866631523458, + "grad_norm": 6.90625, + "learning_rate": 9.786666666666667e-06, + "loss": 0.5783823490142822, + "mean_token_accuracy": 0.8059644535183906, + "num_tokens": 7430145.0, + "step": 1400 + }, + { + "entropy": 0.5778653729707003, + "epoch": 2.969952556668424, + "grad_norm": 6.84375, + "learning_rate": 9.697777777777778e-06, + "loss": 0.5640594005584717, + "mean_token_accuracy": 0.8100298568606377, + "num_tokens": 7482976.0, + "step": 1410 + }, + { + "entropy": 0.5823766525834799, + "epoch": 2.9910384818133897, + "grad_norm": 6.5625, + "learning_rate": 9.60888888888889e-06, + "loss": 0.5931865692138671, + "mean_token_accuracy": 0.8023995622992516, + "num_tokens": 7533043.0, + "step": 1420 + } + ], + "logging_steps": 10, + "max_steps": 2500, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.73147357152727e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1425/training_args.bin b/checkpoint-1425/training_args.bin new file mode 100644 index 0000000..f138a50 --- /dev/null +++ b/checkpoint-1425/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7103b0ce3340ea41aca3b7441d01d7c38791e3d3ce8c9b28c906c9b1d49bbee9 +size 5713 diff --git a/checkpoint-189/chat_template.jinja b/checkpoint-189/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-189/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-189/config.json b/checkpoint-189/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-189/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-189/generation_config.json b/checkpoint-189/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-189/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-189/model.safetensors b/checkpoint-189/model.safetensors new file mode 100644 index 0000000..3f72dcb --- /dev/null +++ b/checkpoint-189/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:901f734b20929e6d89aa77c20ce5eba58304d4594b4a3cdbb5d0b63462300ce2 +size 3441185608 diff --git a/checkpoint-189/optimizer.pt b/checkpoint-189/optimizer.pt new file mode 100644 index 0000000..db3d0dd --- /dev/null +++ b/checkpoint-189/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7da65283d99ce5cf7edfabd85835029bfd04f12378dba203bd16b429133eeb +size 6882572207 diff --git a/checkpoint-189/rng_state.pth b/checkpoint-189/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-189/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-189/scheduler.pt b/checkpoint-189/scheduler.pt new file mode 100644 index 0000000..59e7cbe --- /dev/null +++ b/checkpoint-189/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aead4b343e2bf68c616057b87b8e51106ac2226e0139a6361df37f62d53abeb3 +size 1465 diff --git a/checkpoint-189/tokenizer.json b/checkpoint-189/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-189/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-189/tokenizer_config.json b/checkpoint-189/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-189/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-189/trainer_state.json b/checkpoint-189/trainer_state.json new file mode 100644 index 0000000..9ea4b0b --- /dev/null +++ b/checkpoint-189/trainer_state.json @@ -0,0 +1,214 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 189, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.9081946484744549, + "epoch": 0.053015241882041084, + "grad_norm": 51.5, + "learning_rate": 3e-06, + "loss": 2.320839500427246, + "mean_token_accuracy": 0.5708254482597113, + "num_tokens": 30761.0, + "step": 10 + }, + { + "entropy": 1.032745758444071, + "epoch": 0.10603048376408217, + "grad_norm": 21.125, + "learning_rate": 6.333333333333333e-06, + "loss": 1.9631355285644532, + "mean_token_accuracy": 0.5809337079524994, + "num_tokens": 61277.0, + "step": 20 + }, + { + "entropy": 1.2370933540165425, + "epoch": 0.15904572564612326, + "grad_norm": 11.0625, + "learning_rate": 9.666666666666667e-06, + "loss": 1.4653306961059571, + "mean_token_accuracy": 0.6311864696443081, + "num_tokens": 91730.0, + "step": 30 + }, + { + "entropy": 1.1646109029650689, + "epoch": 0.21206096752816433, + "grad_norm": 8.5625, + "learning_rate": 1.3000000000000001e-05, + "loss": 1.200753402709961, + "mean_token_accuracy": 0.6730783767998219, + "num_tokens": 122896.0, + "step": 40 + }, + { + "entropy": 1.0138436160981654, + "epoch": 0.26507620941020543, + "grad_norm": 7.4375, + "learning_rate": 1.6333333333333335e-05, + "loss": 1.0109354972839355, + "mean_token_accuracy": 0.7080431424081326, + "num_tokens": 153323.0, + "step": 50 + }, + { + "entropy": 0.8669569574296474, + "epoch": 0.31809145129224653, + "grad_norm": 6.75, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8859419822692871, + "mean_token_accuracy": 0.7356305047869682, + "num_tokens": 183928.0, + "step": 60 + }, + { + "entropy": 0.856675973534584, + "epoch": 0.3711066931742876, + "grad_norm": 7.6875, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8721119880676269, + "mean_token_accuracy": 0.7318881675601006, + "num_tokens": 215054.0, + "step": 70 + }, + { + "entropy": 0.7872001446783543, + "epoch": 0.42412193505632867, + "grad_norm": 7.46875, + "learning_rate": 1.92962962962963e-05, + "loss": 0.7920477390289307, + "mean_token_accuracy": 0.7553018122911453, + "num_tokens": 245614.0, + "step": 80 + }, + { + "entropy": 0.7739131577312947, + "epoch": 0.47713717693836977, + "grad_norm": 7.09375, + "learning_rate": 1.8925925925925928e-05, + "loss": 0.7949441432952881, + "mean_token_accuracy": 0.7514781631529331, + "num_tokens": 276651.0, + "step": 90 + }, + { + "entropy": 0.7763891488313674, + "epoch": 0.5301524188204109, + "grad_norm": 7.8125, + "learning_rate": 1.8555555555555557e-05, + "loss": 0.7867375373840332, + "mean_token_accuracy": 0.7523738898336887, + "num_tokens": 308561.0, + "step": 100 + }, + { + "entropy": 0.7757978036999702, + "epoch": 0.583167660702452, + "grad_norm": 7.21875, + "learning_rate": 1.8185185185185186e-05, + "loss": 0.7776416301727295, + "mean_token_accuracy": 0.7548527352511882, + "num_tokens": 339537.0, + "step": 110 + }, + { + "entropy": 0.7528716944158077, + "epoch": 0.6361829025844931, + "grad_norm": 6.28125, + "learning_rate": 1.7814814814814815e-05, + "loss": 0.7566296100616455, + "mean_token_accuracy": 0.7637521453201771, + "num_tokens": 370938.0, + "step": 120 + }, + { + "entropy": 0.7483146995306015, + "epoch": 0.6891981444665342, + "grad_norm": 7.0, + "learning_rate": 1.7444444444444448e-05, + "loss": 0.7508650302886963, + "mean_token_accuracy": 0.758705198019743, + "num_tokens": 402708.0, + "step": 130 + }, + { + "entropy": 0.7206243231892586, + "epoch": 0.7422133863485753, + "grad_norm": 6.65625, + "learning_rate": 1.7074074074074077e-05, + "loss": 0.7531259536743165, + "mean_token_accuracy": 0.7641572073101998, + "num_tokens": 433645.0, + "step": 140 + }, + { + "entropy": 0.7596748031675815, + "epoch": 0.7952286282306164, + "grad_norm": 6.03125, + "learning_rate": 1.6703703703703703e-05, + "loss": 0.7591583728790283, + "mean_token_accuracy": 0.7570627965033054, + "num_tokens": 465791.0, + "step": 150 + }, + { + "entropy": 0.7133973561227321, + "epoch": 0.8482438701126573, + "grad_norm": 7.125, + "learning_rate": 1.6333333333333335e-05, + "loss": 0.7458220005035401, + "mean_token_accuracy": 0.7626615799963474, + "num_tokens": 497296.0, + "step": 160 + }, + { + "entropy": 0.7134327009320259, + "epoch": 0.9012591119946984, + "grad_norm": 7.0, + "learning_rate": 1.5962962962962964e-05, + "loss": 0.7290266990661621, + "mean_token_accuracy": 0.7681502625346184, + "num_tokens": 528181.0, + "step": 170 + }, + { + "entropy": 0.7324420973658562, + "epoch": 0.9542743538767395, + "grad_norm": 7.5, + "learning_rate": 1.5592592592592593e-05, + "loss": 0.731742525100708, + "mean_token_accuracy": 0.7631207950413227, + "num_tokens": 558915.0, + "step": 180 + } + ], + "logging_steps": 10, + "max_steps": 600, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5671984696049664.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-189/training_args.bin b/checkpoint-189/training_args.bin new file mode 100644 index 0000000..6618e60 --- /dev/null +++ b/checkpoint-189/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c0557c0612cd65ef8abf2a6c3d767d5aa1d732f9c315d62b560fc3d0f5beafe +size 5713 diff --git a/checkpoint-1900/chat_template.jinja b/checkpoint-1900/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-1900/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-1900/config.json b/checkpoint-1900/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-1900/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-1900/generation_config.json b/checkpoint-1900/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-1900/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-1900/model.safetensors b/checkpoint-1900/model.safetensors new file mode 100644 index 0000000..ddb84bd --- /dev/null +++ b/checkpoint-1900/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b37c56e5baaf6199dbde95f51d8e51bec316b3fb2cd43e0660e25a9243f9554 +size 3441185608 diff --git a/checkpoint-1900/optimizer.pt b/checkpoint-1900/optimizer.pt new file mode 100644 index 0000000..5e81759 --- /dev/null +++ b/checkpoint-1900/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a3a9b26672193949cf5aa6e433f776eb74651b4844db4532a091f7977e16b95 +size 6882572207 diff --git a/checkpoint-1900/rng_state.pth b/checkpoint-1900/rng_state.pth new file mode 100644 index 0000000..435e005 --- /dev/null +++ b/checkpoint-1900/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e2011629d8bed3ef560fa11175cac55684c4e12a72634bb24abf767b6c7399 +size 14645 diff --git a/checkpoint-1900/scheduler.pt b/checkpoint-1900/scheduler.pt new file mode 100644 index 0000000..00c7def --- /dev/null +++ b/checkpoint-1900/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2e459170007d9de820073662df01980a67fc2b28884fc51f4cf1c72df9aa955 +size 1465 diff --git a/checkpoint-1900/tokenizer.json b/checkpoint-1900/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-1900/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-1900/tokenizer_config.json b/checkpoint-1900/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-1900/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-1900/trainer_state.json b/checkpoint-1900/trainer_state.json new file mode 100644 index 0000000..4089367 --- /dev/null +++ b/checkpoint-1900/trainer_state.json @@ -0,0 +1,1934 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.8691443920135498, + "epoch": 0.021085925144965736, + "grad_norm": 47.5, + "learning_rate": 7.2e-07, + "loss": 2.1268569946289064, + "mean_token_accuracy": 0.5856592752039432, + "num_tokens": 53385.0, + "step": 10 + }, + { + "entropy": 0.9018128983676433, + "epoch": 0.04217185028993147, + "grad_norm": 49.25, + "learning_rate": 1.52e-06, + "loss": 2.1235822677612304, + "mean_token_accuracy": 0.5826416034251451, + "num_tokens": 108949.0, + "step": 20 + }, + { + "entropy": 0.9313402399420738, + "epoch": 0.0632577754348972, + "grad_norm": 38.25, + "learning_rate": 2.3200000000000002e-06, + "loss": 1.9997358322143555, + "mean_token_accuracy": 0.5817345686256885, + "num_tokens": 162034.0, + "step": 30 + }, + { + "entropy": 0.9819381944835186, + "epoch": 0.08434370057986294, + "grad_norm": 28.25, + "learning_rate": 3.12e-06, + "loss": 1.8760208129882812, + "mean_token_accuracy": 0.5874959267675877, + "num_tokens": 214165.0, + "step": 40 + }, + { + "entropy": 1.0624087303876877, + "epoch": 0.10542962572482868, + "grad_norm": 14.1875, + "learning_rate": 3.920000000000001e-06, + "loss": 1.6169570922851562, + "mean_token_accuracy": 0.601640059798956, + "num_tokens": 265527.0, + "step": 50 + }, + { + "entropy": 1.1721510097384453, + "epoch": 0.1265155508697944, + "grad_norm": 11.3125, + "learning_rate": 4.7200000000000005e-06, + "loss": 1.4622878074645995, + "mean_token_accuracy": 0.6209174185991287, + "num_tokens": 319341.0, + "step": 60 + }, + { + "entropy": 1.2009219758212566, + "epoch": 0.14760147601476015, + "grad_norm": 9.0625, + "learning_rate": 5.5200000000000005e-06, + "loss": 1.2933337211608886, + "mean_token_accuracy": 0.6467074818909169, + "num_tokens": 373709.0, + "step": 70 + }, + { + "entropy": 1.1439887225627898, + "epoch": 0.16868740115972589, + "grad_norm": 9.3125, + "learning_rate": 6.3200000000000005e-06, + "loss": 1.2084498405456543, + "mean_token_accuracy": 0.6660686552524566, + "num_tokens": 425923.0, + "step": 80 + }, + { + "entropy": 1.0713621236383915, + "epoch": 0.18977332630469163, + "grad_norm": 7.84375, + "learning_rate": 7.1200000000000004e-06, + "loss": 1.1175435066223145, + "mean_token_accuracy": 0.6812180563807487, + "num_tokens": 480012.0, + "step": 90 + }, + { + "entropy": 1.0369222231209279, + "epoch": 0.21085925144965736, + "grad_norm": 7.09375, + "learning_rate": 7.92e-06, + "loss": 1.062850570678711, + "mean_token_accuracy": 0.6899570092558861, + "num_tokens": 533890.0, + "step": 100 + }, + { + "entropy": 0.9894454136490822, + "epoch": 0.23194517659462308, + "grad_norm": 7.75, + "learning_rate": 8.720000000000001e-06, + "loss": 1.0150799751281738, + "mean_token_accuracy": 0.7009239844977856, + "num_tokens": 590654.0, + "step": 110 + }, + { + "entropy": 0.9410129584372043, + "epoch": 0.2530311017395888, + "grad_norm": 7.28125, + "learning_rate": 9.52e-06, + "loss": 0.9810515403747558, + "mean_token_accuracy": 0.709618617594242, + "num_tokens": 643080.0, + "step": 120 + }, + { + "entropy": 0.9265115469694137, + "epoch": 0.2741170268845546, + "grad_norm": 7.46875, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.9485825538635254, + "mean_token_accuracy": 0.7162063598632813, + "num_tokens": 697254.0, + "step": 130 + }, + { + "entropy": 0.9009825974702835, + "epoch": 0.2952029520295203, + "grad_norm": 6.96875, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.9119878768920898, + "mean_token_accuracy": 0.7238413639366627, + "num_tokens": 751774.0, + "step": 140 + }, + { + "entropy": 0.9040703341364861, + "epoch": 0.316288877174486, + "grad_norm": 7.21875, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.9283308029174805, + "mean_token_accuracy": 0.7166316144168376, + "num_tokens": 808927.0, + "step": 150 + }, + { + "entropy": 0.8878034949302673, + "epoch": 0.33737480231945177, + "grad_norm": 7.375, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.9068665504455566, + "mean_token_accuracy": 0.7254431843757629, + "num_tokens": 858559.0, + "step": 160 + }, + { + "entropy": 0.887480016052723, + "epoch": 0.3584607274644175, + "grad_norm": 7.65625, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.8971159934997559, + "mean_token_accuracy": 0.7258390121161937, + "num_tokens": 912285.0, + "step": 170 + }, + { + "entropy": 0.8527634374797344, + "epoch": 0.37954665260938325, + "grad_norm": 7.6875, + "learning_rate": 1.432e-05, + "loss": 0.8586221694946289, + "mean_token_accuracy": 0.7359919100999832, + "num_tokens": 969385.0, + "step": 180 + }, + { + "entropy": 0.8445627830922604, + "epoch": 0.40063257775434896, + "grad_norm": 7.03125, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.855714225769043, + "mean_token_accuracy": 0.7341729424893856, + "num_tokens": 1026106.0, + "step": 190 + }, + { + "entropy": 0.841130618005991, + "epoch": 0.42171850289931473, + "grad_norm": 7.21875, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.8620488166809082, + "mean_token_accuracy": 0.7361006513237953, + "num_tokens": 1079743.0, + "step": 200 + }, + { + "entropy": 0.831258487701416, + "epoch": 0.44280442804428044, + "grad_norm": 7.0, + "learning_rate": 1.672e-05, + "loss": 0.8441590309143067, + "mean_token_accuracy": 0.7379129216074943, + "num_tokens": 1131453.0, + "step": 210 + }, + { + "entropy": 0.79731200709939, + "epoch": 0.46389035318924615, + "grad_norm": 7.875, + "learning_rate": 1.752e-05, + "loss": 0.8135190963745117, + "mean_token_accuracy": 0.7497627973556519, + "num_tokens": 1182486.0, + "step": 220 + }, + { + "entropy": 0.8269097700715065, + "epoch": 0.4849762783342119, + "grad_norm": 7.5625, + "learning_rate": 1.832e-05, + "loss": 0.8399410247802734, + "mean_token_accuracy": 0.7389348305761814, + "num_tokens": 1238136.0, + "step": 230 + }, + { + "entropy": 0.8093138128519058, + "epoch": 0.5060622034791776, + "grad_norm": 7.40625, + "learning_rate": 1.912e-05, + "loss": 0.810025691986084, + "mean_token_accuracy": 0.7460841409862041, + "num_tokens": 1294939.0, + "step": 240 + }, + { + "entropy": 0.8069386176764966, + "epoch": 0.5271481286241434, + "grad_norm": 6.84375, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8215888023376465, + "mean_token_accuracy": 0.7438369721174241, + "num_tokens": 1346462.0, + "step": 250 + }, + { + "entropy": 0.8088659413158894, + "epoch": 0.5482340537691092, + "grad_norm": 6.75, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8201449394226075, + "mean_token_accuracy": 0.7393839374184609, + "num_tokens": 1397444.0, + "step": 260 + }, + { + "entropy": 0.7969542622566224, + "epoch": 0.5693199789140748, + "grad_norm": 6.59375, + "learning_rate": 1.9831111111111113e-05, + "loss": 0.8009759902954101, + "mean_token_accuracy": 0.7512488305568695, + "num_tokens": 1453900.0, + "step": 270 + }, + { + "entropy": 0.780416651815176, + "epoch": 0.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.9742222222222223e-05, + "loss": 0.7874263763427735, + "mean_token_accuracy": 0.7526524804532528, + "num_tokens": 1505529.0, + "step": 280 + }, + { + "entropy": 0.7752278268337249, + "epoch": 0.6114918292040064, + "grad_norm": 6.875, + "learning_rate": 1.9653333333333334e-05, + "loss": 0.8075678825378418, + "mean_token_accuracy": 0.746569037437439, + "num_tokens": 1562473.0, + "step": 290 + }, + { + "entropy": 0.7895354829728604, + "epoch": 0.632577754348972, + "grad_norm": 6.6875, + "learning_rate": 1.9564444444444444e-05, + "loss": 0.7880948543548584, + "mean_token_accuracy": 0.7494045957922936, + "num_tokens": 1613320.0, + "step": 300 + }, + { + "entropy": 0.7653828673064709, + "epoch": 0.6536636794939378, + "grad_norm": 6.78125, + "learning_rate": 1.9475555555555558e-05, + "loss": 0.7780828475952148, + "mean_token_accuracy": 0.7556171745061875, + "num_tokens": 1662847.0, + "step": 310 + }, + { + "entropy": 0.7652887552976608, + "epoch": 0.6747496046389035, + "grad_norm": 6.375, + "learning_rate": 1.938666666666667e-05, + "loss": 0.7539787292480469, + "mean_token_accuracy": 0.7582654722034932, + "num_tokens": 1718553.0, + "step": 320 + }, + { + "entropy": 0.7076033115386963, + "epoch": 0.6958355297838693, + "grad_norm": 7.15625, + "learning_rate": 1.929777777777778e-05, + "loss": 0.7396605491638184, + "mean_token_accuracy": 0.7659220136702061, + "num_tokens": 1771024.0, + "step": 330 + }, + { + "entropy": 0.7509547933936119, + "epoch": 0.716921454928835, + "grad_norm": 6.15625, + "learning_rate": 1.920888888888889e-05, + "loss": 0.7459388256072998, + "mean_token_accuracy": 0.7644543245434761, + "num_tokens": 1821303.0, + "step": 340 + }, + { + "entropy": 0.7591827407479286, + "epoch": 0.7380073800738007, + "grad_norm": 7.03125, + "learning_rate": 1.912e-05, + "loss": 0.7627717971801757, + "mean_token_accuracy": 0.7567056350409984, + "num_tokens": 1868674.0, + "step": 350 + }, + { + "entropy": 0.7316406652331352, + "epoch": 0.7590933052187665, + "grad_norm": 6.59375, + "learning_rate": 1.9031111111111114e-05, + "loss": 0.7425707340240478, + "mean_token_accuracy": 0.7653280735015869, + "num_tokens": 1924619.0, + "step": 360 + }, + { + "entropy": 0.7539067789912224, + "epoch": 0.7801792303637322, + "grad_norm": 7.0625, + "learning_rate": 1.8942222222222224e-05, + "loss": 0.7716987609863282, + "mean_token_accuracy": 0.7524559155106545, + "num_tokens": 1978795.0, + "step": 370 + }, + { + "entropy": 0.7687754213809967, + "epoch": 0.8012651555086979, + "grad_norm": 7.1875, + "learning_rate": 1.8853333333333335e-05, + "loss": 0.7734373569488525, + "mean_token_accuracy": 0.7497503250837326, + "num_tokens": 2029994.0, + "step": 380 + }, + { + "entropy": 0.7508607231080532, + "epoch": 0.8223510806536637, + "grad_norm": 7.09375, + "learning_rate": 1.8764444444444445e-05, + "loss": 0.7575721740722656, + "mean_token_accuracy": 0.7572125047445297, + "num_tokens": 2082653.0, + "step": 390 + }, + { + "entropy": 0.7252861328423024, + "epoch": 0.8434370057986295, + "grad_norm": 6.5625, + "learning_rate": 1.867555555555556e-05, + "loss": 0.7381186485290527, + "mean_token_accuracy": 0.7612978041172027, + "num_tokens": 2135276.0, + "step": 400 + }, + { + "entropy": 0.7339485622942448, + "epoch": 0.8645229309435951, + "grad_norm": 6.625, + "learning_rate": 1.858666666666667e-05, + "loss": 0.7348906993865967, + "mean_token_accuracy": 0.7663561321794987, + "num_tokens": 2187466.0, + "step": 410 + }, + { + "entropy": 0.7272624362260103, + "epoch": 0.8856088560885609, + "grad_norm": 6.90625, + "learning_rate": 1.849777777777778e-05, + "loss": 0.749025821685791, + "mean_token_accuracy": 0.7617379955947399, + "num_tokens": 2237434.0, + "step": 420 + }, + { + "entropy": 0.7477380312979222, + "epoch": 0.9066947812335266, + "grad_norm": 6.34375, + "learning_rate": 1.840888888888889e-05, + "loss": 0.7498275756835937, + "mean_token_accuracy": 0.7604753144085408, + "num_tokens": 2288985.0, + "step": 430 + }, + { + "entropy": 0.7346535548567772, + "epoch": 0.9277807063784923, + "grad_norm": 6.9375, + "learning_rate": 1.832e-05, + "loss": 0.739972448348999, + "mean_token_accuracy": 0.7658773727715016, + "num_tokens": 2341187.0, + "step": 440 + }, + { + "entropy": 0.7167556572705507, + "epoch": 0.9488666315234581, + "grad_norm": 6.4375, + "learning_rate": 1.823111111111111e-05, + "loss": 0.7350784778594971, + "mean_token_accuracy": 0.7644179269671441, + "num_tokens": 2393385.0, + "step": 450 + }, + { + "entropy": 0.7537519961595536, + "epoch": 0.9699525566684238, + "grad_norm": 7.28125, + "learning_rate": 1.814222222222222e-05, + "loss": 0.757652759552002, + "mean_token_accuracy": 0.7587068513035774, + "num_tokens": 2442248.0, + "step": 460 + }, + { + "entropy": 0.72038439437747, + "epoch": 0.9910384818133896, + "grad_norm": 7.40625, + "learning_rate": 1.8053333333333332e-05, + "loss": 0.7361824035644531, + "mean_token_accuracy": 0.7609050884842873, + "num_tokens": 2498394.0, + "step": 470 + }, + { + "entropy": 0.735570513718837, + "epoch": 1.0105429625724829, + "grad_norm": 6.875, + "learning_rate": 1.7964444444444446e-05, + "loss": 0.730868148803711, + "mean_token_accuracy": 0.7710521068121936, + "num_tokens": 2545422.0, + "step": 480 + }, + { + "entropy": 0.6775223582983017, + "epoch": 1.0316288877174487, + "grad_norm": 7.125, + "learning_rate": 1.7875555555555556e-05, + "loss": 0.6740085124969483, + "mean_token_accuracy": 0.7795436836779117, + "num_tokens": 2597144.0, + "step": 490 + }, + { + "entropy": 0.6628748029470444, + "epoch": 1.0527148128624144, + "grad_norm": 6.84375, + "learning_rate": 1.7786666666666667e-05, + "loss": 0.6774313926696778, + "mean_token_accuracy": 0.7789938881993294, + "num_tokens": 2651938.0, + "step": 500 + }, + { + "entropy": 0.6695188779383898, + "epoch": 1.07380073800738, + "grad_norm": 7.0, + "learning_rate": 1.7697777777777777e-05, + "loss": 0.664512825012207, + "mean_token_accuracy": 0.7832319870591163, + "num_tokens": 2704948.0, + "step": 510 + }, + { + "entropy": 0.6812492057681083, + "epoch": 1.0948866631523457, + "grad_norm": 7.0, + "learning_rate": 1.760888888888889e-05, + "loss": 0.7036191940307617, + "mean_token_accuracy": 0.7707728065550328, + "num_tokens": 2757576.0, + "step": 520 + }, + { + "entropy": 0.6861097872257232, + "epoch": 1.1159725882973115, + "grad_norm": 7.3125, + "learning_rate": 1.752e-05, + "loss": 0.6651991844177246, + "mean_token_accuracy": 0.7841649115085602, + "num_tokens": 2808989.0, + "step": 530 + }, + { + "entropy": 0.6694958411157131, + "epoch": 1.1370585134422773, + "grad_norm": 6.78125, + "learning_rate": 1.7431111111111112e-05, + "loss": 0.6833772659301758, + "mean_token_accuracy": 0.7775424145162105, + "num_tokens": 2864507.0, + "step": 540 + }, + { + "entropy": 0.6568067740648985, + "epoch": 1.158144438587243, + "grad_norm": 6.21875, + "learning_rate": 1.7342222222222226e-05, + "loss": 0.6552294254302978, + "mean_token_accuracy": 0.7832994520664215, + "num_tokens": 2919201.0, + "step": 550 + }, + { + "entropy": 0.6432577680796385, + "epoch": 1.1792303637322088, + "grad_norm": 6.65625, + "learning_rate": 1.7253333333333336e-05, + "loss": 0.6560451030731201, + "mean_token_accuracy": 0.7863748580217361, + "num_tokens": 2972434.0, + "step": 560 + }, + { + "entropy": 0.6655397050082683, + "epoch": 1.2003162888771746, + "grad_norm": 6.09375, + "learning_rate": 1.7164444444444447e-05, + "loss": 0.6878513336181641, + "mean_token_accuracy": 0.7753379993140698, + "num_tokens": 3025099.0, + "step": 570 + }, + { + "entropy": 0.686093359068036, + "epoch": 1.2214022140221403, + "grad_norm": 7.09375, + "learning_rate": 1.7075555555555557e-05, + "loss": 0.6911579132080078, + "mean_token_accuracy": 0.7787310972809791, + "num_tokens": 3076307.0, + "step": 580 + }, + { + "entropy": 0.6812980778515338, + "epoch": 1.2424881391671059, + "grad_norm": 7.125, + "learning_rate": 1.6986666666666668e-05, + "loss": 0.6704156398773193, + "mean_token_accuracy": 0.7793935135006904, + "num_tokens": 3130053.0, + "step": 590 + }, + { + "entropy": 0.6409769468009472, + "epoch": 1.2635740643120716, + "grad_norm": 7.4375, + "learning_rate": 1.6897777777777778e-05, + "loss": 0.661864185333252, + "mean_token_accuracy": 0.7865670621395111, + "num_tokens": 3183331.0, + "step": 600 + }, + { + "entropy": 0.6799454756081105, + "epoch": 1.2846599894570374, + "grad_norm": 6.65625, + "learning_rate": 1.680888888888889e-05, + "loss": 0.6818383693695068, + "mean_token_accuracy": 0.778287123888731, + "num_tokens": 3239561.0, + "step": 610 + }, + { + "entropy": 0.6814555354416371, + "epoch": 1.3057459146020032, + "grad_norm": 6.3125, + "learning_rate": 1.672e-05, + "loss": 0.679498815536499, + "mean_token_accuracy": 0.7788635492324829, + "num_tokens": 3292996.0, + "step": 620 + }, + { + "entropy": 0.667108029127121, + "epoch": 1.326831839746969, + "grad_norm": 6.90625, + "learning_rate": 1.6631111111111113e-05, + "loss": 0.6758651733398438, + "mean_token_accuracy": 0.7792007848620415, + "num_tokens": 3345736.0, + "step": 630 + }, + { + "entropy": 0.6692709110677242, + "epoch": 1.3479177648919347, + "grad_norm": 6.96875, + "learning_rate": 1.6542222222222223e-05, + "loss": 0.6796128273010253, + "mean_token_accuracy": 0.7766051791608334, + "num_tokens": 3398076.0, + "step": 640 + }, + { + "entropy": 0.6564048878848553, + "epoch": 1.3690036900369003, + "grad_norm": 6.9375, + "learning_rate": 1.6453333333333334e-05, + "loss": 0.6633264064788819, + "mean_token_accuracy": 0.7854946702718735, + "num_tokens": 3450796.0, + "step": 650 + }, + { + "entropy": 0.6611979272216558, + "epoch": 1.390089615181866, + "grad_norm": 6.65625, + "learning_rate": 1.6364444444444444e-05, + "loss": 0.6661429405212402, + "mean_token_accuracy": 0.7835663430392742, + "num_tokens": 3503863.0, + "step": 660 + }, + { + "entropy": 0.6534961331635714, + "epoch": 1.4111755403268318, + "grad_norm": 6.9375, + "learning_rate": 1.6275555555555558e-05, + "loss": 0.6586530208587646, + "mean_token_accuracy": 0.7844067431986332, + "num_tokens": 3556265.0, + "step": 670 + }, + { + "entropy": 0.6516873501241207, + "epoch": 1.4322614654717976, + "grad_norm": 6.40625, + "learning_rate": 1.618666666666667e-05, + "loss": 0.6522529125213623, + "mean_token_accuracy": 0.7855845786631107, + "num_tokens": 3611423.0, + "step": 680 + }, + { + "entropy": 0.6764013767242432, + "epoch": 1.4533473906167633, + "grad_norm": 6.71875, + "learning_rate": 1.609777777777778e-05, + "loss": 0.6884442806243897, + "mean_token_accuracy": 0.7756352722644806, + "num_tokens": 3666630.0, + "step": 690 + }, + { + "entropy": 0.6554486058652401, + "epoch": 1.474433315761729, + "grad_norm": 7.40625, + "learning_rate": 1.600888888888889e-05, + "loss": 0.6563316822052002, + "mean_token_accuracy": 0.7854672558605671, + "num_tokens": 3718719.0, + "step": 700 + }, + { + "entropy": 0.6385185327380896, + "epoch": 1.4955192409066949, + "grad_norm": 6.59375, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.6513954639434815, + "mean_token_accuracy": 0.7851097173988819, + "num_tokens": 3771822.0, + "step": 710 + }, + { + "entropy": 0.654766371101141, + "epoch": 1.5166051660516606, + "grad_norm": 6.5, + "learning_rate": 1.5831111111111114e-05, + "loss": 0.6622748851776123, + "mean_token_accuracy": 0.7847663886845112, + "num_tokens": 3823795.0, + "step": 720 + }, + { + "entropy": 0.6597310703247785, + "epoch": 1.5376910911966264, + "grad_norm": 6.375, + "learning_rate": 1.5742222222222224e-05, + "loss": 0.6634318351745605, + "mean_token_accuracy": 0.7861979939043522, + "num_tokens": 3875784.0, + "step": 730 + }, + { + "entropy": 0.6637851595878601, + "epoch": 1.558777016341592, + "grad_norm": 6.9375, + "learning_rate": 1.5653333333333335e-05, + "loss": 0.6621220111846924, + "mean_token_accuracy": 0.7886320613324642, + "num_tokens": 3929412.0, + "step": 740 + }, + { + "entropy": 0.637692479044199, + "epoch": 1.5798629414865577, + "grad_norm": 6.59375, + "learning_rate": 1.5564444444444445e-05, + "loss": 0.6402386665344239, + "mean_token_accuracy": 0.7895873270928859, + "num_tokens": 3982472.0, + "step": 750 + }, + { + "entropy": 0.6578179821372032, + "epoch": 1.6009488666315235, + "grad_norm": 7.25, + "learning_rate": 1.5475555555555556e-05, + "loss": 0.6751936435699463, + "mean_token_accuracy": 0.7776105716824532, + "num_tokens": 4037829.0, + "step": 760 + }, + { + "entropy": 0.6653612300753593, + "epoch": 1.6220347917764892, + "grad_norm": 6.9375, + "learning_rate": 1.5386666666666666e-05, + "loss": 0.6717413425445556, + "mean_token_accuracy": 0.777862798422575, + "num_tokens": 4093735.0, + "step": 770 + }, + { + "entropy": 0.6561707239598036, + "epoch": 1.6431207169214548, + "grad_norm": 7.03125, + "learning_rate": 1.5297777777777777e-05, + "loss": 0.6501868724822998, + "mean_token_accuracy": 0.7863908343017101, + "num_tokens": 4148410.0, + "step": 780 + }, + { + "entropy": 0.6655665650963783, + "epoch": 1.6642066420664205, + "grad_norm": 6.59375, + "learning_rate": 1.520888888888889e-05, + "loss": 0.679684829711914, + "mean_token_accuracy": 0.778567586839199, + "num_tokens": 4202907.0, + "step": 790 + }, + { + "entropy": 0.6781473312526941, + "epoch": 1.6852925672113863, + "grad_norm": 6.875, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.6722404479980468, + "mean_token_accuracy": 0.7763243250548839, + "num_tokens": 4256213.0, + "step": 800 + }, + { + "entropy": 0.6321199681609869, + "epoch": 1.706378492356352, + "grad_norm": 6.78125, + "learning_rate": 1.5031111111111111e-05, + "loss": 0.6404708385467529, + "mean_token_accuracy": 0.7903632469475269, + "num_tokens": 4309268.0, + "step": 810 + }, + { + "entropy": 0.6424700867384672, + "epoch": 1.7274644175013179, + "grad_norm": 7.15625, + "learning_rate": 1.4942222222222222e-05, + "loss": 0.6444578170776367, + "mean_token_accuracy": 0.7881530813872815, + "num_tokens": 4360761.0, + "step": 820 + }, + { + "entropy": 0.6615507304668427, + "epoch": 1.7485503426462836, + "grad_norm": 6.0625, + "learning_rate": 1.4853333333333336e-05, + "loss": 0.6604718208312989, + "mean_token_accuracy": 0.782805722206831, + "num_tokens": 4411678.0, + "step": 830 + }, + { + "entropy": 0.6196760050952435, + "epoch": 1.7696362677912494, + "grad_norm": 6.6875, + "learning_rate": 1.4764444444444446e-05, + "loss": 0.6261836051940918, + "mean_token_accuracy": 0.7926589228212834, + "num_tokens": 4467254.0, + "step": 840 + }, + { + "entropy": 0.6139154363423586, + "epoch": 1.7907221929362152, + "grad_norm": 7.21875, + "learning_rate": 1.4675555555555557e-05, + "loss": 0.6301932811737061, + "mean_token_accuracy": 0.7918383464217186, + "num_tokens": 4517383.0, + "step": 850 + }, + { + "entropy": 0.6410171519964933, + "epoch": 1.811808118081181, + "grad_norm": 7.1875, + "learning_rate": 1.4586666666666667e-05, + "loss": 0.655810022354126, + "mean_token_accuracy": 0.7856725618243218, + "num_tokens": 4569980.0, + "step": 860 + }, + { + "entropy": 0.6666022066026926, + "epoch": 1.8328940432261467, + "grad_norm": 6.5625, + "learning_rate": 1.449777777777778e-05, + "loss": 0.6677482604980469, + "mean_token_accuracy": 0.7798060193657875, + "num_tokens": 4622612.0, + "step": 870 + }, + { + "entropy": 0.6398684818297624, + "epoch": 1.8539799683711122, + "grad_norm": 7.84375, + "learning_rate": 1.440888888888889e-05, + "loss": 0.6501501083374024, + "mean_token_accuracy": 0.7857734531164169, + "num_tokens": 4676525.0, + "step": 880 + }, + { + "entropy": 0.6285852700471878, + "epoch": 1.875065893516078, + "grad_norm": 7.0625, + "learning_rate": 1.432e-05, + "loss": 0.6422877311706543, + "mean_token_accuracy": 0.7884316690266132, + "num_tokens": 4734090.0, + "step": 890 + }, + { + "entropy": 0.6409385818988085, + "epoch": 1.8961518186610438, + "grad_norm": 7.9375, + "learning_rate": 1.4231111111111114e-05, + "loss": 0.6352181434631348, + "mean_token_accuracy": 0.7952412232756615, + "num_tokens": 4788537.0, + "step": 900 + }, + { + "entropy": 0.6456746399402619, + "epoch": 1.9172377438060095, + "grad_norm": 6.90625, + "learning_rate": 1.4142222222222224e-05, + "loss": 0.6564863204956055, + "mean_token_accuracy": 0.789187741279602, + "num_tokens": 4840342.0, + "step": 910 + }, + { + "entropy": 0.6360713161528111, + "epoch": 1.938323668950975, + "grad_norm": 6.3125, + "learning_rate": 1.4053333333333335e-05, + "loss": 0.6300373554229737, + "mean_token_accuracy": 0.7876299776136875, + "num_tokens": 4891524.0, + "step": 920 + }, + { + "entropy": 0.6285211775451899, + "epoch": 1.9594095940959408, + "grad_norm": 7.78125, + "learning_rate": 1.3964444444444445e-05, + "loss": 0.6337003231048584, + "mean_token_accuracy": 0.7889726974070073, + "num_tokens": 4939573.0, + "step": 930 + }, + { + "entropy": 0.6301535632461309, + "epoch": 1.9804955192409066, + "grad_norm": 6.84375, + "learning_rate": 1.3875555555555557e-05, + "loss": 0.6374162673950196, + "mean_token_accuracy": 0.7898242607712745, + "num_tokens": 4991267.0, + "step": 940 + }, + { + "entropy": 0.6537058566873138, + "epoch": 2.0, + "grad_norm": 13.3125, + "learning_rate": 1.3786666666666668e-05, + "loss": 0.641156530380249, + "mean_token_accuracy": 0.7822844466647586, + "num_tokens": 5036304.0, + "step": 950 + }, + { + "entropy": 0.6411334682255984, + "epoch": 2.0210859251449658, + "grad_norm": 7.1875, + "learning_rate": 1.3697777777777778e-05, + "loss": 0.6083747386932373, + "mean_token_accuracy": 0.8002627611160278, + "num_tokens": 5089737.0, + "step": 960 + }, + { + "entropy": 0.5841103594750165, + "epoch": 2.0421718502899315, + "grad_norm": 6.53125, + "learning_rate": 1.3608888888888889e-05, + "loss": 0.6012133121490478, + "mean_token_accuracy": 0.8027123242616654, + "num_tokens": 5143330.0, + "step": 970 + }, + { + "entropy": 0.590392829477787, + "epoch": 2.0632577754348973, + "grad_norm": 6.46875, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.5829256534576416, + "mean_token_accuracy": 0.8068650290369987, + "num_tokens": 5198614.0, + "step": 980 + }, + { + "entropy": 0.6148912344127894, + "epoch": 2.084343700579863, + "grad_norm": 6.59375, + "learning_rate": 1.3431111111111113e-05, + "loss": 0.6117685317993165, + "mean_token_accuracy": 0.7951427385210991, + "num_tokens": 5252755.0, + "step": 990 + }, + { + "entropy": 0.5893940377980471, + "epoch": 2.105429625724829, + "grad_norm": 6.625, + "learning_rate": 1.3342222222222224e-05, + "loss": 0.5790496349334717, + "mean_token_accuracy": 0.809234394133091, + "num_tokens": 5304859.0, + "step": 1000 + }, + { + "entropy": 0.587013079598546, + "epoch": 2.1265155508697946, + "grad_norm": 6.9375, + "learning_rate": 1.3253333333333334e-05, + "loss": 0.5892822265625, + "mean_token_accuracy": 0.8040889374911785, + "num_tokens": 5357877.0, + "step": 1010 + }, + { + "entropy": 0.5877181258052587, + "epoch": 2.14760147601476, + "grad_norm": 6.09375, + "learning_rate": 1.3164444444444446e-05, + "loss": 0.595665693283081, + "mean_token_accuracy": 0.8036225430667401, + "num_tokens": 5413504.0, + "step": 1020 + }, + { + "entropy": 0.5848948258906603, + "epoch": 2.1686874011597257, + "grad_norm": 6.4375, + "learning_rate": 1.3075555555555557e-05, + "loss": 0.5771171092987061, + "mean_token_accuracy": 0.8047491163015366, + "num_tokens": 5466549.0, + "step": 1030 + }, + { + "entropy": 0.5815035384148359, + "epoch": 2.1897733263046915, + "grad_norm": 6.96875, + "learning_rate": 1.2986666666666667e-05, + "loss": 0.5768617153167724, + "mean_token_accuracy": 0.8057669088244438, + "num_tokens": 5519515.0, + "step": 1040 + }, + { + "entropy": 0.58335498906672, + "epoch": 2.2108592514496572, + "grad_norm": 7.6875, + "learning_rate": 1.2897777777777778e-05, + "loss": 0.6007503509521485, + "mean_token_accuracy": 0.8005498602986336, + "num_tokens": 5570188.0, + "step": 1050 + }, + { + "entropy": 0.6068603809922933, + "epoch": 2.231945176594623, + "grad_norm": 6.5625, + "learning_rate": 1.2808888888888891e-05, + "loss": 0.6024115562438965, + "mean_token_accuracy": 0.7973541341722011, + "num_tokens": 5624190.0, + "step": 1060 + }, + { + "entropy": 0.5774707745760679, + "epoch": 2.2530311017395888, + "grad_norm": 6.53125, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.5705479145050049, + "mean_token_accuracy": 0.8100002221763134, + "num_tokens": 5675656.0, + "step": 1070 + }, + { + "entropy": 0.5851722996681928, + "epoch": 2.2741170268845545, + "grad_norm": 7.3125, + "learning_rate": 1.2631111111111112e-05, + "loss": 0.5905722141265869, + "mean_token_accuracy": 0.8069308035075664, + "num_tokens": 5728878.0, + "step": 1080 + }, + { + "entropy": 0.594813584536314, + "epoch": 2.2952029520295203, + "grad_norm": 6.875, + "learning_rate": 1.2542222222222223e-05, + "loss": 0.5923901557922363, + "mean_token_accuracy": 0.8034852437675, + "num_tokens": 5783257.0, + "step": 1090 + }, + { + "entropy": 0.5841789487749338, + "epoch": 2.316288877174486, + "grad_norm": 7.0, + "learning_rate": 1.2453333333333335e-05, + "loss": 0.5864346504211426, + "mean_token_accuracy": 0.8033530995249748, + "num_tokens": 5834581.0, + "step": 1100 + }, + { + "entropy": 0.6010521166026592, + "epoch": 2.337374802319452, + "grad_norm": 6.9375, + "learning_rate": 1.2364444444444445e-05, + "loss": 0.5941137313842774, + "mean_token_accuracy": 0.8033333070576191, + "num_tokens": 5887458.0, + "step": 1110 + }, + { + "entropy": 0.5919341973960399, + "epoch": 2.3584607274644176, + "grad_norm": 8.75, + "learning_rate": 1.2275555555555556e-05, + "loss": 0.6103182315826416, + "mean_token_accuracy": 0.7999237082898617, + "num_tokens": 5941284.0, + "step": 1120 + }, + { + "entropy": 0.5942621320486069, + "epoch": 2.3795466526093834, + "grad_norm": 7.375, + "learning_rate": 1.2186666666666666e-05, + "loss": 0.5983363151550293, + "mean_token_accuracy": 0.8016222022473812, + "num_tokens": 5995817.0, + "step": 1130 + }, + { + "entropy": 0.6046227600425482, + "epoch": 2.400632577754349, + "grad_norm": 6.40625, + "learning_rate": 1.209777777777778e-05, + "loss": 0.6026669025421143, + "mean_token_accuracy": 0.8024684950709343, + "num_tokens": 6050783.0, + "step": 1140 + }, + { + "entropy": 0.5998306684195995, + "epoch": 2.421718502899315, + "grad_norm": 6.59375, + "learning_rate": 1.200888888888889e-05, + "loss": 0.5893473625183105, + "mean_token_accuracy": 0.803314708918333, + "num_tokens": 6103371.0, + "step": 1150 + }, + { + "entropy": 0.593693134188652, + "epoch": 2.4428044280442807, + "grad_norm": 6.5, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.6076583862304688, + "mean_token_accuracy": 0.7995239071547985, + "num_tokens": 6155502.0, + "step": 1160 + }, + { + "entropy": 0.5827030725777149, + "epoch": 2.463890353189246, + "grad_norm": 6.84375, + "learning_rate": 1.1831111111111112e-05, + "loss": 0.5882821559906006, + "mean_token_accuracy": 0.7929977595806121, + "num_tokens": 6208060.0, + "step": 1170 + }, + { + "entropy": 0.5910260166972876, + "epoch": 2.4849762783342118, + "grad_norm": 7.53125, + "learning_rate": 1.1742222222222224e-05, + "loss": 0.5860947132110595, + "mean_token_accuracy": 0.8049415498971939, + "num_tokens": 6261588.0, + "step": 1180 + }, + { + "entropy": 0.5704694345593453, + "epoch": 2.5060622034791775, + "grad_norm": 6.75, + "learning_rate": 1.1653333333333334e-05, + "loss": 0.5749064445495605, + "mean_token_accuracy": 0.8084491945803165, + "num_tokens": 6314963.0, + "step": 1190 + }, + { + "entropy": 0.597440430521965, + "epoch": 2.5271481286241433, + "grad_norm": 6.5, + "learning_rate": 1.1564444444444445e-05, + "loss": 0.5958696842193604, + "mean_token_accuracy": 0.8057812102138996, + "num_tokens": 6368741.0, + "step": 1200 + }, + { + "entropy": 0.6070981964468956, + "epoch": 2.548234053769109, + "grad_norm": 7.96875, + "learning_rate": 1.1475555555555557e-05, + "loss": 0.6143896102905273, + "mean_token_accuracy": 0.7980599500238895, + "num_tokens": 6425424.0, + "step": 1210 + }, + { + "entropy": 0.5948727998882533, + "epoch": 2.569319978914075, + "grad_norm": 6.5, + "learning_rate": 1.1386666666666669e-05, + "loss": 0.5917119026184082, + "mean_token_accuracy": 0.8025051333010197, + "num_tokens": 6476263.0, + "step": 1220 + }, + { + "entropy": 0.5811738461256027, + "epoch": 2.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.129777777777778e-05, + "loss": 0.5865864753723145, + "mean_token_accuracy": 0.8044181451201439, + "num_tokens": 6528012.0, + "step": 1230 + }, + { + "entropy": 0.5937653455883265, + "epoch": 2.6114918292040064, + "grad_norm": 6.59375, + "learning_rate": 1.120888888888889e-05, + "loss": 0.5843213558197021, + "mean_token_accuracy": 0.8025668740272522, + "num_tokens": 6583421.0, + "step": 1240 + }, + { + "entropy": 0.573565949127078, + "epoch": 2.632577754348972, + "grad_norm": 6.5625, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.5826557159423829, + "mean_token_accuracy": 0.8070438347756863, + "num_tokens": 6636858.0, + "step": 1250 + }, + { + "entropy": 0.6216809719800949, + "epoch": 2.653663679493938, + "grad_norm": 7.09375, + "learning_rate": 1.1031111111111112e-05, + "loss": 0.6125890254974365, + "mean_token_accuracy": 0.7970895677804947, + "num_tokens": 6692168.0, + "step": 1260 + }, + { + "entropy": 0.5998566258698702, + "epoch": 2.6747496046389037, + "grad_norm": 7.0625, + "learning_rate": 1.0942222222222223e-05, + "loss": 0.5921674728393554, + "mean_token_accuracy": 0.8061520479619503, + "num_tokens": 6745834.0, + "step": 1270 + }, + { + "entropy": 0.5880189586430788, + "epoch": 2.6958355297838694, + "grad_norm": 7.59375, + "learning_rate": 1.0853333333333333e-05, + "loss": 0.5986497402191162, + "mean_token_accuracy": 0.8006219923496246, + "num_tokens": 6800244.0, + "step": 1280 + }, + { + "entropy": 0.5981763809919357, + "epoch": 2.716921454928835, + "grad_norm": 6.6875, + "learning_rate": 1.0764444444444445e-05, + "loss": 0.6020579814910889, + "mean_token_accuracy": 0.8032843962311744, + "num_tokens": 6853454.0, + "step": 1290 + }, + { + "entropy": 0.5963539965450764, + "epoch": 2.7380073800738005, + "grad_norm": 6.59375, + "learning_rate": 1.0675555555555558e-05, + "loss": 0.5908561706542969, + "mean_token_accuracy": 0.8011696465313435, + "num_tokens": 6904473.0, + "step": 1300 + }, + { + "entropy": 0.5899917531758547, + "epoch": 2.7590933052187667, + "grad_norm": 6.6875, + "learning_rate": 1.0586666666666668e-05, + "loss": 0.5744058132171631, + "mean_token_accuracy": 0.8053683824837208, + "num_tokens": 6956756.0, + "step": 1310 + }, + { + "entropy": 0.582448274642229, + "epoch": 2.780179230363732, + "grad_norm": 6.78125, + "learning_rate": 1.0497777777777779e-05, + "loss": 0.5848373889923095, + "mean_token_accuracy": 0.8054254874587059, + "num_tokens": 7008611.0, + "step": 1320 + }, + { + "entropy": 0.5906594522297383, + "epoch": 2.801265155508698, + "grad_norm": 7.75, + "learning_rate": 1.040888888888889e-05, + "loss": 0.6006948471069335, + "mean_token_accuracy": 0.7971956506371498, + "num_tokens": 7062235.0, + "step": 1330 + }, + { + "entropy": 0.5976275954395532, + "epoch": 2.8223510806536636, + "grad_norm": 6.34375, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.5997600078582763, + "mean_token_accuracy": 0.7998531877994537, + "num_tokens": 7115737.0, + "step": 1340 + }, + { + "entropy": 0.6079881630837918, + "epoch": 2.8434370057986293, + "grad_norm": 7.0, + "learning_rate": 1.0231111111111112e-05, + "loss": 0.6024625778198243, + "mean_token_accuracy": 0.7988371469080449, + "num_tokens": 7166870.0, + "step": 1350 + }, + { + "entropy": 0.6036868289113044, + "epoch": 2.864522930943595, + "grad_norm": 6.6875, + "learning_rate": 1.0142222222222222e-05, + "loss": 0.5962271213531494, + "mean_token_accuracy": 0.8011133573949337, + "num_tokens": 7219785.0, + "step": 1360 + }, + { + "entropy": 0.5746289525181055, + "epoch": 2.885608856088561, + "grad_norm": 7.5625, + "learning_rate": 1.0053333333333334e-05, + "loss": 0.5769401073455811, + "mean_token_accuracy": 0.809252617508173, + "num_tokens": 7275411.0, + "step": 1370 + }, + { + "entropy": 0.5846736520528794, + "epoch": 2.9066947812335266, + "grad_norm": 7.125, + "learning_rate": 9.964444444444445e-06, + "loss": 0.5899319171905517, + "mean_token_accuracy": 0.8028473742306232, + "num_tokens": 7326752.0, + "step": 1380 + }, + { + "entropy": 0.5813163112848997, + "epoch": 2.9277807063784924, + "grad_norm": 7.46875, + "learning_rate": 9.875555555555557e-06, + "loss": 0.5799898147583008, + "mean_token_accuracy": 0.8026903085410595, + "num_tokens": 7375608.0, + "step": 1390 + }, + { + "entropy": 0.5724870227277279, + "epoch": 2.948866631523458, + "grad_norm": 6.90625, + "learning_rate": 9.786666666666667e-06, + "loss": 0.5783823490142822, + "mean_token_accuracy": 0.8059644535183906, + "num_tokens": 7430145.0, + "step": 1400 + }, + { + "entropy": 0.5778653729707003, + "epoch": 2.969952556668424, + "grad_norm": 6.84375, + "learning_rate": 9.697777777777778e-06, + "loss": 0.5640594005584717, + "mean_token_accuracy": 0.8100298568606377, + "num_tokens": 7482976.0, + "step": 1410 + }, + { + "entropy": 0.5823766525834799, + "epoch": 2.9910384818133897, + "grad_norm": 6.5625, + "learning_rate": 9.60888888888889e-06, + "loss": 0.5931865692138671, + "mean_token_accuracy": 0.8023995622992516, + "num_tokens": 7533043.0, + "step": 1420 + }, + { + "entropy": 0.5924574036050487, + "epoch": 3.0105429625724827, + "grad_norm": 6.53125, + "learning_rate": 9.52e-06, + "loss": 0.5867043972015381, + "mean_token_accuracy": 0.803342911842707, + "num_tokens": 7581993.0, + "step": 1430 + }, + { + "entropy": 0.579814862459898, + "epoch": 3.0316288877174484, + "grad_norm": 6.34375, + "learning_rate": 9.431111111111112e-06, + "loss": 0.5730488300323486, + "mean_token_accuracy": 0.8101175747811794, + "num_tokens": 7639343.0, + "step": 1440 + }, + { + "entropy": 0.560778708383441, + "epoch": 3.052714812862414, + "grad_norm": 7.40625, + "learning_rate": 9.342222222222223e-06, + "loss": 0.5500640392303466, + "mean_token_accuracy": 0.8135757654905319, + "num_tokens": 7690695.0, + "step": 1450 + }, + { + "entropy": 0.5529176875948906, + "epoch": 3.07380073800738, + "grad_norm": 7.21875, + "learning_rate": 9.253333333333333e-06, + "loss": 0.549665117263794, + "mean_token_accuracy": 0.81509620398283, + "num_tokens": 7746705.0, + "step": 1460 + }, + { + "entropy": 0.5382925808429718, + "epoch": 3.0948866631523457, + "grad_norm": 7.5, + "learning_rate": 9.164444444444446e-06, + "loss": 0.54415283203125, + "mean_token_accuracy": 0.819620954990387, + "num_tokens": 7799255.0, + "step": 1470 + }, + { + "entropy": 0.551479097828269, + "epoch": 3.1159725882973115, + "grad_norm": 7.125, + "learning_rate": 9.075555555555556e-06, + "loss": 0.5520545959472656, + "mean_token_accuracy": 0.8147840730845928, + "num_tokens": 7852753.0, + "step": 1480 + }, + { + "entropy": 0.571834359690547, + "epoch": 3.1370585134422773, + "grad_norm": 7.59375, + "learning_rate": 8.986666666666666e-06, + "loss": 0.5553510189056396, + "mean_token_accuracy": 0.8117231354117393, + "num_tokens": 7904298.0, + "step": 1490 + }, + { + "entropy": 0.5676855646073818, + "epoch": 3.158144438587243, + "grad_norm": 7.75, + "learning_rate": 8.897777777777779e-06, + "loss": 0.5595914840698242, + "mean_token_accuracy": 0.8130396403372288, + "num_tokens": 7956705.0, + "step": 1500 + }, + { + "entropy": 0.5613869089633227, + "epoch": 3.179230363732209, + "grad_norm": 7.78125, + "learning_rate": 8.80888888888889e-06, + "loss": 0.544979190826416, + "mean_token_accuracy": 0.8170888110995292, + "num_tokens": 8008023.0, + "step": 1510 + }, + { + "entropy": 0.570792680233717, + "epoch": 3.2003162888771746, + "grad_norm": 6.75, + "learning_rate": 8.720000000000001e-06, + "loss": 0.560499906539917, + "mean_token_accuracy": 0.8146887816488743, + "num_tokens": 8058344.0, + "step": 1520 + }, + { + "entropy": 0.5498528245836496, + "epoch": 3.2214022140221403, + "grad_norm": 7.15625, + "learning_rate": 8.631111111111112e-06, + "loss": 0.5479342937469482, + "mean_token_accuracy": 0.8137361407279968, + "num_tokens": 8110587.0, + "step": 1530 + }, + { + "entropy": 0.5499754682183265, + "epoch": 3.242488139167106, + "grad_norm": 6.9375, + "learning_rate": 8.542222222222222e-06, + "loss": 0.5445993423461915, + "mean_token_accuracy": 0.8179522223770619, + "num_tokens": 8160497.0, + "step": 1540 + }, + { + "entropy": 0.572133193910122, + "epoch": 3.263574064312072, + "grad_norm": 6.84375, + "learning_rate": 8.453333333333334e-06, + "loss": 0.5740565776824951, + "mean_token_accuracy": 0.8101011097431183, + "num_tokens": 8214878.0, + "step": 1550 + }, + { + "entropy": 0.5666386324912309, + "epoch": 3.2846599894570376, + "grad_norm": 7.59375, + "learning_rate": 8.364444444444445e-06, + "loss": 0.5723002433776856, + "mean_token_accuracy": 0.8091295748949051, + "num_tokens": 8267314.0, + "step": 1560 + }, + { + "entropy": 0.5659929499030113, + "epoch": 3.305745914602003, + "grad_norm": 6.96875, + "learning_rate": 8.275555555555557e-06, + "loss": 0.5623570442199707, + "mean_token_accuracy": 0.8093601904809475, + "num_tokens": 8320145.0, + "step": 1570 + }, + { + "entropy": 0.5665242202579975, + "epoch": 3.3268318397469687, + "grad_norm": 7.0625, + "learning_rate": 8.186666666666667e-06, + "loss": 0.5534502983093261, + "mean_token_accuracy": 0.8129527278244495, + "num_tokens": 8370852.0, + "step": 1580 + }, + { + "entropy": 0.5829939197748899, + "epoch": 3.3479177648919345, + "grad_norm": 7.40625, + "learning_rate": 8.09777777777778e-06, + "loss": 0.5778002738952637, + "mean_token_accuracy": 0.8072332389652729, + "num_tokens": 8424209.0, + "step": 1590 + }, + { + "entropy": 0.5479087311774492, + "epoch": 3.3690036900369003, + "grad_norm": 6.15625, + "learning_rate": 8.00888888888889e-06, + "loss": 0.5359113216400146, + "mean_token_accuracy": 0.819999934732914, + "num_tokens": 8476617.0, + "step": 1600 + }, + { + "entropy": 0.5520685013383627, + "epoch": 3.390089615181866, + "grad_norm": 6.53125, + "learning_rate": 7.92e-06, + "loss": 0.540053415298462, + "mean_token_accuracy": 0.817080944776535, + "num_tokens": 8529074.0, + "step": 1610 + }, + { + "entropy": 0.5533283669501543, + "epoch": 3.411175540326832, + "grad_norm": 7.25, + "learning_rate": 7.831111111111111e-06, + "loss": 0.5461609363555908, + "mean_token_accuracy": 0.8130916491150856, + "num_tokens": 8582416.0, + "step": 1620 + }, + { + "entropy": 0.5606811255216598, + "epoch": 3.4322614654717976, + "grad_norm": 8.25, + "learning_rate": 7.742222222222223e-06, + "loss": 0.565118408203125, + "mean_token_accuracy": 0.813844607770443, + "num_tokens": 8634249.0, + "step": 1630 + }, + { + "entropy": 0.5458527609705925, + "epoch": 3.4533473906167633, + "grad_norm": 7.6875, + "learning_rate": 7.653333333333333e-06, + "loss": 0.5410670757293701, + "mean_token_accuracy": 0.8188620381057262, + "num_tokens": 8685168.0, + "step": 1640 + }, + { + "entropy": 0.5717878874391318, + "epoch": 3.474433315761729, + "grad_norm": 7.6875, + "learning_rate": 7.564444444444446e-06, + "loss": 0.5788788795471191, + "mean_token_accuracy": 0.8078153640031814, + "num_tokens": 8740562.0, + "step": 1650 + }, + { + "entropy": 0.5646973233669996, + "epoch": 3.495519240906695, + "grad_norm": 6.75, + "learning_rate": 7.475555555555556e-06, + "loss": 0.5595187187194824, + "mean_token_accuracy": 0.8135132856667042, + "num_tokens": 8794411.0, + "step": 1660 + }, + { + "entropy": 0.5641634043306112, + "epoch": 3.5166051660516606, + "grad_norm": 6.6875, + "learning_rate": 7.386666666666667e-06, + "loss": 0.5511392116546631, + "mean_token_accuracy": 0.8159678690135479, + "num_tokens": 8844547.0, + "step": 1670 + }, + { + "entropy": 0.5724293068051338, + "epoch": 3.5376910911966264, + "grad_norm": 6.875, + "learning_rate": 7.297777777777778e-06, + "loss": 0.5668179988861084, + "mean_token_accuracy": 0.8106696806848049, + "num_tokens": 8902311.0, + "step": 1680 + }, + { + "entropy": 0.5610478781163692, + "epoch": 3.558777016341592, + "grad_norm": 6.96875, + "learning_rate": 7.20888888888889e-06, + "loss": 0.5528025150299072, + "mean_token_accuracy": 0.8120494581758976, + "num_tokens": 8957569.0, + "step": 1690 + }, + { + "entropy": 0.5370181340724229, + "epoch": 3.5798629414865575, + "grad_norm": 7.28125, + "learning_rate": 7.1200000000000004e-06, + "loss": 0.5477783203125, + "mean_token_accuracy": 0.8162322834134101, + "num_tokens": 9009083.0, + "step": 1700 + }, + { + "entropy": 0.5545388571918011, + "epoch": 3.6009488666315237, + "grad_norm": 6.5, + "learning_rate": 7.031111111111112e-06, + "loss": 0.5430552005767822, + "mean_token_accuracy": 0.8191993653774261, + "num_tokens": 9061980.0, + "step": 1710 + }, + { + "entropy": 0.5662489753216505, + "epoch": 3.622034791776489, + "grad_norm": 7.3125, + "learning_rate": 6.942222222222222e-06, + "loss": 0.5623671531677246, + "mean_token_accuracy": 0.8118648871779441, + "num_tokens": 9117653.0, + "step": 1720 + }, + { + "entropy": 0.5656806524842978, + "epoch": 3.643120716921455, + "grad_norm": 7.40625, + "learning_rate": 6.853333333333334e-06, + "loss": 0.5693060398101807, + "mean_token_accuracy": 0.8121738277375699, + "num_tokens": 9168884.0, + "step": 1730 + }, + { + "entropy": 0.5669469233602286, + "epoch": 3.6642066420664205, + "grad_norm": 6.875, + "learning_rate": 6.764444444444445e-06, + "loss": 0.5708917617797852, + "mean_token_accuracy": 0.8130646869540215, + "num_tokens": 9221794.0, + "step": 1740 + }, + { + "entropy": 0.5680123705416917, + "epoch": 3.6852925672113863, + "grad_norm": 7.3125, + "learning_rate": 6.675555555555556e-06, + "loss": 0.565749979019165, + "mean_token_accuracy": 0.8114100478589534, + "num_tokens": 9277320.0, + "step": 1750 + }, + { + "entropy": 0.5670568168163299, + "epoch": 3.706378492356352, + "grad_norm": 6.8125, + "learning_rate": 6.5866666666666666e-06, + "loss": 0.5531346797943115, + "mean_token_accuracy": 0.815721620619297, + "num_tokens": 9328654.0, + "step": 1760 + }, + { + "entropy": 0.5794487085193396, + "epoch": 3.727464417501318, + "grad_norm": 7.25, + "learning_rate": 6.497777777777779e-06, + "loss": 0.5695443630218506, + "mean_token_accuracy": 0.8125023365020752, + "num_tokens": 9382781.0, + "step": 1770 + }, + { + "entropy": 0.5529596284031868, + "epoch": 3.7485503426462836, + "grad_norm": 7.21875, + "learning_rate": 6.408888888888889e-06, + "loss": 0.5538197040557862, + "mean_token_accuracy": 0.8132442943751812, + "num_tokens": 9434446.0, + "step": 1780 + }, + { + "entropy": 0.554557716101408, + "epoch": 3.7696362677912494, + "grad_norm": 7.34375, + "learning_rate": 6.3200000000000005e-06, + "loss": 0.5493638515472412, + "mean_token_accuracy": 0.8154775679111481, + "num_tokens": 9488504.0, + "step": 1790 + }, + { + "entropy": 0.5696977652609349, + "epoch": 3.790722192936215, + "grad_norm": 6.28125, + "learning_rate": 6.231111111111111e-06, + "loss": 0.5688616275787354, + "mean_token_accuracy": 0.8128006041049958, + "num_tokens": 9545943.0, + "step": 1800 + }, + { + "entropy": 0.5715777482837439, + "epoch": 3.811808118081181, + "grad_norm": 6.78125, + "learning_rate": 6.142222222222223e-06, + "loss": 0.5713305473327637, + "mean_token_accuracy": 0.811994157731533, + "num_tokens": 9597820.0, + "step": 1810 + }, + { + "entropy": 0.5638129018247128, + "epoch": 3.8328940432261467, + "grad_norm": 7.25, + "learning_rate": 6.0533333333333335e-06, + "loss": 0.5548198699951172, + "mean_token_accuracy": 0.8136275447905064, + "num_tokens": 9651480.0, + "step": 1820 + }, + { + "entropy": 0.5706254750490188, + "epoch": 3.853979968371112, + "grad_norm": 6.8125, + "learning_rate": 5.964444444444445e-06, + "loss": 0.5752524852752685, + "mean_token_accuracy": 0.8053847931325435, + "num_tokens": 9706114.0, + "step": 1830 + }, + { + "entropy": 0.5612337995320559, + "epoch": 3.875065893516078, + "grad_norm": 7.0, + "learning_rate": 5.875555555555556e-06, + "loss": 0.5460340499877929, + "mean_token_accuracy": 0.8159381657838821, + "num_tokens": 9760141.0, + "step": 1840 + }, + { + "entropy": 0.5514645285904407, + "epoch": 3.8961518186610435, + "grad_norm": 8.0, + "learning_rate": 5.7866666666666674e-06, + "loss": 0.5530869960784912, + "mean_token_accuracy": 0.8169759206473828, + "num_tokens": 9812466.0, + "step": 1850 + }, + { + "entropy": 0.5574501186609269, + "epoch": 3.9172377438060098, + "grad_norm": 7.5, + "learning_rate": 5.697777777777779e-06, + "loss": 0.5557053089141846, + "mean_token_accuracy": 0.8142473429441452, + "num_tokens": 9863547.0, + "step": 1860 + }, + { + "entropy": 0.560401076450944, + "epoch": 3.938323668950975, + "grad_norm": 6.53125, + "learning_rate": 5.608888888888889e-06, + "loss": 0.5598348140716553, + "mean_token_accuracy": 0.810575645416975, + "num_tokens": 9917210.0, + "step": 1870 + }, + { + "entropy": 0.5611382886767388, + "epoch": 3.959409594095941, + "grad_norm": 6.875, + "learning_rate": 5.5200000000000005e-06, + "loss": 0.5510398387908936, + "mean_token_accuracy": 0.8139252245426178, + "num_tokens": 9971753.0, + "step": 1880 + }, + { + "entropy": 0.571844469755888, + "epoch": 3.9804955192409066, + "grad_norm": 7.15625, + "learning_rate": 5.431111111111112e-06, + "loss": 0.5825337409973145, + "mean_token_accuracy": 0.8051016978919506, + "num_tokens": 10024121.0, + "step": 1890 + }, + { + "entropy": 0.5610769511880102, + "epoch": 4.0, + "grad_norm": 15.75, + "learning_rate": 5.342222222222223e-06, + "loss": 0.5544355392456055, + "mean_token_accuracy": 0.8145432029221509, + "num_tokens": 10072608.0, + "step": 1900 + } + ], + "logging_steps": 10, + "max_steps": 2500, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0305589639937434e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1900/training_args.bin b/checkpoint-1900/training_args.bin new file mode 100644 index 0000000..f138a50 --- /dev/null +++ b/checkpoint-1900/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7103b0ce3340ea41aca3b7441d01d7c38791e3d3ce8c9b28c906c9b1d49bbee9 +size 5713 diff --git a/checkpoint-2000/chat_template.jinja b/checkpoint-2000/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-2000/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-2000/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-2000/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-2000/model.safetensors b/checkpoint-2000/model.safetensors new file mode 100644 index 0000000..6d56052 --- /dev/null +++ b/checkpoint-2000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:370cb506beed0dd1a310bc725c687278b4397ef57db54676c874a95a4968604a +size 3441185608 diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt new file mode 100644 index 0000000..cad41e3 --- /dev/null +++ b/checkpoint-2000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a261be50b1cbb39394923dc53bd358b9a2ef400169cfc93091b94134a78f0793 +size 6882572207 diff --git a/checkpoint-2000/rng_state.pth b/checkpoint-2000/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-2000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000..4ab0505 --- /dev/null +++ b/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:197cfcfbf9a36f4f36efb39330b481554a65e22895ce4f427a55d7698d737749 +size 1465 diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000..f92b774 --- /dev/null +++ b/checkpoint-2000/trainer_state.json @@ -0,0 +1,2034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4849513533173703, + "eval_steps": 500, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.09651942026102915, + "epoch": 0.002424756766586852, + "grad_norm": 552.0, + "learning_rate": 9.000000000000001e-07, + "loss": 6.736698150634766, + "mean_token_accuracy": 0.5771780241280794, + "num_tokens": 36846.0, + "step": 10 + }, + { + "entropy": 0.10363724352791906, + "epoch": 0.004849513533173704, + "grad_norm": 296.0, + "learning_rate": 1.9000000000000002e-06, + "loss": 6.29327507019043, + "mean_token_accuracy": 0.6011363569647074, + "num_tokens": 81163.0, + "step": 20 + }, + { + "entropy": 0.09438130774069578, + "epoch": 0.007274270299760555, + "grad_norm": 222.0, + "learning_rate": 2.9e-06, + "loss": 5.125759887695312, + "mean_token_accuracy": 0.6545454517006875, + "num_tokens": 123610.0, + "step": 30 + }, + { + "entropy": 0.10434843641705811, + "epoch": 0.009699027066347407, + "grad_norm": 205.0, + "learning_rate": 3.900000000000001e-06, + "loss": 3.081037902832031, + "mean_token_accuracy": 0.6937500037252903, + "num_tokens": 163947.0, + "step": 40 + }, + { + "entropy": 0.06968767816433682, + "epoch": 0.012123783832934259, + "grad_norm": 39.75, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.6972839832305908, + "mean_token_accuracy": 0.8579545550048351, + "num_tokens": 203175.0, + "step": 50 + }, + { + "entropy": 0.09051208726596087, + "epoch": 0.01454854059952111, + "grad_norm": 14.75, + "learning_rate": 5.9e-06, + "loss": 0.1402425527572632, + "mean_token_accuracy": 0.9636363588273525, + "num_tokens": 242004.0, + "step": 60 + }, + { + "entropy": 0.04463538663112558, + "epoch": 0.01697329736610796, + "grad_norm": 3.453125, + "learning_rate": 6.9e-06, + "loss": 0.059400951862335204, + "mean_token_accuracy": 0.9761093020439148, + "num_tokens": 282287.0, + "step": 70 + }, + { + "entropy": 0.03611970616912004, + "epoch": 0.019398054132694815, + "grad_norm": 1.78125, + "learning_rate": 7.9e-06, + "loss": 0.03747022747993469, + "mean_token_accuracy": 0.9812499955296516, + "num_tokens": 320685.0, + "step": 80 + }, + { + "entropy": 0.03572612675052369, + "epoch": 0.021822810899281665, + "grad_norm": 3.765625, + "learning_rate": 8.900000000000001e-06, + "loss": 0.03938577175140381, + "mean_token_accuracy": 0.9784090861678123, + "num_tokens": 359560.0, + "step": 90 + }, + { + "entropy": 0.035298487018735614, + "epoch": 0.024247567665868518, + "grad_norm": 11.9375, + "learning_rate": 9.9e-06, + "loss": 0.03732649385929108, + "mean_token_accuracy": 0.9772456660866737, + "num_tokens": 403870.0, + "step": 100 + }, + { + "entropy": 0.03189297981371055, + "epoch": 0.02667232443245537, + "grad_norm": 2.484375, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.03071807622909546, + "mean_token_accuracy": 0.9823863603174686, + "num_tokens": 443514.0, + "step": 110 + }, + { + "entropy": 0.027393289806786923, + "epoch": 0.02909708119904222, + "grad_norm": 7.75, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.03194451332092285, + "mean_token_accuracy": 0.9812499962747097, + "num_tokens": 483941.0, + "step": 120 + }, + { + "entropy": 0.02901054713729536, + "epoch": 0.031521837965629074, + "grad_norm": 2.078125, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.04001099765300751, + "mean_token_accuracy": 0.9801136322319508, + "num_tokens": 524184.0, + "step": 130 + }, + { + "entropy": 0.036186478319723395, + "epoch": 0.03394659473221592, + "grad_norm": 2.453125, + "learning_rate": 1.39e-05, + "loss": 0.032131227850914004, + "mean_token_accuracy": 0.977840906381607, + "num_tokens": 565235.0, + "step": 140 + }, + { + "entropy": 0.02588132777964347, + "epoch": 0.03637135149880277, + "grad_norm": 2.59375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.032545623183250424, + "mean_token_accuracy": 0.9795183934271335, + "num_tokens": 604912.0, + "step": 150 + }, + { + "entropy": 0.024791866151281282, + "epoch": 0.03879610826538963, + "grad_norm": 4.71875, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.029649946093559264, + "mean_token_accuracy": 0.9835227228701114, + "num_tokens": 643428.0, + "step": 160 + }, + { + "entropy": 0.035497133302123986, + "epoch": 0.04122086503197648, + "grad_norm": 2.421875, + "learning_rate": 1.69e-05, + "loss": 0.03451511561870575, + "mean_token_accuracy": 0.977840905636549, + "num_tokens": 682578.0, + "step": 170 + }, + { + "entropy": 0.032462505592047816, + "epoch": 0.04364562179856333, + "grad_norm": 1.3671875, + "learning_rate": 1.79e-05, + "loss": 0.03423822820186615, + "mean_token_accuracy": 0.9778138473629951, + "num_tokens": 723563.0, + "step": 180 + }, + { + "entropy": 0.03153426277203834, + "epoch": 0.046070378565150186, + "grad_norm": 1.3984375, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.030463039875030518, + "mean_token_accuracy": 0.9818181775510311, + "num_tokens": 765374.0, + "step": 190 + }, + { + "entropy": 0.020951804842843556, + "epoch": 0.048495135331737035, + "grad_norm": 4.6875, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.04620547294616699, + "mean_token_accuracy": 0.9801136329770088, + "num_tokens": 806149.0, + "step": 200 + }, + { + "entropy": 0.03319120884152653, + "epoch": 0.050919892098323885, + "grad_norm": 3.40625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.03888022005558014, + "mean_token_accuracy": 0.9784090884029866, + "num_tokens": 852548.0, + "step": 210 + }, + { + "entropy": 0.034044423646992075, + "epoch": 0.05334464886491074, + "grad_norm": 1.875, + "learning_rate": 1.978888888888889e-05, + "loss": 0.03665586411952972, + "mean_token_accuracy": 0.9772727221250535, + "num_tokens": 891570.0, + "step": 220 + }, + { + "entropy": 0.020276623886820742, + "epoch": 0.05576940563149759, + "grad_norm": 1.4921875, + "learning_rate": 1.967777777777778e-05, + "loss": 0.034099149703979495, + "mean_token_accuracy": 0.9846590891480446, + "num_tokens": 931221.0, + "step": 230 + }, + { + "entropy": 0.03405958858202211, + "epoch": 0.05819416239808444, + "grad_norm": 2.671875, + "learning_rate": 1.956666666666667e-05, + "loss": 0.03181847333908081, + "mean_token_accuracy": 0.9812499970197678, + "num_tokens": 973029.0, + "step": 240 + }, + { + "entropy": 0.0318044990162889, + "epoch": 0.06061891916467129, + "grad_norm": 1.71875, + "learning_rate": 1.9455555555555555e-05, + "loss": 0.031788495182991025, + "mean_token_accuracy": 0.9806818142533302, + "num_tokens": 1012763.0, + "step": 250 + }, + { + "entropy": 0.031602241041036906, + "epoch": 0.06304367593125815, + "grad_norm": 3.546875, + "learning_rate": 1.9344444444444446e-05, + "loss": 0.03268867433071136, + "mean_token_accuracy": 0.98295454159379, + "num_tokens": 1050122.0, + "step": 260 + }, + { + "entropy": 0.024405305968684843, + "epoch": 0.065468432697845, + "grad_norm": 2.234375, + "learning_rate": 1.9233333333333334e-05, + "loss": 0.03167384266853333, + "mean_token_accuracy": 0.9846590869128704, + "num_tokens": 1092243.0, + "step": 270 + }, + { + "entropy": 0.026620667469615, + "epoch": 0.06789318946443185, + "grad_norm": 1.21875, + "learning_rate": 1.9122222222222226e-05, + "loss": 0.02862342298030853, + "mean_token_accuracy": 0.9817911215126515, + "num_tokens": 1130648.0, + "step": 280 + }, + { + "entropy": 0.03168776935272035, + "epoch": 0.0703179462310187, + "grad_norm": 2.921875, + "learning_rate": 1.9011111111111114e-05, + "loss": 0.02828059494495392, + "mean_token_accuracy": 0.9823863588273525, + "num_tokens": 1173427.0, + "step": 290 + }, + { + "entropy": 0.01820039304529928, + "epoch": 0.07274270299760555, + "grad_norm": 3.46875, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.033576938509941104, + "mean_token_accuracy": 0.985227269679308, + "num_tokens": 1210875.0, + "step": 300 + }, + { + "entropy": 0.03017507032072899, + "epoch": 0.07516745976419241, + "grad_norm": 2.40625, + "learning_rate": 1.878888888888889e-05, + "loss": 0.03038310110569, + "mean_token_accuracy": 0.9852272674441338, + "num_tokens": 1250450.0, + "step": 310 + }, + { + "entropy": 0.031056866744256693, + "epoch": 0.07759221653077926, + "grad_norm": 10.5625, + "learning_rate": 1.8677777777777778e-05, + "loss": 0.02981090545654297, + "mean_token_accuracy": 0.9846590854227543, + "num_tokens": 1292161.0, + "step": 320 + }, + { + "entropy": 0.02875728546641767, + "epoch": 0.08001697329736611, + "grad_norm": 2.078125, + "learning_rate": 1.856666666666667e-05, + "loss": 0.028395554423332213, + "mean_token_accuracy": 0.9857954517006874, + "num_tokens": 1335278.0, + "step": 330 + }, + { + "entropy": 0.029322995853544854, + "epoch": 0.08244173006395296, + "grad_norm": 3.84375, + "learning_rate": 1.8455555555555558e-05, + "loss": 0.029434293508529663, + "mean_token_accuracy": 0.9835227228701114, + "num_tokens": 1373606.0, + "step": 340 + }, + { + "entropy": 0.027826807415840447, + "epoch": 0.08486648683053981, + "grad_norm": 2.21875, + "learning_rate": 1.8344444444444446e-05, + "loss": 0.03300779163837433, + "mean_token_accuracy": 0.978143934905529, + "num_tokens": 1417353.0, + "step": 350 + }, + { + "entropy": 0.0319683638830611, + "epoch": 0.08729124359712666, + "grad_norm": 2.203125, + "learning_rate": 1.8233333333333334e-05, + "loss": 0.03151697814464569, + "mean_token_accuracy": 0.9823863580822945, + "num_tokens": 1453720.0, + "step": 360 + }, + { + "entropy": 0.025839641344646225, + "epoch": 0.08971600036371352, + "grad_norm": 2.6875, + "learning_rate": 1.8122222222222222e-05, + "loss": 0.02670169174671173, + "mean_token_accuracy": 0.9874999955296516, + "num_tokens": 1496830.0, + "step": 370 + }, + { + "entropy": 0.02322939161786053, + "epoch": 0.09214075713030037, + "grad_norm": 1.921875, + "learning_rate": 1.8011111111111113e-05, + "loss": 0.029871052503585814, + "mean_token_accuracy": 0.9823863603174686, + "num_tokens": 1539244.0, + "step": 380 + }, + { + "entropy": 0.028905938124444218, + "epoch": 0.09456551389688722, + "grad_norm": 4.9375, + "learning_rate": 1.79e-05, + "loss": 0.031239384412765504, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 1581506.0, + "step": 390 + }, + { + "entropy": 0.020863897850722422, + "epoch": 0.09699027066347407, + "grad_norm": 3.3125, + "learning_rate": 1.778888888888889e-05, + "loss": 0.0223777636885643, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 1623446.0, + "step": 400 + }, + { + "entropy": 0.020877818292319716, + "epoch": 0.09941502743006092, + "grad_norm": 3.90625, + "learning_rate": 1.7677777777777778e-05, + "loss": 0.025508558750152587, + "mean_token_accuracy": 0.9880681782960892, + "num_tokens": 1662111.0, + "step": 410 + }, + { + "entropy": 0.026274691119397175, + "epoch": 0.10183978419664777, + "grad_norm": 1.5, + "learning_rate": 1.756666666666667e-05, + "loss": 0.03404468297958374, + "mean_token_accuracy": 0.9801136307418347, + "num_tokens": 1702735.0, + "step": 420 + }, + { + "entropy": 0.0291934731634683, + "epoch": 0.10426454096323462, + "grad_norm": 0.90625, + "learning_rate": 1.7455555555555557e-05, + "loss": 0.02973931431770325, + "mean_token_accuracy": 0.9846590869128704, + "num_tokens": 1740129.0, + "step": 430 + }, + { + "entropy": 0.02555293397508649, + "epoch": 0.10668929772982148, + "grad_norm": 0.84375, + "learning_rate": 1.7344444444444445e-05, + "loss": 0.02494020164012909, + "mean_token_accuracy": 0.9869318164885044, + "num_tokens": 1784479.0, + "step": 440 + }, + { + "entropy": 0.028070836007191246, + "epoch": 0.10911405449640833, + "grad_norm": 2.125, + "learning_rate": 1.7233333333333337e-05, + "loss": 0.029692673683166505, + "mean_token_accuracy": 0.9835227228701114, + "num_tokens": 1826847.0, + "step": 450 + }, + { + "entropy": 0.02322534844988695, + "epoch": 0.11153881126299518, + "grad_norm": 2.5, + "learning_rate": 1.7122222222222225e-05, + "loss": 0.02545521855354309, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 1862712.0, + "step": 460 + }, + { + "entropy": 0.022724387578819005, + "epoch": 0.11396356802958203, + "grad_norm": 1.640625, + "learning_rate": 1.7011111111111113e-05, + "loss": 0.0265019953250885, + "mean_token_accuracy": 0.9846590869128704, + "num_tokens": 1903077.0, + "step": 470 + }, + { + "entropy": 0.02751030181589158, + "epoch": 0.11638832479616888, + "grad_norm": 2.03125, + "learning_rate": 1.69e-05, + "loss": 0.027713432908058167, + "mean_token_accuracy": 0.9829545430839062, + "num_tokens": 1948431.0, + "step": 480 + }, + { + "entropy": 0.02874135026240765, + "epoch": 0.11881308156275573, + "grad_norm": 0.88671875, + "learning_rate": 1.678888888888889e-05, + "loss": 0.03457841873168945, + "mean_token_accuracy": 0.9806818127632141, + "num_tokens": 1990633.0, + "step": 490 + }, + { + "entropy": 0.02799490848346977, + "epoch": 0.12123783832934258, + "grad_norm": 1.3359375, + "learning_rate": 1.667777777777778e-05, + "loss": 0.030140578746795654, + "mean_token_accuracy": 0.9835227251052856, + "num_tokens": 2028217.0, + "step": 500 + }, + { + "entropy": 0.025997174895564967, + "epoch": 0.12366259509592945, + "grad_norm": 1.71875, + "learning_rate": 1.656666666666667e-05, + "loss": 0.025564667582511903, + "mean_token_accuracy": 0.9857954509556294, + "num_tokens": 2072644.0, + "step": 510 + }, + { + "entropy": 0.023441067839212338, + "epoch": 0.1260873518625163, + "grad_norm": 2.34375, + "learning_rate": 1.6455555555555556e-05, + "loss": 0.024340197443962097, + "mean_token_accuracy": 0.9863636337220669, + "num_tokens": 2110975.0, + "step": 520 + }, + { + "entropy": 0.021592267712185274, + "epoch": 0.12851210862910314, + "grad_norm": 3.21875, + "learning_rate": 1.6344444444444445e-05, + "loss": 0.02947987914085388, + "mean_token_accuracy": 0.9886363595724106, + "num_tokens": 2150997.0, + "step": 530 + }, + { + "entropy": 0.02172533786206259, + "epoch": 0.13093686539569, + "grad_norm": 1.890625, + "learning_rate": 1.6233333333333333e-05, + "loss": 0.027436545491218566, + "mean_token_accuracy": 0.9846590884029865, + "num_tokens": 2190804.0, + "step": 540 + }, + { + "entropy": 0.02379558993197861, + "epoch": 0.13336162216227684, + "grad_norm": 2.21875, + "learning_rate": 1.6122222222222224e-05, + "loss": 0.025879621505737305, + "mean_token_accuracy": 0.9863636337220669, + "num_tokens": 2240073.0, + "step": 550 + }, + { + "entropy": 0.02549256883594353, + "epoch": 0.1357863789288637, + "grad_norm": 0.92578125, + "learning_rate": 1.6011111111111112e-05, + "loss": 0.024750103056430817, + "mean_token_accuracy": 0.9880681797862053, + "num_tokens": 2280612.0, + "step": 560 + }, + { + "entropy": 0.03142027879757734, + "epoch": 0.13821113569545054, + "grad_norm": 3.34375, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.03994172811508179, + "mean_token_accuracy": 0.9778138488531113, + "num_tokens": 2320775.0, + "step": 570 + }, + { + "entropy": 0.028627974635946886, + "epoch": 0.1406358924620374, + "grad_norm": 0.88671875, + "learning_rate": 1.5788888888888888e-05, + "loss": 0.03192804753780365, + "mean_token_accuracy": 0.9823863580822945, + "num_tokens": 2359848.0, + "step": 580 + }, + { + "entropy": 0.027760646163096682, + "epoch": 0.14306064922862424, + "grad_norm": 1.4140625, + "learning_rate": 1.567777777777778e-05, + "loss": 0.023938073217868804, + "mean_token_accuracy": 0.9863636314868927, + "num_tokens": 2398716.0, + "step": 590 + }, + { + "entropy": 0.01857751572433699, + "epoch": 0.1454854059952111, + "grad_norm": 2.0, + "learning_rate": 1.5566666666666668e-05, + "loss": 0.021452714502811433, + "mean_token_accuracy": 0.9909090891480445, + "num_tokens": 2439221.0, + "step": 600 + }, + { + "entropy": 0.020551329124737096, + "epoch": 0.14791016276179797, + "grad_norm": 2.09375, + "learning_rate": 1.5455555555555556e-05, + "loss": 0.03473537266254425, + "mean_token_accuracy": 0.9840909041464329, + "num_tokens": 2477390.0, + "step": 610 + }, + { + "entropy": 0.025016580805640842, + "epoch": 0.15033491952838482, + "grad_norm": 1.828125, + "learning_rate": 1.5344444444444447e-05, + "loss": 0.022924990952014924, + "mean_token_accuracy": 0.9863636314868927, + "num_tokens": 2515438.0, + "step": 620 + }, + { + "entropy": 0.02900541319813783, + "epoch": 0.15275967629497167, + "grad_norm": 1.6171875, + "learning_rate": 1.5233333333333335e-05, + "loss": 0.030890285968780518, + "mean_token_accuracy": 0.9829545401036739, + "num_tokens": 2554303.0, + "step": 630 + }, + { + "entropy": 0.027651957263333315, + "epoch": 0.15518443306155852, + "grad_norm": 1.3515625, + "learning_rate": 1.5122222222222222e-05, + "loss": 0.026059484481811522, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 2593917.0, + "step": 640 + }, + { + "entropy": 0.025979451155399146, + "epoch": 0.15760918982814537, + "grad_norm": 0.51171875, + "learning_rate": 1.5011111111111112e-05, + "loss": 0.026576137542724608, + "mean_token_accuracy": 0.9846590891480446, + "num_tokens": 2633471.0, + "step": 650 + }, + { + "entropy": 0.0226533382314301, + "epoch": 0.16003394659473222, + "grad_norm": 3.703125, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.03458902537822724, + "mean_token_accuracy": 0.9806818149983882, + "num_tokens": 2673584.0, + "step": 660 + }, + { + "entropy": 0.026275359961891807, + "epoch": 0.16245870336131907, + "grad_norm": 1.7421875, + "learning_rate": 1.4788888888888891e-05, + "loss": 0.023309774696826935, + "mean_token_accuracy": 0.9886363610625267, + "num_tokens": 2718495.0, + "step": 670 + }, + { + "entropy": 0.02142264891299419, + "epoch": 0.16488346012790592, + "grad_norm": 1.40625, + "learning_rate": 1.467777777777778e-05, + "loss": 0.027975478768348695, + "mean_token_accuracy": 0.9863636322319508, + "num_tokens": 2758475.0, + "step": 680 + }, + { + "entropy": 0.023169603949736484, + "epoch": 0.16730821689449277, + "grad_norm": 3.46875, + "learning_rate": 1.4566666666666669e-05, + "loss": 0.027786874771118165, + "mean_token_accuracy": 0.9846320301294327, + "num_tokens": 2801461.0, + "step": 690 + }, + { + "entropy": 0.025932194874985725, + "epoch": 0.16973297366107962, + "grad_norm": 1.40625, + "learning_rate": 1.4455555555555555e-05, + "loss": 0.024787703156471254, + "mean_token_accuracy": 0.9863636329770088, + "num_tokens": 2843561.0, + "step": 700 + }, + { + "entropy": 0.02573719730553421, + "epoch": 0.17215773042766647, + "grad_norm": 1.1953125, + "learning_rate": 1.4344444444444445e-05, + "loss": 0.023608550429344177, + "mean_token_accuracy": 0.9874999955296516, + "num_tokens": 2883906.0, + "step": 710 + }, + { + "entropy": 0.023726833695400275, + "epoch": 0.17458248719425332, + "grad_norm": 1.828125, + "learning_rate": 1.4233333333333335e-05, + "loss": 0.02043382525444031, + "mean_token_accuracy": 0.98920454159379, + "num_tokens": 2925983.0, + "step": 720 + }, + { + "entropy": 0.02000678656895616, + "epoch": 0.17700724396084017, + "grad_norm": 4.125, + "learning_rate": 1.4122222222222225e-05, + "loss": 0.03365764915943146, + "mean_token_accuracy": 0.9886363618075847, + "num_tokens": 2964454.0, + "step": 730 + }, + { + "entropy": 0.02043497935082996, + "epoch": 0.17943200072742704, + "grad_norm": 2.859375, + "learning_rate": 1.4011111111111113e-05, + "loss": 0.02920505404472351, + "mean_token_accuracy": 0.9857954502105712, + "num_tokens": 3005068.0, + "step": 740 + }, + { + "entropy": 0.029566307516233792, + "epoch": 0.1818567574940139, + "grad_norm": 1.3828125, + "learning_rate": 1.39e-05, + "loss": 0.029777100682258605, + "mean_token_accuracy": 0.9852272711694241, + "num_tokens": 3046898.0, + "step": 750 + }, + { + "entropy": 0.024274932581920438, + "epoch": 0.18428151426060074, + "grad_norm": 1.265625, + "learning_rate": 1.3788888888888889e-05, + "loss": 0.0237515389919281, + "mean_token_accuracy": 0.9869318157434464, + "num_tokens": 3088514.0, + "step": 760 + }, + { + "entropy": 0.02470423321283306, + "epoch": 0.1867062710271876, + "grad_norm": 1.125, + "learning_rate": 1.3677777777777779e-05, + "loss": 0.030222481489181517, + "mean_token_accuracy": 0.9846590884029865, + "num_tokens": 3129575.0, + "step": 770 + }, + { + "entropy": 0.028905244976613176, + "epoch": 0.18913102779377444, + "grad_norm": 1.7421875, + "learning_rate": 1.3566666666666668e-05, + "loss": 0.0264810711145401, + "mean_token_accuracy": 0.985227269679308, + "num_tokens": 3170056.0, + "step": 780 + }, + { + "entropy": 0.024943281395098892, + "epoch": 0.1915557845603613, + "grad_norm": 1.7578125, + "learning_rate": 1.3455555555555556e-05, + "loss": 0.029622524976730347, + "mean_token_accuracy": 0.9823863595724106, + "num_tokens": 3210101.0, + "step": 790 + }, + { + "entropy": 0.03016873870701602, + "epoch": 0.19398054132694814, + "grad_norm": 1.3046875, + "learning_rate": 1.3344444444444446e-05, + "loss": 0.035280358791351316, + "mean_token_accuracy": 0.9806818120181561, + "num_tokens": 3252969.0, + "step": 800 + }, + { + "entropy": 0.0313864836067296, + "epoch": 0.196405298093535, + "grad_norm": 2.046875, + "learning_rate": 1.3233333333333334e-05, + "loss": 0.02824702560901642, + "mean_token_accuracy": 0.9823863610625267, + "num_tokens": 3293318.0, + "step": 810 + }, + { + "entropy": 0.031964658715151015, + "epoch": 0.19883005486012184, + "grad_norm": 0.83984375, + "learning_rate": 1.3122222222222222e-05, + "loss": 0.02947427034378052, + "mean_token_accuracy": 0.9863636322319508, + "num_tokens": 3338604.0, + "step": 820 + }, + { + "entropy": 0.024338746762441587, + "epoch": 0.2012548116267087, + "grad_norm": 1.9609375, + "learning_rate": 1.3011111111111112e-05, + "loss": 0.023631574213504793, + "mean_token_accuracy": 0.984090905636549, + "num_tokens": 3376591.0, + "step": 830 + }, + { + "entropy": 0.022459183935188774, + "epoch": 0.20367956839329554, + "grad_norm": 2.390625, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.029895466566085816, + "mean_token_accuracy": 0.9846590891480446, + "num_tokens": 3417635.0, + "step": 840 + }, + { + "entropy": 0.022913600040919847, + "epoch": 0.2061043251598824, + "grad_norm": 2.296875, + "learning_rate": 1.278888888888889e-05, + "loss": 0.02521446943283081, + "mean_token_accuracy": 0.9857954502105712, + "num_tokens": 3459294.0, + "step": 850 + }, + { + "entropy": 0.025265456292163434, + "epoch": 0.20852908192646924, + "grad_norm": 1.4765625, + "learning_rate": 1.2677777777777778e-05, + "loss": 0.022895295917987824, + "mean_token_accuracy": 0.9886363618075847, + "num_tokens": 3500919.0, + "step": 860 + }, + { + "entropy": 0.026354352129692414, + "epoch": 0.2109538386930561, + "grad_norm": 1.2265625, + "learning_rate": 1.2566666666666668e-05, + "loss": 0.02891395688056946, + "mean_token_accuracy": 0.9857954509556294, + "num_tokens": 3542022.0, + "step": 870 + }, + { + "entropy": 0.026587546153950824, + "epoch": 0.21337859545964297, + "grad_norm": 2.15625, + "learning_rate": 1.2455555555555556e-05, + "loss": 0.025007185339927674, + "mean_token_accuracy": 0.9880681782960892, + "num_tokens": 3580600.0, + "step": 880 + }, + { + "entropy": 0.021694945237868524, + "epoch": 0.21580335222622982, + "grad_norm": 1.578125, + "learning_rate": 1.2344444444444446e-05, + "loss": 0.025805145502090454, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 3620910.0, + "step": 890 + }, + { + "entropy": 0.02144139350803016, + "epoch": 0.21822810899281667, + "grad_norm": 1.90625, + "learning_rate": 1.2233333333333335e-05, + "loss": 0.020484310388565064, + "mean_token_accuracy": 0.9892045423388481, + "num_tokens": 3660844.0, + "step": 900 + }, + { + "entropy": 0.02579846643930068, + "epoch": 0.22065286575940352, + "grad_norm": 1.6640625, + "learning_rate": 1.2122222222222223e-05, + "loss": 0.03399317860603333, + "mean_token_accuracy": 0.9812499992549419, + "num_tokens": 3702701.0, + "step": 910 + }, + { + "entropy": 0.025248465246386333, + "epoch": 0.22307762252599037, + "grad_norm": 1.4140625, + "learning_rate": 1.2011111111111111e-05, + "loss": 0.025516167283058167, + "mean_token_accuracy": 0.9846590876579284, + "num_tokens": 3744019.0, + "step": 920 + }, + { + "entropy": 0.02416455528455117, + "epoch": 0.22550237929257722, + "grad_norm": 1.234375, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0251032292842865, + "mean_token_accuracy": 0.9857683956623078, + "num_tokens": 3783437.0, + "step": 930 + }, + { + "entropy": 0.024388600795100503, + "epoch": 0.22792713605916406, + "grad_norm": 5.96875, + "learning_rate": 1.178888888888889e-05, + "loss": 0.026069679856300355, + "mean_token_accuracy": 0.984090905636549, + "num_tokens": 3823298.0, + "step": 940 + }, + { + "entropy": 0.02505718378297388, + "epoch": 0.23035189282575091, + "grad_norm": 1.03125, + "learning_rate": 1.1677777777777779e-05, + "loss": 0.025333744287490845, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 3863057.0, + "step": 950 + }, + { + "entropy": 0.02888807645626912, + "epoch": 0.23277664959233776, + "grad_norm": 2.640625, + "learning_rate": 1.1566666666666669e-05, + "loss": 0.02982906103134155, + "mean_token_accuracy": 0.9852272674441338, + "num_tokens": 3902174.0, + "step": 960 + }, + { + "entropy": 0.026199724662365042, + "epoch": 0.23520140635892461, + "grad_norm": 1.53125, + "learning_rate": 1.1455555555555557e-05, + "loss": 0.03483479619026184, + "mean_token_accuracy": 0.9795454509556294, + "num_tokens": 3940800.0, + "step": 970 + }, + { + "entropy": 0.02737266756594181, + "epoch": 0.23762616312551146, + "grad_norm": 2.1875, + "learning_rate": 1.1344444444444445e-05, + "loss": 0.024009138345718384, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 3980912.0, + "step": 980 + }, + { + "entropy": 0.02674058018892538, + "epoch": 0.2400509198920983, + "grad_norm": 0.83203125, + "learning_rate": 1.1233333333333333e-05, + "loss": 0.02231251299381256, + "mean_token_accuracy": 0.9880681790411472, + "num_tokens": 4020042.0, + "step": 990 + }, + { + "entropy": 0.023251514391176897, + "epoch": 0.24247567665868516, + "grad_norm": 1.8515625, + "learning_rate": 1.1122222222222223e-05, + "loss": 0.02937265336513519, + "mean_token_accuracy": 0.9801136322319508, + "num_tokens": 4056666.0, + "step": 1000 + }, + { + "entropy": 0.023756788905575376, + "epoch": 0.24490043342527204, + "grad_norm": 2.875, + "learning_rate": 1.1011111111111113e-05, + "loss": 0.021810129284858704, + "mean_token_accuracy": 0.9880681782960892, + "num_tokens": 4100066.0, + "step": 1010 + }, + { + "entropy": 0.023163225039979807, + "epoch": 0.2473251901918589, + "grad_norm": 1.6875, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.02502768337726593, + "mean_token_accuracy": 0.9874999955296516, + "num_tokens": 4139561.0, + "step": 1020 + }, + { + "entropy": 0.02136901097264854, + "epoch": 0.24974994695844574, + "grad_norm": 1.609375, + "learning_rate": 1.0788888888888889e-05, + "loss": 0.019149383902549742, + "mean_token_accuracy": 0.9897456675767898, + "num_tokens": 4177327.0, + "step": 1030 + }, + { + "entropy": 0.02029619142294905, + "epoch": 0.2521747037250326, + "grad_norm": 2.328125, + "learning_rate": 1.0677777777777779e-05, + "loss": 0.031534403562545776, + "mean_token_accuracy": 0.984090905636549, + "num_tokens": 4218398.0, + "step": 1040 + }, + { + "entropy": 0.02094364000367932, + "epoch": 0.2545994604916194, + "grad_norm": 2.546875, + "learning_rate": 1.0566666666666667e-05, + "loss": 0.03177621364593506, + "mean_token_accuracy": 0.9869318149983883, + "num_tokens": 4261625.0, + "step": 1050 + }, + { + "entropy": 0.025660122126373607, + "epoch": 0.2570242172582063, + "grad_norm": 1.03125, + "learning_rate": 1.0455555555555556e-05, + "loss": 0.029423072934150696, + "mean_token_accuracy": 0.9846590876579284, + "num_tokens": 4300872.0, + "step": 1060 + }, + { + "entropy": 0.027266277611670376, + "epoch": 0.2594489740247931, + "grad_norm": 1.3515625, + "learning_rate": 1.0344444444444446e-05, + "loss": 0.028110802173614502, + "mean_token_accuracy": 0.9840909071266651, + "num_tokens": 4342744.0, + "step": 1070 + }, + { + "entropy": 0.02724119477079512, + "epoch": 0.26187373079138, + "grad_norm": 1.0390625, + "learning_rate": 1.0233333333333336e-05, + "loss": 0.023805345594882964, + "mean_token_accuracy": 0.9863636329770088, + "num_tokens": 4382536.0, + "step": 1080 + }, + { + "entropy": 0.02513553084399973, + "epoch": 0.26429848755796687, + "grad_norm": 1.265625, + "learning_rate": 1.0122222222222222e-05, + "loss": 0.026082912087440492, + "mean_token_accuracy": 0.9874999962747097, + "num_tokens": 4427451.0, + "step": 1090 + }, + { + "entropy": 0.024046725058724405, + "epoch": 0.2667232443245537, + "grad_norm": 1.84375, + "learning_rate": 1.0011111111111112e-05, + "loss": 0.025588032603263856, + "mean_token_accuracy": 0.9869318135082722, + "num_tokens": 4464955.0, + "step": 1100 + }, + { + "entropy": 0.022660849900512404, + "epoch": 0.26914800109114057, + "grad_norm": 1.421875, + "learning_rate": 9.9e-06, + "loss": 0.021866945922374724, + "mean_token_accuracy": 0.9880681782960892, + "num_tokens": 4505114.0, + "step": 1110 + }, + { + "entropy": 0.02450128130503799, + "epoch": 0.2715727578577274, + "grad_norm": 1.28125, + "learning_rate": 9.78888888888889e-06, + "loss": 0.024013830721378325, + "mean_token_accuracy": 0.9869318149983883, + "num_tokens": 4545307.0, + "step": 1120 + }, + { + "entropy": 0.023239308406709826, + "epoch": 0.27399751462431426, + "grad_norm": 1.9296875, + "learning_rate": 9.677777777777778e-06, + "loss": 0.026031026244163515, + "mean_token_accuracy": 0.9869318135082722, + "num_tokens": 4583174.0, + "step": 1130 + }, + { + "entropy": 0.02481268622832431, + "epoch": 0.2764222713909011, + "grad_norm": 0.859375, + "learning_rate": 9.566666666666668e-06, + "loss": 0.028152582049369813, + "mean_token_accuracy": 0.98295454159379, + "num_tokens": 4623363.0, + "step": 1140 + }, + { + "entropy": 0.02144863496614562, + "epoch": 0.27884702815748796, + "grad_norm": 1.8515625, + "learning_rate": 9.455555555555557e-06, + "loss": 0.018676747381687165, + "mean_token_accuracy": 0.9897727243602276, + "num_tokens": 4663530.0, + "step": 1150 + }, + { + "entropy": 0.021853329628083885, + "epoch": 0.2812717849240748, + "grad_norm": 1.03125, + "learning_rate": 9.344444444444446e-06, + "loss": 0.023481160402297974, + "mean_token_accuracy": 0.9880681790411472, + "num_tokens": 4703751.0, + "step": 1160 + }, + { + "entropy": 0.020845431580073636, + "epoch": 0.28369654169066166, + "grad_norm": 1.828125, + "learning_rate": 9.233333333333334e-06, + "loss": 0.025833714008331298, + "mean_token_accuracy": 0.9892045423388481, + "num_tokens": 4742890.0, + "step": 1170 + }, + { + "entropy": 0.020562034113299887, + "epoch": 0.2861212984572485, + "grad_norm": 3.359375, + "learning_rate": 9.122222222222223e-06, + "loss": 0.02122436910867691, + "mean_token_accuracy": 0.9869318149983883, + "num_tokens": 4786008.0, + "step": 1180 + }, + { + "entropy": 0.0207685565927477, + "epoch": 0.28854605522383536, + "grad_norm": 0.92578125, + "learning_rate": 9.011111111111111e-06, + "loss": 0.016763387620449065, + "mean_token_accuracy": 0.9914772696793079, + "num_tokens": 4829612.0, + "step": 1190 + }, + { + "entropy": 0.022811952112988365, + "epoch": 0.2909708119904222, + "grad_norm": 2.5625, + "learning_rate": 8.900000000000001e-06, + "loss": 0.025601431727409363, + "mean_token_accuracy": 0.985227269679308, + "num_tokens": 4871202.0, + "step": 1200 + }, + { + "entropy": 0.021590291650136352, + "epoch": 0.29339556875700906, + "grad_norm": 2.09375, + "learning_rate": 8.788888888888891e-06, + "loss": 0.02018619328737259, + "mean_token_accuracy": 0.9886363603174686, + "num_tokens": 4911931.0, + "step": 1210 + }, + { + "entropy": 0.02163595509409788, + "epoch": 0.29582032552359594, + "grad_norm": 1.8984375, + "learning_rate": 8.677777777777779e-06, + "loss": 0.025696516036987305, + "mean_token_accuracy": 0.9863636329770088, + "num_tokens": 4951183.0, + "step": 1220 + }, + { + "entropy": 0.02311627586459508, + "epoch": 0.29824508229018276, + "grad_norm": 0.83203125, + "learning_rate": 8.566666666666667e-06, + "loss": 0.028527703881263734, + "mean_token_accuracy": 0.9835227243602276, + "num_tokens": 4994778.0, + "step": 1230 + }, + { + "entropy": 0.020521293884030456, + "epoch": 0.30066983905676964, + "grad_norm": 0.7421875, + "learning_rate": 8.455555555555555e-06, + "loss": 0.021116530895233153, + "mean_token_accuracy": 0.9914772696793079, + "num_tokens": 5034446.0, + "step": 1240 + }, + { + "entropy": 0.021558842277681834, + "epoch": 0.30309459582335646, + "grad_norm": 1.1953125, + "learning_rate": 8.344444444444445e-06, + "loss": 0.022282668948173524, + "mean_token_accuracy": 0.9897727251052857, + "num_tokens": 5074406.0, + "step": 1250 + }, + { + "entropy": 0.01967097778419884, + "epoch": 0.30551935258994334, + "grad_norm": 1.1953125, + "learning_rate": 8.233333333333335e-06, + "loss": 0.0207993745803833, + "mean_token_accuracy": 0.9920454524457455, + "num_tokens": 5121427.0, + "step": 1260 + }, + { + "entropy": 0.02370198413136677, + "epoch": 0.30794410935653016, + "grad_norm": 0.85546875, + "learning_rate": 8.122222222222223e-06, + "loss": 0.028493967652320863, + "mean_token_accuracy": 0.9846590861678124, + "num_tokens": 5160392.0, + "step": 1270 + }, + { + "entropy": 0.023408127912307463, + "epoch": 0.31036886612311704, + "grad_norm": 3.046875, + "learning_rate": 8.011111111111113e-06, + "loss": 0.024442848563194276, + "mean_token_accuracy": 0.9857954502105712, + "num_tokens": 5199665.0, + "step": 1280 + }, + { + "entropy": 0.018637936454115333, + "epoch": 0.31279362288970386, + "grad_norm": 1.59375, + "learning_rate": 7.9e-06, + "loss": 0.02052522748708725, + "mean_token_accuracy": 0.9869318135082722, + "num_tokens": 5237561.0, + "step": 1290 + }, + { + "entropy": 0.02345360331178199, + "epoch": 0.31521837965629074, + "grad_norm": 2.234375, + "learning_rate": 7.788888888888889e-06, + "loss": 0.03127675354480743, + "mean_token_accuracy": 0.9840909071266651, + "num_tokens": 5276996.0, + "step": 1300 + }, + { + "entropy": 0.024497390102942516, + "epoch": 0.31764313642287756, + "grad_norm": 1.3984375, + "learning_rate": 7.677777777777778e-06, + "loss": 0.02247738242149353, + "mean_token_accuracy": 0.9874999955296516, + "num_tokens": 5318441.0, + "step": 1310 + }, + { + "entropy": 0.018883616356470158, + "epoch": 0.32006789318946444, + "grad_norm": 3.203125, + "learning_rate": 7.566666666666667e-06, + "loss": 0.01997789889574051, + "mean_token_accuracy": 0.9897727236151695, + "num_tokens": 5358209.0, + "step": 1320 + }, + { + "entropy": 0.024151582821377814, + "epoch": 0.32249264995605126, + "grad_norm": 2.078125, + "learning_rate": 7.455555555555556e-06, + "loss": 0.028815129399299623, + "mean_token_accuracy": 0.9823863588273525, + "num_tokens": 5397538.0, + "step": 1330 + }, + { + "entropy": 0.020284308634154515, + "epoch": 0.32491740672263814, + "grad_norm": 1.046875, + "learning_rate": 7.344444444444445e-06, + "loss": 0.019358839094638824, + "mean_token_accuracy": 0.9897727258503437, + "num_tokens": 5435900.0, + "step": 1340 + }, + { + "entropy": 0.02354962546473871, + "epoch": 0.327342163489225, + "grad_norm": 1.3515625, + "learning_rate": 7.233333333333334e-06, + "loss": 0.031699517369270326, + "mean_token_accuracy": 0.9835227228701114, + "num_tokens": 5476573.0, + "step": 1350 + }, + { + "entropy": 0.02446860164272948, + "epoch": 0.32976692025581184, + "grad_norm": 0.92578125, + "learning_rate": 7.122222222222222e-06, + "loss": 0.023002330958843232, + "mean_token_accuracy": 0.9869318142533302, + "num_tokens": 5513779.0, + "step": 1360 + }, + { + "entropy": 0.022691848923705037, + "epoch": 0.3321916770223987, + "grad_norm": 3.484375, + "learning_rate": 7.011111111111112e-06, + "loss": 0.028314879536628722, + "mean_token_accuracy": 0.984090904891491, + "num_tokens": 5555000.0, + "step": 1370 + }, + { + "entropy": 0.019440177543947355, + "epoch": 0.33461643378898553, + "grad_norm": 0.59375, + "learning_rate": 6.9e-06, + "loss": 0.01770046055316925, + "mean_token_accuracy": 0.9914772711694241, + "num_tokens": 5593016.0, + "step": 1380 + }, + { + "entropy": 0.023765943434227665, + "epoch": 0.3370411905555724, + "grad_norm": 1.09375, + "learning_rate": 6.788888888888889e-06, + "loss": 0.02077813595533371, + "mean_token_accuracy": 0.9903409063816071, + "num_tokens": 5635379.0, + "step": 1390 + }, + { + "entropy": 0.01887716218980131, + "epoch": 0.33946594732215923, + "grad_norm": 1.90625, + "learning_rate": 6.677777777777779e-06, + "loss": 0.021604160964488982, + "mean_token_accuracy": 0.9880681782960892, + "num_tokens": 5673576.0, + "step": 1400 + }, + { + "entropy": 0.023854930577999766, + "epoch": 0.3418907040887461, + "grad_norm": 4.21875, + "learning_rate": 6.566666666666667e-06, + "loss": 0.025638672709465026, + "mean_token_accuracy": 0.9846590854227543, + "num_tokens": 5714108.0, + "step": 1410 + }, + { + "entropy": 0.020290096460576022, + "epoch": 0.34431546085533293, + "grad_norm": 0.9609375, + "learning_rate": 6.455555555555556e-06, + "loss": 0.020940312743186952, + "mean_token_accuracy": 0.9909090884029865, + "num_tokens": 5755604.0, + "step": 1420 + }, + { + "entropy": 0.019387308810064497, + "epoch": 0.3467402176219198, + "grad_norm": 3.09375, + "learning_rate": 6.3444444444444454e-06, + "loss": 0.02511414885520935, + "mean_token_accuracy": 0.9880681790411472, + "num_tokens": 5793500.0, + "step": 1430 + }, + { + "entropy": 0.02229226140134415, + "epoch": 0.34916497438850663, + "grad_norm": 0.8359375, + "learning_rate": 6.2333333333333335e-06, + "loss": 0.0231765940785408, + "mean_token_accuracy": 0.9869318157434464, + "num_tokens": 5833950.0, + "step": 1440 + }, + { + "entropy": 0.02084504545036907, + "epoch": 0.3515897311550935, + "grad_norm": 1.734375, + "learning_rate": 6.1222222222222224e-06, + "loss": 0.03049103319644928, + "mean_token_accuracy": 0.9835227228701114, + "num_tokens": 5872957.0, + "step": 1450 + }, + { + "entropy": 0.01889653230196018, + "epoch": 0.35401448792168033, + "grad_norm": 1.4765625, + "learning_rate": 6.011111111111112e-06, + "loss": 0.019928357005119322, + "mean_token_accuracy": 0.990340905636549, + "num_tokens": 5913588.0, + "step": 1460 + }, + { + "entropy": 0.019694303501910326, + "epoch": 0.3564392446882672, + "grad_norm": 3.5, + "learning_rate": 5.9e-06, + "loss": 0.01947687119245529, + "mean_token_accuracy": 0.9909090884029865, + "num_tokens": 5953428.0, + "step": 1470 + }, + { + "entropy": 0.019155515848706273, + "epoch": 0.3588640014548541, + "grad_norm": 2.671875, + "learning_rate": 5.788888888888889e-06, + "loss": 0.022465848922729494, + "mean_token_accuracy": 0.9897727243602276, + "num_tokens": 5993281.0, + "step": 1480 + }, + { + "entropy": 0.021128093433253524, + "epoch": 0.3612887582214409, + "grad_norm": 1.8203125, + "learning_rate": 5.677777777777779e-06, + "loss": 0.027405399084091186, + "mean_token_accuracy": 0.9835227221250534, + "num_tokens": 6035357.0, + "step": 1490 + }, + { + "entropy": 0.022639484232195174, + "epoch": 0.3637135149880278, + "grad_norm": 2.265625, + "learning_rate": 5.566666666666667e-06, + "loss": 0.021980252861976624, + "mean_token_accuracy": 0.9892045430839062, + "num_tokens": 6077438.0, + "step": 1500 + }, + { + "entropy": 0.021943628234339485, + "epoch": 0.3661382717546146, + "grad_norm": 3.640625, + "learning_rate": 5.455555555555556e-06, + "loss": 0.03260737359523773, + "mean_token_accuracy": 0.9852272681891918, + "num_tokens": 6117468.0, + "step": 1510 + }, + { + "entropy": 0.025014979231627876, + "epoch": 0.3685630285212015, + "grad_norm": 1.0546875, + "learning_rate": 5.344444444444446e-06, + "loss": 0.023974181711673738, + "mean_token_accuracy": 0.9869318142533302, + "num_tokens": 6157286.0, + "step": 1520 + }, + { + "entropy": 0.01908629078325248, + "epoch": 0.3709877852877883, + "grad_norm": 1.15625, + "learning_rate": 5.233333333333334e-06, + "loss": 0.015591822564601898, + "mean_token_accuracy": 0.9926136344671249, + "num_tokens": 6198312.0, + "step": 1530 + }, + { + "entropy": 0.020979891011893414, + "epoch": 0.3734125420543752, + "grad_norm": 1.8515625, + "learning_rate": 5.122222222222223e-06, + "loss": 0.022910787165164946, + "mean_token_accuracy": 0.989559655636549, + "num_tokens": 6240280.0, + "step": 1540 + }, + { + "entropy": 0.025256870118482766, + "epoch": 0.375837298820962, + "grad_norm": 3.25, + "learning_rate": 5.011111111111111e-06, + "loss": 0.03658706545829773, + "mean_token_accuracy": 0.9801136322319508, + "num_tokens": 6284875.0, + "step": 1550 + }, + { + "entropy": 0.02342147537783603, + "epoch": 0.3782620555875489, + "grad_norm": 1.2734375, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.025882428884506224, + "mean_token_accuracy": 0.9738636329770088, + "num_tokens": 6329857.0, + "step": 1560 + }, + { + "entropy": 0.0226567227995929, + "epoch": 0.3806868123541357, + "grad_norm": 1.265625, + "learning_rate": 4.7888888888888894e-06, + "loss": 0.018890340626239777, + "mean_token_accuracy": 0.990340905636549, + "num_tokens": 6370302.0, + "step": 1570 + }, + { + "entropy": 0.022661514219635137, + "epoch": 0.3831115691207226, + "grad_norm": 1.921875, + "learning_rate": 4.677777777777778e-06, + "loss": 0.021921022236347197, + "mean_token_accuracy": 0.9874999985098839, + "num_tokens": 6407249.0, + "step": 1580 + }, + { + "entropy": 0.022505474984882312, + "epoch": 0.3855363258873094, + "grad_norm": 4.125, + "learning_rate": 4.566666666666667e-06, + "loss": 0.0322005420923233, + "mean_token_accuracy": 0.9846590884029865, + "num_tokens": 6446978.0, + "step": 1590 + }, + { + "entropy": 0.022300617951987078, + "epoch": 0.3879610826538963, + "grad_norm": 3.046875, + "learning_rate": 4.455555555555555e-06, + "loss": 0.023643006384372712, + "mean_token_accuracy": 0.98920454159379, + "num_tokens": 6486520.0, + "step": 1600 + }, + { + "entropy": 0.02587687493778503, + "epoch": 0.3903858394204831, + "grad_norm": 2.78125, + "learning_rate": 4.344444444444445e-06, + "loss": 0.027235689759254455, + "mean_token_accuracy": 0.985227269679308, + "num_tokens": 6528137.0, + "step": 1610 + }, + { + "entropy": 0.021340829327573373, + "epoch": 0.39281059618707, + "grad_norm": 0.609375, + "learning_rate": 4.233333333333334e-06, + "loss": 0.021160535514354706, + "mean_token_accuracy": 0.9880681797862053, + "num_tokens": 6565831.0, + "step": 1620 + }, + { + "entropy": 0.02321274704027019, + "epoch": 0.39523535295365686, + "grad_norm": 1.609375, + "learning_rate": 4.122222222222222e-06, + "loss": 0.026870492100715637, + "mean_token_accuracy": 0.9869318164885044, + "num_tokens": 6605622.0, + "step": 1630 + }, + { + "entropy": 0.024162391825620945, + "epoch": 0.3976601097202437, + "grad_norm": 1.15625, + "learning_rate": 4.011111111111111e-06, + "loss": 0.025860154628753663, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 6647782.0, + "step": 1640 + }, + { + "entropy": 0.024304146416397998, + "epoch": 0.40008486648683056, + "grad_norm": 2.5625, + "learning_rate": 3.900000000000001e-06, + "loss": 0.02480360269546509, + "mean_token_accuracy": 0.9857954509556294, + "num_tokens": 6688774.0, + "step": 1650 + }, + { + "entropy": 0.024905602099897804, + "epoch": 0.4025096232534174, + "grad_norm": 0.828125, + "learning_rate": 3.7888888888888893e-06, + "loss": 0.02738499641418457, + "mean_token_accuracy": 0.9863636329770088, + "num_tokens": 6731921.0, + "step": 1660 + }, + { + "entropy": 0.022832987389074334, + "epoch": 0.40493438002000426, + "grad_norm": 2.390625, + "learning_rate": 3.6777777777777778e-06, + "loss": 0.027183347940444948, + "mean_token_accuracy": 0.9840909034013748, + "num_tokens": 6771015.0, + "step": 1670 + }, + { + "entropy": 0.024453511039519073, + "epoch": 0.4073591367865911, + "grad_norm": 2.515625, + "learning_rate": 3.566666666666667e-06, + "loss": 0.027268880605697633, + "mean_token_accuracy": 0.9886363595724106, + "num_tokens": 6811078.0, + "step": 1680 + }, + { + "entropy": 0.02214456081451317, + "epoch": 0.40978389355317796, + "grad_norm": 0.98828125, + "learning_rate": 3.455555555555556e-06, + "loss": 0.02482033222913742, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 6848500.0, + "step": 1690 + }, + { + "entropy": 0.02487115224266745, + "epoch": 0.4122086503197648, + "grad_norm": 2.5, + "learning_rate": 3.3444444444444445e-06, + "loss": 0.021296854317188262, + "mean_token_accuracy": 0.9897727251052857, + "num_tokens": 6894353.0, + "step": 1700 + }, + { + "entropy": 0.025811467690800782, + "epoch": 0.41463340708635166, + "grad_norm": 0.9296875, + "learning_rate": 3.2333333333333334e-06, + "loss": 0.032750701904296874, + "mean_token_accuracy": 0.9823863610625267, + "num_tokens": 6937570.0, + "step": 1710 + }, + { + "entropy": 0.01883704749238859, + "epoch": 0.4170581638529385, + "grad_norm": 1.7734375, + "learning_rate": 3.1222222222222228e-06, + "loss": 0.023237355053424835, + "mean_token_accuracy": 0.987472939491272, + "num_tokens": 6979578.0, + "step": 1720 + }, + { + "entropy": 0.022163561169008972, + "epoch": 0.41948292061952536, + "grad_norm": 2.359375, + "learning_rate": 3.0111111111111113e-06, + "loss": 0.0246141716837883, + "mean_token_accuracy": 0.9874999977648258, + "num_tokens": 7015596.0, + "step": 1730 + }, + { + "entropy": 0.026256153864324006, + "epoch": 0.4219076773861122, + "grad_norm": 2.203125, + "learning_rate": 2.9e-06, + "loss": 0.028805309534072877, + "mean_token_accuracy": 0.9823863595724106, + "num_tokens": 7055852.0, + "step": 1740 + }, + { + "entropy": 0.02054688634939339, + "epoch": 0.42433243415269906, + "grad_norm": 1.3125, + "learning_rate": 2.788888888888889e-06, + "loss": 0.018951752781867982, + "mean_token_accuracy": 0.991477270424366, + "num_tokens": 7092886.0, + "step": 1750 + }, + { + "entropy": 0.024736029255382163, + "epoch": 0.42675719091928593, + "grad_norm": 1.1640625, + "learning_rate": 2.677777777777778e-06, + "loss": 0.026023554801940917, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 7134382.0, + "step": 1760 + }, + { + "entropy": 0.023883560596868848, + "epoch": 0.42918194768587276, + "grad_norm": 2.171875, + "learning_rate": 2.566666666666667e-06, + "loss": 0.022034844756126402, + "mean_token_accuracy": 0.9874999970197678, + "num_tokens": 7175153.0, + "step": 1770 + }, + { + "entropy": 0.023680539545011925, + "epoch": 0.43160670445245963, + "grad_norm": 2.65625, + "learning_rate": 2.455555555555556e-06, + "loss": 0.021315962076187134, + "mean_token_accuracy": 0.9880681790411472, + "num_tokens": 7216758.0, + "step": 1780 + }, + { + "entropy": 0.025864069088902396, + "epoch": 0.43403146121904645, + "grad_norm": 3.640625, + "learning_rate": 2.3444444444444448e-06, + "loss": 0.023368486762046815, + "mean_token_accuracy": 0.990340905636549, + "num_tokens": 7257744.0, + "step": 1790 + }, + { + "entropy": 0.021552615765995143, + "epoch": 0.43645621798563333, + "grad_norm": 2.359375, + "learning_rate": 2.2333333333333333e-06, + "loss": 0.02335650324821472, + "mean_token_accuracy": 0.9863636337220669, + "num_tokens": 7293067.0, + "step": 1800 + }, + { + "entropy": 0.023214141586140612, + "epoch": 0.43888097475222015, + "grad_norm": 4.21875, + "learning_rate": 2.1222222222222226e-06, + "loss": 0.028553092479705812, + "mean_token_accuracy": 0.9852272689342498, + "num_tokens": 7334737.0, + "step": 1810 + }, + { + "entropy": 0.019011126364603115, + "epoch": 0.44130573151880703, + "grad_norm": 1.6796875, + "learning_rate": 2.011111111111111e-06, + "loss": 0.027167025208473205, + "mean_token_accuracy": 0.9852272689342498, + "num_tokens": 7377606.0, + "step": 1820 + }, + { + "entropy": 0.022004775167988554, + "epoch": 0.44373048828539385, + "grad_norm": 0.96484375, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.03003118634223938, + "mean_token_accuracy": 0.981818176060915, + "num_tokens": 7417763.0, + "step": 1830 + }, + { + "entropy": 0.022360649610618566, + "epoch": 0.44615524505198073, + "grad_norm": 1.0078125, + "learning_rate": 1.788888888888889e-06, + "loss": 0.02517186403274536, + "mean_token_accuracy": 0.9863636329770088, + "num_tokens": 7459114.0, + "step": 1840 + }, + { + "entropy": 0.02276606035607074, + "epoch": 0.44858000181856755, + "grad_norm": 1.390625, + "learning_rate": 1.6777777777777779e-06, + "loss": 0.020943556725978852, + "mean_token_accuracy": 0.98920454159379, + "num_tokens": 7498162.0, + "step": 1850 + }, + { + "entropy": 0.027342419610067736, + "epoch": 0.45100475858515443, + "grad_norm": 2.1875, + "learning_rate": 1.566666666666667e-06, + "loss": 0.02874574959278107, + "mean_token_accuracy": 0.9846590876579284, + "num_tokens": 7534837.0, + "step": 1860 + }, + { + "entropy": 0.02010235228917736, + "epoch": 0.45342951535174125, + "grad_norm": 1.375, + "learning_rate": 1.4555555555555557e-06, + "loss": 0.029143670201301576, + "mean_token_accuracy": 0.9863636314868927, + "num_tokens": 7575437.0, + "step": 1870 + }, + { + "entropy": 0.019995948372934436, + "epoch": 0.45585427211832813, + "grad_norm": 1.8203125, + "learning_rate": 1.3444444444444446e-06, + "loss": 0.026229032874107362, + "mean_token_accuracy": 0.9846590861678124, + "num_tokens": 7615212.0, + "step": 1880 + }, + { + "entropy": 0.019795780479307724, + "epoch": 0.458279028884915, + "grad_norm": 2.90625, + "learning_rate": 1.2333333333333335e-06, + "loss": 0.020921708643436433, + "mean_token_accuracy": 0.9880681775510312, + "num_tokens": 7661315.0, + "step": 1890 + }, + { + "entropy": 0.021343442313173, + "epoch": 0.46070378565150183, + "grad_norm": 0.703125, + "learning_rate": 1.1222222222222222e-06, + "loss": 0.02051656097173691, + "mean_token_accuracy": 0.9880681790411472, + "num_tokens": 7704690.0, + "step": 1900 + }, + { + "entropy": 0.02385749567615676, + "epoch": 0.4631285424180887, + "grad_norm": 3.765625, + "learning_rate": 1.0111111111111111e-06, + "loss": 0.029207244515419006, + "mean_token_accuracy": 0.9880681790411472, + "num_tokens": 7745332.0, + "step": 1910 + }, + { + "entropy": 0.024054063166568086, + "epoch": 0.46555329918467553, + "grad_norm": 2.1875, + "learning_rate": 9.000000000000001e-07, + "loss": 0.023930326104164124, + "mean_token_accuracy": 0.9863636322319508, + "num_tokens": 7785168.0, + "step": 1920 + }, + { + "entropy": 0.026037648288092897, + "epoch": 0.4679780559512624, + "grad_norm": 0.92578125, + "learning_rate": 7.888888888888889e-07, + "loss": 0.022882741689682008, + "mean_token_accuracy": 0.98920454159379, + "num_tokens": 7826084.0, + "step": 1930 + }, + { + "entropy": 0.02522805301673543, + "epoch": 0.47040281271784923, + "grad_norm": 1.046875, + "learning_rate": 6.777777777777779e-07, + "loss": 0.02458210289478302, + "mean_token_accuracy": 0.9857954509556294, + "num_tokens": 7868131.0, + "step": 1940 + }, + { + "entropy": 0.022025877615305944, + "epoch": 0.4728275694844361, + "grad_norm": 2.1875, + "learning_rate": 5.666666666666667e-07, + "loss": 0.021030083298683167, + "mean_token_accuracy": 0.98920454159379, + "num_tokens": 7910667.0, + "step": 1950 + }, + { + "entropy": 0.022395222801515048, + "epoch": 0.4752523262510229, + "grad_norm": 1.5390625, + "learning_rate": 4.5555555555555563e-07, + "loss": 0.022039563953876497, + "mean_token_accuracy": 0.98920454159379, + "num_tokens": 7951358.0, + "step": 1960 + }, + { + "entropy": 0.021953649059105372, + "epoch": 0.4776770830176098, + "grad_norm": 0.921875, + "learning_rate": 3.444444444444445e-07, + "loss": 0.022923427820205688, + "mean_token_accuracy": 0.9863636337220669, + "num_tokens": 7989788.0, + "step": 1970 + }, + { + "entropy": 0.02497839363559251, + "epoch": 0.4801018397841966, + "grad_norm": 1.3125, + "learning_rate": 2.3333333333333336e-07, + "loss": 0.02633815407752991, + "mean_token_accuracy": 0.9869318149983883, + "num_tokens": 8034242.0, + "step": 1980 + }, + { + "entropy": 0.024830547462852338, + "epoch": 0.4825265965507835, + "grad_norm": 1.375, + "learning_rate": 1.2222222222222225e-07, + "loss": 0.02468583881855011, + "mean_token_accuracy": 0.984090904891491, + "num_tokens": 8077885.0, + "step": 1990 + }, + { + "entropy": 0.02569021463914396, + "epoch": 0.4849513533173703, + "grad_norm": 1.046875, + "learning_rate": 1.1111111111111112e-08, + "loss": 0.02596776783466339, + "mean_token_accuracy": 0.9857954502105712, + "num_tokens": 8117574.0, + "step": 2000 + } + ], + "logging_steps": 10, + "max_steps": 2000, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 8.864313676398182e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin new file mode 100644 index 0000000..0507855 --- /dev/null +++ b/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9dd6eea8c961c9bab212f7162e4662fd69575c17f29490aaa2c230f652a7295 +size 5713 diff --git a/checkpoint-2375/chat_template.jinja b/checkpoint-2375/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-2375/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-2375/config.json b/checkpoint-2375/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-2375/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2375/generation_config.json b/checkpoint-2375/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-2375/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-2375/model.safetensors b/checkpoint-2375/model.safetensors new file mode 100644 index 0000000..5353c30 --- /dev/null +++ b/checkpoint-2375/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c990a63da859bdef134031039e52479dab7fc87d87d1d990c535241f983424 +size 3441185608 diff --git a/checkpoint-2375/optimizer.pt b/checkpoint-2375/optimizer.pt new file mode 100644 index 0000000..9069dc4 --- /dev/null +++ b/checkpoint-2375/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f07290f83300d4aff4200bb111573782d9b3212a5da4a55d5edd597384e8d1 +size 6882572207 diff --git a/checkpoint-2375/rng_state.pth b/checkpoint-2375/rng_state.pth new file mode 100644 index 0000000..e92cf8f --- /dev/null +++ b/checkpoint-2375/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:718a0f3db00824213036a2c0441849791319b7d9cf189065873bb26a7020738e +size 14645 diff --git a/checkpoint-2375/scheduler.pt b/checkpoint-2375/scheduler.pt new file mode 100644 index 0000000..5e9abfc --- /dev/null +++ b/checkpoint-2375/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ee3c7b2814d586afc5f1bcb4770555a94f55b719a9c6f3add7955d43a0df316 +size 1465 diff --git a/checkpoint-2375/tokenizer.json b/checkpoint-2375/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-2375/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-2375/tokenizer_config.json b/checkpoint-2375/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-2375/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2375/trainer_state.json b/checkpoint-2375/trainer_state.json new file mode 100644 index 0000000..db31819 --- /dev/null +++ b/checkpoint-2375/trainer_state.json @@ -0,0 +1,2404 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2375, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.8691443920135498, + "epoch": 0.021085925144965736, + "grad_norm": 47.5, + "learning_rate": 7.2e-07, + "loss": 2.1268569946289064, + "mean_token_accuracy": 0.5856592752039432, + "num_tokens": 53385.0, + "step": 10 + }, + { + "entropy": 0.9018128983676433, + "epoch": 0.04217185028993147, + "grad_norm": 49.25, + "learning_rate": 1.52e-06, + "loss": 2.1235822677612304, + "mean_token_accuracy": 0.5826416034251451, + "num_tokens": 108949.0, + "step": 20 + }, + { + "entropy": 0.9313402399420738, + "epoch": 0.0632577754348972, + "grad_norm": 38.25, + "learning_rate": 2.3200000000000002e-06, + "loss": 1.9997358322143555, + "mean_token_accuracy": 0.5817345686256885, + "num_tokens": 162034.0, + "step": 30 + }, + { + "entropy": 0.9819381944835186, + "epoch": 0.08434370057986294, + "grad_norm": 28.25, + "learning_rate": 3.12e-06, + "loss": 1.8760208129882812, + "mean_token_accuracy": 0.5874959267675877, + "num_tokens": 214165.0, + "step": 40 + }, + { + "entropy": 1.0624087303876877, + "epoch": 0.10542962572482868, + "grad_norm": 14.1875, + "learning_rate": 3.920000000000001e-06, + "loss": 1.6169570922851562, + "mean_token_accuracy": 0.601640059798956, + "num_tokens": 265527.0, + "step": 50 + }, + { + "entropy": 1.1721510097384453, + "epoch": 0.1265155508697944, + "grad_norm": 11.3125, + "learning_rate": 4.7200000000000005e-06, + "loss": 1.4622878074645995, + "mean_token_accuracy": 0.6209174185991287, + "num_tokens": 319341.0, + "step": 60 + }, + { + "entropy": 1.2009219758212566, + "epoch": 0.14760147601476015, + "grad_norm": 9.0625, + "learning_rate": 5.5200000000000005e-06, + "loss": 1.2933337211608886, + "mean_token_accuracy": 0.6467074818909169, + "num_tokens": 373709.0, + "step": 70 + }, + { + "entropy": 1.1439887225627898, + "epoch": 0.16868740115972589, + "grad_norm": 9.3125, + "learning_rate": 6.3200000000000005e-06, + "loss": 1.2084498405456543, + "mean_token_accuracy": 0.6660686552524566, + "num_tokens": 425923.0, + "step": 80 + }, + { + "entropy": 1.0713621236383915, + "epoch": 0.18977332630469163, + "grad_norm": 7.84375, + "learning_rate": 7.1200000000000004e-06, + "loss": 1.1175435066223145, + "mean_token_accuracy": 0.6812180563807487, + "num_tokens": 480012.0, + "step": 90 + }, + { + "entropy": 1.0369222231209279, + "epoch": 0.21085925144965736, + "grad_norm": 7.09375, + "learning_rate": 7.92e-06, + "loss": 1.062850570678711, + "mean_token_accuracy": 0.6899570092558861, + "num_tokens": 533890.0, + "step": 100 + }, + { + "entropy": 0.9894454136490822, + "epoch": 0.23194517659462308, + "grad_norm": 7.75, + "learning_rate": 8.720000000000001e-06, + "loss": 1.0150799751281738, + "mean_token_accuracy": 0.7009239844977856, + "num_tokens": 590654.0, + "step": 110 + }, + { + "entropy": 0.9410129584372043, + "epoch": 0.2530311017395888, + "grad_norm": 7.28125, + "learning_rate": 9.52e-06, + "loss": 0.9810515403747558, + "mean_token_accuracy": 0.709618617594242, + "num_tokens": 643080.0, + "step": 120 + }, + { + "entropy": 0.9265115469694137, + "epoch": 0.2741170268845546, + "grad_norm": 7.46875, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.9485825538635254, + "mean_token_accuracy": 0.7162063598632813, + "num_tokens": 697254.0, + "step": 130 + }, + { + "entropy": 0.9009825974702835, + "epoch": 0.2952029520295203, + "grad_norm": 6.96875, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.9119878768920898, + "mean_token_accuracy": 0.7238413639366627, + "num_tokens": 751774.0, + "step": 140 + }, + { + "entropy": 0.9040703341364861, + "epoch": 0.316288877174486, + "grad_norm": 7.21875, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.9283308029174805, + "mean_token_accuracy": 0.7166316144168376, + "num_tokens": 808927.0, + "step": 150 + }, + { + "entropy": 0.8878034949302673, + "epoch": 0.33737480231945177, + "grad_norm": 7.375, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.9068665504455566, + "mean_token_accuracy": 0.7254431843757629, + "num_tokens": 858559.0, + "step": 160 + }, + { + "entropy": 0.887480016052723, + "epoch": 0.3584607274644175, + "grad_norm": 7.65625, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.8971159934997559, + "mean_token_accuracy": 0.7258390121161937, + "num_tokens": 912285.0, + "step": 170 + }, + { + "entropy": 0.8527634374797344, + "epoch": 0.37954665260938325, + "grad_norm": 7.6875, + "learning_rate": 1.432e-05, + "loss": 0.8586221694946289, + "mean_token_accuracy": 0.7359919100999832, + "num_tokens": 969385.0, + "step": 180 + }, + { + "entropy": 0.8445627830922604, + "epoch": 0.40063257775434896, + "grad_norm": 7.03125, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.855714225769043, + "mean_token_accuracy": 0.7341729424893856, + "num_tokens": 1026106.0, + "step": 190 + }, + { + "entropy": 0.841130618005991, + "epoch": 0.42171850289931473, + "grad_norm": 7.21875, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.8620488166809082, + "mean_token_accuracy": 0.7361006513237953, + "num_tokens": 1079743.0, + "step": 200 + }, + { + "entropy": 0.831258487701416, + "epoch": 0.44280442804428044, + "grad_norm": 7.0, + "learning_rate": 1.672e-05, + "loss": 0.8441590309143067, + "mean_token_accuracy": 0.7379129216074943, + "num_tokens": 1131453.0, + "step": 210 + }, + { + "entropy": 0.79731200709939, + "epoch": 0.46389035318924615, + "grad_norm": 7.875, + "learning_rate": 1.752e-05, + "loss": 0.8135190963745117, + "mean_token_accuracy": 0.7497627973556519, + "num_tokens": 1182486.0, + "step": 220 + }, + { + "entropy": 0.8269097700715065, + "epoch": 0.4849762783342119, + "grad_norm": 7.5625, + "learning_rate": 1.832e-05, + "loss": 0.8399410247802734, + "mean_token_accuracy": 0.7389348305761814, + "num_tokens": 1238136.0, + "step": 230 + }, + { + "entropy": 0.8093138128519058, + "epoch": 0.5060622034791776, + "grad_norm": 7.40625, + "learning_rate": 1.912e-05, + "loss": 0.810025691986084, + "mean_token_accuracy": 0.7460841409862041, + "num_tokens": 1294939.0, + "step": 240 + }, + { + "entropy": 0.8069386176764966, + "epoch": 0.5271481286241434, + "grad_norm": 6.84375, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8215888023376465, + "mean_token_accuracy": 0.7438369721174241, + "num_tokens": 1346462.0, + "step": 250 + }, + { + "entropy": 0.8088659413158894, + "epoch": 0.5482340537691092, + "grad_norm": 6.75, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8201449394226075, + "mean_token_accuracy": 0.7393839374184609, + "num_tokens": 1397444.0, + "step": 260 + }, + { + "entropy": 0.7969542622566224, + "epoch": 0.5693199789140748, + "grad_norm": 6.59375, + "learning_rate": 1.9831111111111113e-05, + "loss": 0.8009759902954101, + "mean_token_accuracy": 0.7512488305568695, + "num_tokens": 1453900.0, + "step": 270 + }, + { + "entropy": 0.780416651815176, + "epoch": 0.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.9742222222222223e-05, + "loss": 0.7874263763427735, + "mean_token_accuracy": 0.7526524804532528, + "num_tokens": 1505529.0, + "step": 280 + }, + { + "entropy": 0.7752278268337249, + "epoch": 0.6114918292040064, + "grad_norm": 6.875, + "learning_rate": 1.9653333333333334e-05, + "loss": 0.8075678825378418, + "mean_token_accuracy": 0.746569037437439, + "num_tokens": 1562473.0, + "step": 290 + }, + { + "entropy": 0.7895354829728604, + "epoch": 0.632577754348972, + "grad_norm": 6.6875, + "learning_rate": 1.9564444444444444e-05, + "loss": 0.7880948543548584, + "mean_token_accuracy": 0.7494045957922936, + "num_tokens": 1613320.0, + "step": 300 + }, + { + "entropy": 0.7653828673064709, + "epoch": 0.6536636794939378, + "grad_norm": 6.78125, + "learning_rate": 1.9475555555555558e-05, + "loss": 0.7780828475952148, + "mean_token_accuracy": 0.7556171745061875, + "num_tokens": 1662847.0, + "step": 310 + }, + { + "entropy": 0.7652887552976608, + "epoch": 0.6747496046389035, + "grad_norm": 6.375, + "learning_rate": 1.938666666666667e-05, + "loss": 0.7539787292480469, + "mean_token_accuracy": 0.7582654722034932, + "num_tokens": 1718553.0, + "step": 320 + }, + { + "entropy": 0.7076033115386963, + "epoch": 0.6958355297838693, + "grad_norm": 7.15625, + "learning_rate": 1.929777777777778e-05, + "loss": 0.7396605491638184, + "mean_token_accuracy": 0.7659220136702061, + "num_tokens": 1771024.0, + "step": 330 + }, + { + "entropy": 0.7509547933936119, + "epoch": 0.716921454928835, + "grad_norm": 6.15625, + "learning_rate": 1.920888888888889e-05, + "loss": 0.7459388256072998, + "mean_token_accuracy": 0.7644543245434761, + "num_tokens": 1821303.0, + "step": 340 + }, + { + "entropy": 0.7591827407479286, + "epoch": 0.7380073800738007, + "grad_norm": 7.03125, + "learning_rate": 1.912e-05, + "loss": 0.7627717971801757, + "mean_token_accuracy": 0.7567056350409984, + "num_tokens": 1868674.0, + "step": 350 + }, + { + "entropy": 0.7316406652331352, + "epoch": 0.7590933052187665, + "grad_norm": 6.59375, + "learning_rate": 1.9031111111111114e-05, + "loss": 0.7425707340240478, + "mean_token_accuracy": 0.7653280735015869, + "num_tokens": 1924619.0, + "step": 360 + }, + { + "entropy": 0.7539067789912224, + "epoch": 0.7801792303637322, + "grad_norm": 7.0625, + "learning_rate": 1.8942222222222224e-05, + "loss": 0.7716987609863282, + "mean_token_accuracy": 0.7524559155106545, + "num_tokens": 1978795.0, + "step": 370 + }, + { + "entropy": 0.7687754213809967, + "epoch": 0.8012651555086979, + "grad_norm": 7.1875, + "learning_rate": 1.8853333333333335e-05, + "loss": 0.7734373569488525, + "mean_token_accuracy": 0.7497503250837326, + "num_tokens": 2029994.0, + "step": 380 + }, + { + "entropy": 0.7508607231080532, + "epoch": 0.8223510806536637, + "grad_norm": 7.09375, + "learning_rate": 1.8764444444444445e-05, + "loss": 0.7575721740722656, + "mean_token_accuracy": 0.7572125047445297, + "num_tokens": 2082653.0, + "step": 390 + }, + { + "entropy": 0.7252861328423024, + "epoch": 0.8434370057986295, + "grad_norm": 6.5625, + "learning_rate": 1.867555555555556e-05, + "loss": 0.7381186485290527, + "mean_token_accuracy": 0.7612978041172027, + "num_tokens": 2135276.0, + "step": 400 + }, + { + "entropy": 0.7339485622942448, + "epoch": 0.8645229309435951, + "grad_norm": 6.625, + "learning_rate": 1.858666666666667e-05, + "loss": 0.7348906993865967, + "mean_token_accuracy": 0.7663561321794987, + "num_tokens": 2187466.0, + "step": 410 + }, + { + "entropy": 0.7272624362260103, + "epoch": 0.8856088560885609, + "grad_norm": 6.90625, + "learning_rate": 1.849777777777778e-05, + "loss": 0.749025821685791, + "mean_token_accuracy": 0.7617379955947399, + "num_tokens": 2237434.0, + "step": 420 + }, + { + "entropy": 0.7477380312979222, + "epoch": 0.9066947812335266, + "grad_norm": 6.34375, + "learning_rate": 1.840888888888889e-05, + "loss": 0.7498275756835937, + "mean_token_accuracy": 0.7604753144085408, + "num_tokens": 2288985.0, + "step": 430 + }, + { + "entropy": 0.7346535548567772, + "epoch": 0.9277807063784923, + "grad_norm": 6.9375, + "learning_rate": 1.832e-05, + "loss": 0.739972448348999, + "mean_token_accuracy": 0.7658773727715016, + "num_tokens": 2341187.0, + "step": 440 + }, + { + "entropy": 0.7167556572705507, + "epoch": 0.9488666315234581, + "grad_norm": 6.4375, + "learning_rate": 1.823111111111111e-05, + "loss": 0.7350784778594971, + "mean_token_accuracy": 0.7644179269671441, + "num_tokens": 2393385.0, + "step": 450 + }, + { + "entropy": 0.7537519961595536, + "epoch": 0.9699525566684238, + "grad_norm": 7.28125, + "learning_rate": 1.814222222222222e-05, + "loss": 0.757652759552002, + "mean_token_accuracy": 0.7587068513035774, + "num_tokens": 2442248.0, + "step": 460 + }, + { + "entropy": 0.72038439437747, + "epoch": 0.9910384818133896, + "grad_norm": 7.40625, + "learning_rate": 1.8053333333333332e-05, + "loss": 0.7361824035644531, + "mean_token_accuracy": 0.7609050884842873, + "num_tokens": 2498394.0, + "step": 470 + }, + { + "entropy": 0.735570513718837, + "epoch": 1.0105429625724829, + "grad_norm": 6.875, + "learning_rate": 1.7964444444444446e-05, + "loss": 0.730868148803711, + "mean_token_accuracy": 0.7710521068121936, + "num_tokens": 2545422.0, + "step": 480 + }, + { + "entropy": 0.6775223582983017, + "epoch": 1.0316288877174487, + "grad_norm": 7.125, + "learning_rate": 1.7875555555555556e-05, + "loss": 0.6740085124969483, + "mean_token_accuracy": 0.7795436836779117, + "num_tokens": 2597144.0, + "step": 490 + }, + { + "entropy": 0.6628748029470444, + "epoch": 1.0527148128624144, + "grad_norm": 6.84375, + "learning_rate": 1.7786666666666667e-05, + "loss": 0.6774313926696778, + "mean_token_accuracy": 0.7789938881993294, + "num_tokens": 2651938.0, + "step": 500 + }, + { + "entropy": 0.6695188779383898, + "epoch": 1.07380073800738, + "grad_norm": 7.0, + "learning_rate": 1.7697777777777777e-05, + "loss": 0.664512825012207, + "mean_token_accuracy": 0.7832319870591163, + "num_tokens": 2704948.0, + "step": 510 + }, + { + "entropy": 0.6812492057681083, + "epoch": 1.0948866631523457, + "grad_norm": 7.0, + "learning_rate": 1.760888888888889e-05, + "loss": 0.7036191940307617, + "mean_token_accuracy": 0.7707728065550328, + "num_tokens": 2757576.0, + "step": 520 + }, + { + "entropy": 0.6861097872257232, + "epoch": 1.1159725882973115, + "grad_norm": 7.3125, + "learning_rate": 1.752e-05, + "loss": 0.6651991844177246, + "mean_token_accuracy": 0.7841649115085602, + "num_tokens": 2808989.0, + "step": 530 + }, + { + "entropy": 0.6694958411157131, + "epoch": 1.1370585134422773, + "grad_norm": 6.78125, + "learning_rate": 1.7431111111111112e-05, + "loss": 0.6833772659301758, + "mean_token_accuracy": 0.7775424145162105, + "num_tokens": 2864507.0, + "step": 540 + }, + { + "entropy": 0.6568067740648985, + "epoch": 1.158144438587243, + "grad_norm": 6.21875, + "learning_rate": 1.7342222222222226e-05, + "loss": 0.6552294254302978, + "mean_token_accuracy": 0.7832994520664215, + "num_tokens": 2919201.0, + "step": 550 + }, + { + "entropy": 0.6432577680796385, + "epoch": 1.1792303637322088, + "grad_norm": 6.65625, + "learning_rate": 1.7253333333333336e-05, + "loss": 0.6560451030731201, + "mean_token_accuracy": 0.7863748580217361, + "num_tokens": 2972434.0, + "step": 560 + }, + { + "entropy": 0.6655397050082683, + "epoch": 1.2003162888771746, + "grad_norm": 6.09375, + "learning_rate": 1.7164444444444447e-05, + "loss": 0.6878513336181641, + "mean_token_accuracy": 0.7753379993140698, + "num_tokens": 3025099.0, + "step": 570 + }, + { + "entropy": 0.686093359068036, + "epoch": 1.2214022140221403, + "grad_norm": 7.09375, + "learning_rate": 1.7075555555555557e-05, + "loss": 0.6911579132080078, + "mean_token_accuracy": 0.7787310972809791, + "num_tokens": 3076307.0, + "step": 580 + }, + { + "entropy": 0.6812980778515338, + "epoch": 1.2424881391671059, + "grad_norm": 7.125, + "learning_rate": 1.6986666666666668e-05, + "loss": 0.6704156398773193, + "mean_token_accuracy": 0.7793935135006904, + "num_tokens": 3130053.0, + "step": 590 + }, + { + "entropy": 0.6409769468009472, + "epoch": 1.2635740643120716, + "grad_norm": 7.4375, + "learning_rate": 1.6897777777777778e-05, + "loss": 0.661864185333252, + "mean_token_accuracy": 0.7865670621395111, + "num_tokens": 3183331.0, + "step": 600 + }, + { + "entropy": 0.6799454756081105, + "epoch": 1.2846599894570374, + "grad_norm": 6.65625, + "learning_rate": 1.680888888888889e-05, + "loss": 0.6818383693695068, + "mean_token_accuracy": 0.778287123888731, + "num_tokens": 3239561.0, + "step": 610 + }, + { + "entropy": 0.6814555354416371, + "epoch": 1.3057459146020032, + "grad_norm": 6.3125, + "learning_rate": 1.672e-05, + "loss": 0.679498815536499, + "mean_token_accuracy": 0.7788635492324829, + "num_tokens": 3292996.0, + "step": 620 + }, + { + "entropy": 0.667108029127121, + "epoch": 1.326831839746969, + "grad_norm": 6.90625, + "learning_rate": 1.6631111111111113e-05, + "loss": 0.6758651733398438, + "mean_token_accuracy": 0.7792007848620415, + "num_tokens": 3345736.0, + "step": 630 + }, + { + "entropy": 0.6692709110677242, + "epoch": 1.3479177648919347, + "grad_norm": 6.96875, + "learning_rate": 1.6542222222222223e-05, + "loss": 0.6796128273010253, + "mean_token_accuracy": 0.7766051791608334, + "num_tokens": 3398076.0, + "step": 640 + }, + { + "entropy": 0.6564048878848553, + "epoch": 1.3690036900369003, + "grad_norm": 6.9375, + "learning_rate": 1.6453333333333334e-05, + "loss": 0.6633264064788819, + "mean_token_accuracy": 0.7854946702718735, + "num_tokens": 3450796.0, + "step": 650 + }, + { + "entropy": 0.6611979272216558, + "epoch": 1.390089615181866, + "grad_norm": 6.65625, + "learning_rate": 1.6364444444444444e-05, + "loss": 0.6661429405212402, + "mean_token_accuracy": 0.7835663430392742, + "num_tokens": 3503863.0, + "step": 660 + }, + { + "entropy": 0.6534961331635714, + "epoch": 1.4111755403268318, + "grad_norm": 6.9375, + "learning_rate": 1.6275555555555558e-05, + "loss": 0.6586530208587646, + "mean_token_accuracy": 0.7844067431986332, + "num_tokens": 3556265.0, + "step": 670 + }, + { + "entropy": 0.6516873501241207, + "epoch": 1.4322614654717976, + "grad_norm": 6.40625, + "learning_rate": 1.618666666666667e-05, + "loss": 0.6522529125213623, + "mean_token_accuracy": 0.7855845786631107, + "num_tokens": 3611423.0, + "step": 680 + }, + { + "entropy": 0.6764013767242432, + "epoch": 1.4533473906167633, + "grad_norm": 6.71875, + "learning_rate": 1.609777777777778e-05, + "loss": 0.6884442806243897, + "mean_token_accuracy": 0.7756352722644806, + "num_tokens": 3666630.0, + "step": 690 + }, + { + "entropy": 0.6554486058652401, + "epoch": 1.474433315761729, + "grad_norm": 7.40625, + "learning_rate": 1.600888888888889e-05, + "loss": 0.6563316822052002, + "mean_token_accuracy": 0.7854672558605671, + "num_tokens": 3718719.0, + "step": 700 + }, + { + "entropy": 0.6385185327380896, + "epoch": 1.4955192409066949, + "grad_norm": 6.59375, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.6513954639434815, + "mean_token_accuracy": 0.7851097173988819, + "num_tokens": 3771822.0, + "step": 710 + }, + { + "entropy": 0.654766371101141, + "epoch": 1.5166051660516606, + "grad_norm": 6.5, + "learning_rate": 1.5831111111111114e-05, + "loss": 0.6622748851776123, + "mean_token_accuracy": 0.7847663886845112, + "num_tokens": 3823795.0, + "step": 720 + }, + { + "entropy": 0.6597310703247785, + "epoch": 1.5376910911966264, + "grad_norm": 6.375, + "learning_rate": 1.5742222222222224e-05, + "loss": 0.6634318351745605, + "mean_token_accuracy": 0.7861979939043522, + "num_tokens": 3875784.0, + "step": 730 + }, + { + "entropy": 0.6637851595878601, + "epoch": 1.558777016341592, + "grad_norm": 6.9375, + "learning_rate": 1.5653333333333335e-05, + "loss": 0.6621220111846924, + "mean_token_accuracy": 0.7886320613324642, + "num_tokens": 3929412.0, + "step": 740 + }, + { + "entropy": 0.637692479044199, + "epoch": 1.5798629414865577, + "grad_norm": 6.59375, + "learning_rate": 1.5564444444444445e-05, + "loss": 0.6402386665344239, + "mean_token_accuracy": 0.7895873270928859, + "num_tokens": 3982472.0, + "step": 750 + }, + { + "entropy": 0.6578179821372032, + "epoch": 1.6009488666315235, + "grad_norm": 7.25, + "learning_rate": 1.5475555555555556e-05, + "loss": 0.6751936435699463, + "mean_token_accuracy": 0.7776105716824532, + "num_tokens": 4037829.0, + "step": 760 + }, + { + "entropy": 0.6653612300753593, + "epoch": 1.6220347917764892, + "grad_norm": 6.9375, + "learning_rate": 1.5386666666666666e-05, + "loss": 0.6717413425445556, + "mean_token_accuracy": 0.777862798422575, + "num_tokens": 4093735.0, + "step": 770 + }, + { + "entropy": 0.6561707239598036, + "epoch": 1.6431207169214548, + "grad_norm": 7.03125, + "learning_rate": 1.5297777777777777e-05, + "loss": 0.6501868724822998, + "mean_token_accuracy": 0.7863908343017101, + "num_tokens": 4148410.0, + "step": 780 + }, + { + "entropy": 0.6655665650963783, + "epoch": 1.6642066420664205, + "grad_norm": 6.59375, + "learning_rate": 1.520888888888889e-05, + "loss": 0.679684829711914, + "mean_token_accuracy": 0.778567586839199, + "num_tokens": 4202907.0, + "step": 790 + }, + { + "entropy": 0.6781473312526941, + "epoch": 1.6852925672113863, + "grad_norm": 6.875, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.6722404479980468, + "mean_token_accuracy": 0.7763243250548839, + "num_tokens": 4256213.0, + "step": 800 + }, + { + "entropy": 0.6321199681609869, + "epoch": 1.706378492356352, + "grad_norm": 6.78125, + "learning_rate": 1.5031111111111111e-05, + "loss": 0.6404708385467529, + "mean_token_accuracy": 0.7903632469475269, + "num_tokens": 4309268.0, + "step": 810 + }, + { + "entropy": 0.6424700867384672, + "epoch": 1.7274644175013179, + "grad_norm": 7.15625, + "learning_rate": 1.4942222222222222e-05, + "loss": 0.6444578170776367, + "mean_token_accuracy": 0.7881530813872815, + "num_tokens": 4360761.0, + "step": 820 + }, + { + "entropy": 0.6615507304668427, + "epoch": 1.7485503426462836, + "grad_norm": 6.0625, + "learning_rate": 1.4853333333333336e-05, + "loss": 0.6604718208312989, + "mean_token_accuracy": 0.782805722206831, + "num_tokens": 4411678.0, + "step": 830 + }, + { + "entropy": 0.6196760050952435, + "epoch": 1.7696362677912494, + "grad_norm": 6.6875, + "learning_rate": 1.4764444444444446e-05, + "loss": 0.6261836051940918, + "mean_token_accuracy": 0.7926589228212834, + "num_tokens": 4467254.0, + "step": 840 + }, + { + "entropy": 0.6139154363423586, + "epoch": 1.7907221929362152, + "grad_norm": 7.21875, + "learning_rate": 1.4675555555555557e-05, + "loss": 0.6301932811737061, + "mean_token_accuracy": 0.7918383464217186, + "num_tokens": 4517383.0, + "step": 850 + }, + { + "entropy": 0.6410171519964933, + "epoch": 1.811808118081181, + "grad_norm": 7.1875, + "learning_rate": 1.4586666666666667e-05, + "loss": 0.655810022354126, + "mean_token_accuracy": 0.7856725618243218, + "num_tokens": 4569980.0, + "step": 860 + }, + { + "entropy": 0.6666022066026926, + "epoch": 1.8328940432261467, + "grad_norm": 6.5625, + "learning_rate": 1.449777777777778e-05, + "loss": 0.6677482604980469, + "mean_token_accuracy": 0.7798060193657875, + "num_tokens": 4622612.0, + "step": 870 + }, + { + "entropy": 0.6398684818297624, + "epoch": 1.8539799683711122, + "grad_norm": 7.84375, + "learning_rate": 1.440888888888889e-05, + "loss": 0.6501501083374024, + "mean_token_accuracy": 0.7857734531164169, + "num_tokens": 4676525.0, + "step": 880 + }, + { + "entropy": 0.6285852700471878, + "epoch": 1.875065893516078, + "grad_norm": 7.0625, + "learning_rate": 1.432e-05, + "loss": 0.6422877311706543, + "mean_token_accuracy": 0.7884316690266132, + "num_tokens": 4734090.0, + "step": 890 + }, + { + "entropy": 0.6409385818988085, + "epoch": 1.8961518186610438, + "grad_norm": 7.9375, + "learning_rate": 1.4231111111111114e-05, + "loss": 0.6352181434631348, + "mean_token_accuracy": 0.7952412232756615, + "num_tokens": 4788537.0, + "step": 900 + }, + { + "entropy": 0.6456746399402619, + "epoch": 1.9172377438060095, + "grad_norm": 6.90625, + "learning_rate": 1.4142222222222224e-05, + "loss": 0.6564863204956055, + "mean_token_accuracy": 0.789187741279602, + "num_tokens": 4840342.0, + "step": 910 + }, + { + "entropy": 0.6360713161528111, + "epoch": 1.938323668950975, + "grad_norm": 6.3125, + "learning_rate": 1.4053333333333335e-05, + "loss": 0.6300373554229737, + "mean_token_accuracy": 0.7876299776136875, + "num_tokens": 4891524.0, + "step": 920 + }, + { + "entropy": 0.6285211775451899, + "epoch": 1.9594095940959408, + "grad_norm": 7.78125, + "learning_rate": 1.3964444444444445e-05, + "loss": 0.6337003231048584, + "mean_token_accuracy": 0.7889726974070073, + "num_tokens": 4939573.0, + "step": 930 + }, + { + "entropy": 0.6301535632461309, + "epoch": 1.9804955192409066, + "grad_norm": 6.84375, + "learning_rate": 1.3875555555555557e-05, + "loss": 0.6374162673950196, + "mean_token_accuracy": 0.7898242607712745, + "num_tokens": 4991267.0, + "step": 940 + }, + { + "entropy": 0.6537058566873138, + "epoch": 2.0, + "grad_norm": 13.3125, + "learning_rate": 1.3786666666666668e-05, + "loss": 0.641156530380249, + "mean_token_accuracy": 0.7822844466647586, + "num_tokens": 5036304.0, + "step": 950 + }, + { + "entropy": 0.6411334682255984, + "epoch": 2.0210859251449658, + "grad_norm": 7.1875, + "learning_rate": 1.3697777777777778e-05, + "loss": 0.6083747386932373, + "mean_token_accuracy": 0.8002627611160278, + "num_tokens": 5089737.0, + "step": 960 + }, + { + "entropy": 0.5841103594750165, + "epoch": 2.0421718502899315, + "grad_norm": 6.53125, + "learning_rate": 1.3608888888888889e-05, + "loss": 0.6012133121490478, + "mean_token_accuracy": 0.8027123242616654, + "num_tokens": 5143330.0, + "step": 970 + }, + { + "entropy": 0.590392829477787, + "epoch": 2.0632577754348973, + "grad_norm": 6.46875, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.5829256534576416, + "mean_token_accuracy": 0.8068650290369987, + "num_tokens": 5198614.0, + "step": 980 + }, + { + "entropy": 0.6148912344127894, + "epoch": 2.084343700579863, + "grad_norm": 6.59375, + "learning_rate": 1.3431111111111113e-05, + "loss": 0.6117685317993165, + "mean_token_accuracy": 0.7951427385210991, + "num_tokens": 5252755.0, + "step": 990 + }, + { + "entropy": 0.5893940377980471, + "epoch": 2.105429625724829, + "grad_norm": 6.625, + "learning_rate": 1.3342222222222224e-05, + "loss": 0.5790496349334717, + "mean_token_accuracy": 0.809234394133091, + "num_tokens": 5304859.0, + "step": 1000 + }, + { + "entropy": 0.587013079598546, + "epoch": 2.1265155508697946, + "grad_norm": 6.9375, + "learning_rate": 1.3253333333333334e-05, + "loss": 0.5892822265625, + "mean_token_accuracy": 0.8040889374911785, + "num_tokens": 5357877.0, + "step": 1010 + }, + { + "entropy": 0.5877181258052587, + "epoch": 2.14760147601476, + "grad_norm": 6.09375, + "learning_rate": 1.3164444444444446e-05, + "loss": 0.595665693283081, + "mean_token_accuracy": 0.8036225430667401, + "num_tokens": 5413504.0, + "step": 1020 + }, + { + "entropy": 0.5848948258906603, + "epoch": 2.1686874011597257, + "grad_norm": 6.4375, + "learning_rate": 1.3075555555555557e-05, + "loss": 0.5771171092987061, + "mean_token_accuracy": 0.8047491163015366, + "num_tokens": 5466549.0, + "step": 1030 + }, + { + "entropy": 0.5815035384148359, + "epoch": 2.1897733263046915, + "grad_norm": 6.96875, + "learning_rate": 1.2986666666666667e-05, + "loss": 0.5768617153167724, + "mean_token_accuracy": 0.8057669088244438, + "num_tokens": 5519515.0, + "step": 1040 + }, + { + "entropy": 0.58335498906672, + "epoch": 2.2108592514496572, + "grad_norm": 7.6875, + "learning_rate": 1.2897777777777778e-05, + "loss": 0.6007503509521485, + "mean_token_accuracy": 0.8005498602986336, + "num_tokens": 5570188.0, + "step": 1050 + }, + { + "entropy": 0.6068603809922933, + "epoch": 2.231945176594623, + "grad_norm": 6.5625, + "learning_rate": 1.2808888888888891e-05, + "loss": 0.6024115562438965, + "mean_token_accuracy": 0.7973541341722011, + "num_tokens": 5624190.0, + "step": 1060 + }, + { + "entropy": 0.5774707745760679, + "epoch": 2.2530311017395888, + "grad_norm": 6.53125, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.5705479145050049, + "mean_token_accuracy": 0.8100002221763134, + "num_tokens": 5675656.0, + "step": 1070 + }, + { + "entropy": 0.5851722996681928, + "epoch": 2.2741170268845545, + "grad_norm": 7.3125, + "learning_rate": 1.2631111111111112e-05, + "loss": 0.5905722141265869, + "mean_token_accuracy": 0.8069308035075664, + "num_tokens": 5728878.0, + "step": 1080 + }, + { + "entropy": 0.594813584536314, + "epoch": 2.2952029520295203, + "grad_norm": 6.875, + "learning_rate": 1.2542222222222223e-05, + "loss": 0.5923901557922363, + "mean_token_accuracy": 0.8034852437675, + "num_tokens": 5783257.0, + "step": 1090 + }, + { + "entropy": 0.5841789487749338, + "epoch": 2.316288877174486, + "grad_norm": 7.0, + "learning_rate": 1.2453333333333335e-05, + "loss": 0.5864346504211426, + "mean_token_accuracy": 0.8033530995249748, + "num_tokens": 5834581.0, + "step": 1100 + }, + { + "entropy": 0.6010521166026592, + "epoch": 2.337374802319452, + "grad_norm": 6.9375, + "learning_rate": 1.2364444444444445e-05, + "loss": 0.5941137313842774, + "mean_token_accuracy": 0.8033333070576191, + "num_tokens": 5887458.0, + "step": 1110 + }, + { + "entropy": 0.5919341973960399, + "epoch": 2.3584607274644176, + "grad_norm": 8.75, + "learning_rate": 1.2275555555555556e-05, + "loss": 0.6103182315826416, + "mean_token_accuracy": 0.7999237082898617, + "num_tokens": 5941284.0, + "step": 1120 + }, + { + "entropy": 0.5942621320486069, + "epoch": 2.3795466526093834, + "grad_norm": 7.375, + "learning_rate": 1.2186666666666666e-05, + "loss": 0.5983363151550293, + "mean_token_accuracy": 0.8016222022473812, + "num_tokens": 5995817.0, + "step": 1130 + }, + { + "entropy": 0.6046227600425482, + "epoch": 2.400632577754349, + "grad_norm": 6.40625, + "learning_rate": 1.209777777777778e-05, + "loss": 0.6026669025421143, + "mean_token_accuracy": 0.8024684950709343, + "num_tokens": 6050783.0, + "step": 1140 + }, + { + "entropy": 0.5998306684195995, + "epoch": 2.421718502899315, + "grad_norm": 6.59375, + "learning_rate": 1.200888888888889e-05, + "loss": 0.5893473625183105, + "mean_token_accuracy": 0.803314708918333, + "num_tokens": 6103371.0, + "step": 1150 + }, + { + "entropy": 0.593693134188652, + "epoch": 2.4428044280442807, + "grad_norm": 6.5, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.6076583862304688, + "mean_token_accuracy": 0.7995239071547985, + "num_tokens": 6155502.0, + "step": 1160 + }, + { + "entropy": 0.5827030725777149, + "epoch": 2.463890353189246, + "grad_norm": 6.84375, + "learning_rate": 1.1831111111111112e-05, + "loss": 0.5882821559906006, + "mean_token_accuracy": 0.7929977595806121, + "num_tokens": 6208060.0, + "step": 1170 + }, + { + "entropy": 0.5910260166972876, + "epoch": 2.4849762783342118, + "grad_norm": 7.53125, + "learning_rate": 1.1742222222222224e-05, + "loss": 0.5860947132110595, + "mean_token_accuracy": 0.8049415498971939, + "num_tokens": 6261588.0, + "step": 1180 + }, + { + "entropy": 0.5704694345593453, + "epoch": 2.5060622034791775, + "grad_norm": 6.75, + "learning_rate": 1.1653333333333334e-05, + "loss": 0.5749064445495605, + "mean_token_accuracy": 0.8084491945803165, + "num_tokens": 6314963.0, + "step": 1190 + }, + { + "entropy": 0.597440430521965, + "epoch": 2.5271481286241433, + "grad_norm": 6.5, + "learning_rate": 1.1564444444444445e-05, + "loss": 0.5958696842193604, + "mean_token_accuracy": 0.8057812102138996, + "num_tokens": 6368741.0, + "step": 1200 + }, + { + "entropy": 0.6070981964468956, + "epoch": 2.548234053769109, + "grad_norm": 7.96875, + "learning_rate": 1.1475555555555557e-05, + "loss": 0.6143896102905273, + "mean_token_accuracy": 0.7980599500238895, + "num_tokens": 6425424.0, + "step": 1210 + }, + { + "entropy": 0.5948727998882533, + "epoch": 2.569319978914075, + "grad_norm": 6.5, + "learning_rate": 1.1386666666666669e-05, + "loss": 0.5917119026184082, + "mean_token_accuracy": 0.8025051333010197, + "num_tokens": 6476263.0, + "step": 1220 + }, + { + "entropy": 0.5811738461256027, + "epoch": 2.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.129777777777778e-05, + "loss": 0.5865864753723145, + "mean_token_accuracy": 0.8044181451201439, + "num_tokens": 6528012.0, + "step": 1230 + }, + { + "entropy": 0.5937653455883265, + "epoch": 2.6114918292040064, + "grad_norm": 6.59375, + "learning_rate": 1.120888888888889e-05, + "loss": 0.5843213558197021, + "mean_token_accuracy": 0.8025668740272522, + "num_tokens": 6583421.0, + "step": 1240 + }, + { + "entropy": 0.573565949127078, + "epoch": 2.632577754348972, + "grad_norm": 6.5625, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.5826557159423829, + "mean_token_accuracy": 0.8070438347756863, + "num_tokens": 6636858.0, + "step": 1250 + }, + { + "entropy": 0.6216809719800949, + "epoch": 2.653663679493938, + "grad_norm": 7.09375, + "learning_rate": 1.1031111111111112e-05, + "loss": 0.6125890254974365, + "mean_token_accuracy": 0.7970895677804947, + "num_tokens": 6692168.0, + "step": 1260 + }, + { + "entropy": 0.5998566258698702, + "epoch": 2.6747496046389037, + "grad_norm": 7.0625, + "learning_rate": 1.0942222222222223e-05, + "loss": 0.5921674728393554, + "mean_token_accuracy": 0.8061520479619503, + "num_tokens": 6745834.0, + "step": 1270 + }, + { + "entropy": 0.5880189586430788, + "epoch": 2.6958355297838694, + "grad_norm": 7.59375, + "learning_rate": 1.0853333333333333e-05, + "loss": 0.5986497402191162, + "mean_token_accuracy": 0.8006219923496246, + "num_tokens": 6800244.0, + "step": 1280 + }, + { + "entropy": 0.5981763809919357, + "epoch": 2.716921454928835, + "grad_norm": 6.6875, + "learning_rate": 1.0764444444444445e-05, + "loss": 0.6020579814910889, + "mean_token_accuracy": 0.8032843962311744, + "num_tokens": 6853454.0, + "step": 1290 + }, + { + "entropy": 0.5963539965450764, + "epoch": 2.7380073800738005, + "grad_norm": 6.59375, + "learning_rate": 1.0675555555555558e-05, + "loss": 0.5908561706542969, + "mean_token_accuracy": 0.8011696465313435, + "num_tokens": 6904473.0, + "step": 1300 + }, + { + "entropy": 0.5899917531758547, + "epoch": 2.7590933052187667, + "grad_norm": 6.6875, + "learning_rate": 1.0586666666666668e-05, + "loss": 0.5744058132171631, + "mean_token_accuracy": 0.8053683824837208, + "num_tokens": 6956756.0, + "step": 1310 + }, + { + "entropy": 0.582448274642229, + "epoch": 2.780179230363732, + "grad_norm": 6.78125, + "learning_rate": 1.0497777777777779e-05, + "loss": 0.5848373889923095, + "mean_token_accuracy": 0.8054254874587059, + "num_tokens": 7008611.0, + "step": 1320 + }, + { + "entropy": 0.5906594522297383, + "epoch": 2.801265155508698, + "grad_norm": 7.75, + "learning_rate": 1.040888888888889e-05, + "loss": 0.6006948471069335, + "mean_token_accuracy": 0.7971956506371498, + "num_tokens": 7062235.0, + "step": 1330 + }, + { + "entropy": 0.5976275954395532, + "epoch": 2.8223510806536636, + "grad_norm": 6.34375, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.5997600078582763, + "mean_token_accuracy": 0.7998531877994537, + "num_tokens": 7115737.0, + "step": 1340 + }, + { + "entropy": 0.6079881630837918, + "epoch": 2.8434370057986293, + "grad_norm": 7.0, + "learning_rate": 1.0231111111111112e-05, + "loss": 0.6024625778198243, + "mean_token_accuracy": 0.7988371469080449, + "num_tokens": 7166870.0, + "step": 1350 + }, + { + "entropy": 0.6036868289113044, + "epoch": 2.864522930943595, + "grad_norm": 6.6875, + "learning_rate": 1.0142222222222222e-05, + "loss": 0.5962271213531494, + "mean_token_accuracy": 0.8011133573949337, + "num_tokens": 7219785.0, + "step": 1360 + }, + { + "entropy": 0.5746289525181055, + "epoch": 2.885608856088561, + "grad_norm": 7.5625, + "learning_rate": 1.0053333333333334e-05, + "loss": 0.5769401073455811, + "mean_token_accuracy": 0.809252617508173, + "num_tokens": 7275411.0, + "step": 1370 + }, + { + "entropy": 0.5846736520528794, + "epoch": 2.9066947812335266, + "grad_norm": 7.125, + "learning_rate": 9.964444444444445e-06, + "loss": 0.5899319171905517, + "mean_token_accuracy": 0.8028473742306232, + "num_tokens": 7326752.0, + "step": 1380 + }, + { + "entropy": 0.5813163112848997, + "epoch": 2.9277807063784924, + "grad_norm": 7.46875, + "learning_rate": 9.875555555555557e-06, + "loss": 0.5799898147583008, + "mean_token_accuracy": 0.8026903085410595, + "num_tokens": 7375608.0, + "step": 1390 + }, + { + "entropy": 0.5724870227277279, + "epoch": 2.948866631523458, + "grad_norm": 6.90625, + "learning_rate": 9.786666666666667e-06, + "loss": 0.5783823490142822, + "mean_token_accuracy": 0.8059644535183906, + "num_tokens": 7430145.0, + "step": 1400 + }, + { + "entropy": 0.5778653729707003, + "epoch": 2.969952556668424, + "grad_norm": 6.84375, + "learning_rate": 9.697777777777778e-06, + "loss": 0.5640594005584717, + "mean_token_accuracy": 0.8100298568606377, + "num_tokens": 7482976.0, + "step": 1410 + }, + { + "entropy": 0.5823766525834799, + "epoch": 2.9910384818133897, + "grad_norm": 6.5625, + "learning_rate": 9.60888888888889e-06, + "loss": 0.5931865692138671, + "mean_token_accuracy": 0.8023995622992516, + "num_tokens": 7533043.0, + "step": 1420 + }, + { + "entropy": 0.5924574036050487, + "epoch": 3.0105429625724827, + "grad_norm": 6.53125, + "learning_rate": 9.52e-06, + "loss": 0.5867043972015381, + "mean_token_accuracy": 0.803342911842707, + "num_tokens": 7581993.0, + "step": 1430 + }, + { + "entropy": 0.579814862459898, + "epoch": 3.0316288877174484, + "grad_norm": 6.34375, + "learning_rate": 9.431111111111112e-06, + "loss": 0.5730488300323486, + "mean_token_accuracy": 0.8101175747811794, + "num_tokens": 7639343.0, + "step": 1440 + }, + { + "entropy": 0.560778708383441, + "epoch": 3.052714812862414, + "grad_norm": 7.40625, + "learning_rate": 9.342222222222223e-06, + "loss": 0.5500640392303466, + "mean_token_accuracy": 0.8135757654905319, + "num_tokens": 7690695.0, + "step": 1450 + }, + { + "entropy": 0.5529176875948906, + "epoch": 3.07380073800738, + "grad_norm": 7.21875, + "learning_rate": 9.253333333333333e-06, + "loss": 0.549665117263794, + "mean_token_accuracy": 0.81509620398283, + "num_tokens": 7746705.0, + "step": 1460 + }, + { + "entropy": 0.5382925808429718, + "epoch": 3.0948866631523457, + "grad_norm": 7.5, + "learning_rate": 9.164444444444446e-06, + "loss": 0.54415283203125, + "mean_token_accuracy": 0.819620954990387, + "num_tokens": 7799255.0, + "step": 1470 + }, + { + "entropy": 0.551479097828269, + "epoch": 3.1159725882973115, + "grad_norm": 7.125, + "learning_rate": 9.075555555555556e-06, + "loss": 0.5520545959472656, + "mean_token_accuracy": 0.8147840730845928, + "num_tokens": 7852753.0, + "step": 1480 + }, + { + "entropy": 0.571834359690547, + "epoch": 3.1370585134422773, + "grad_norm": 7.59375, + "learning_rate": 8.986666666666666e-06, + "loss": 0.5553510189056396, + "mean_token_accuracy": 0.8117231354117393, + "num_tokens": 7904298.0, + "step": 1490 + }, + { + "entropy": 0.5676855646073818, + "epoch": 3.158144438587243, + "grad_norm": 7.75, + "learning_rate": 8.897777777777779e-06, + "loss": 0.5595914840698242, + "mean_token_accuracy": 0.8130396403372288, + "num_tokens": 7956705.0, + "step": 1500 + }, + { + "entropy": 0.5613869089633227, + "epoch": 3.179230363732209, + "grad_norm": 7.78125, + "learning_rate": 8.80888888888889e-06, + "loss": 0.544979190826416, + "mean_token_accuracy": 0.8170888110995292, + "num_tokens": 8008023.0, + "step": 1510 + }, + { + "entropy": 0.570792680233717, + "epoch": 3.2003162888771746, + "grad_norm": 6.75, + "learning_rate": 8.720000000000001e-06, + "loss": 0.560499906539917, + "mean_token_accuracy": 0.8146887816488743, + "num_tokens": 8058344.0, + "step": 1520 + }, + { + "entropy": 0.5498528245836496, + "epoch": 3.2214022140221403, + "grad_norm": 7.15625, + "learning_rate": 8.631111111111112e-06, + "loss": 0.5479342937469482, + "mean_token_accuracy": 0.8137361407279968, + "num_tokens": 8110587.0, + "step": 1530 + }, + { + "entropy": 0.5499754682183265, + "epoch": 3.242488139167106, + "grad_norm": 6.9375, + "learning_rate": 8.542222222222222e-06, + "loss": 0.5445993423461915, + "mean_token_accuracy": 0.8179522223770619, + "num_tokens": 8160497.0, + "step": 1540 + }, + { + "entropy": 0.572133193910122, + "epoch": 3.263574064312072, + "grad_norm": 6.84375, + "learning_rate": 8.453333333333334e-06, + "loss": 0.5740565776824951, + "mean_token_accuracy": 0.8101011097431183, + "num_tokens": 8214878.0, + "step": 1550 + }, + { + "entropy": 0.5666386324912309, + "epoch": 3.2846599894570376, + "grad_norm": 7.59375, + "learning_rate": 8.364444444444445e-06, + "loss": 0.5723002433776856, + "mean_token_accuracy": 0.8091295748949051, + "num_tokens": 8267314.0, + "step": 1560 + }, + { + "entropy": 0.5659929499030113, + "epoch": 3.305745914602003, + "grad_norm": 6.96875, + "learning_rate": 8.275555555555557e-06, + "loss": 0.5623570442199707, + "mean_token_accuracy": 0.8093601904809475, + "num_tokens": 8320145.0, + "step": 1570 + }, + { + "entropy": 0.5665242202579975, + "epoch": 3.3268318397469687, + "grad_norm": 7.0625, + "learning_rate": 8.186666666666667e-06, + "loss": 0.5534502983093261, + "mean_token_accuracy": 0.8129527278244495, + "num_tokens": 8370852.0, + "step": 1580 + }, + { + "entropy": 0.5829939197748899, + "epoch": 3.3479177648919345, + "grad_norm": 7.40625, + "learning_rate": 8.09777777777778e-06, + "loss": 0.5778002738952637, + "mean_token_accuracy": 0.8072332389652729, + "num_tokens": 8424209.0, + "step": 1590 + }, + { + "entropy": 0.5479087311774492, + "epoch": 3.3690036900369003, + "grad_norm": 6.15625, + "learning_rate": 8.00888888888889e-06, + "loss": 0.5359113216400146, + "mean_token_accuracy": 0.819999934732914, + "num_tokens": 8476617.0, + "step": 1600 + }, + { + "entropy": 0.5520685013383627, + "epoch": 3.390089615181866, + "grad_norm": 6.53125, + "learning_rate": 7.92e-06, + "loss": 0.540053415298462, + "mean_token_accuracy": 0.817080944776535, + "num_tokens": 8529074.0, + "step": 1610 + }, + { + "entropy": 0.5533283669501543, + "epoch": 3.411175540326832, + "grad_norm": 7.25, + "learning_rate": 7.831111111111111e-06, + "loss": 0.5461609363555908, + "mean_token_accuracy": 0.8130916491150856, + "num_tokens": 8582416.0, + "step": 1620 + }, + { + "entropy": 0.5606811255216598, + "epoch": 3.4322614654717976, + "grad_norm": 8.25, + "learning_rate": 7.742222222222223e-06, + "loss": 0.565118408203125, + "mean_token_accuracy": 0.813844607770443, + "num_tokens": 8634249.0, + "step": 1630 + }, + { + "entropy": 0.5458527609705925, + "epoch": 3.4533473906167633, + "grad_norm": 7.6875, + "learning_rate": 7.653333333333333e-06, + "loss": 0.5410670757293701, + "mean_token_accuracy": 0.8188620381057262, + "num_tokens": 8685168.0, + "step": 1640 + }, + { + "entropy": 0.5717878874391318, + "epoch": 3.474433315761729, + "grad_norm": 7.6875, + "learning_rate": 7.564444444444446e-06, + "loss": 0.5788788795471191, + "mean_token_accuracy": 0.8078153640031814, + "num_tokens": 8740562.0, + "step": 1650 + }, + { + "entropy": 0.5646973233669996, + "epoch": 3.495519240906695, + "grad_norm": 6.75, + "learning_rate": 7.475555555555556e-06, + "loss": 0.5595187187194824, + "mean_token_accuracy": 0.8135132856667042, + "num_tokens": 8794411.0, + "step": 1660 + }, + { + "entropy": 0.5641634043306112, + "epoch": 3.5166051660516606, + "grad_norm": 6.6875, + "learning_rate": 7.386666666666667e-06, + "loss": 0.5511392116546631, + "mean_token_accuracy": 0.8159678690135479, + "num_tokens": 8844547.0, + "step": 1670 + }, + { + "entropy": 0.5724293068051338, + "epoch": 3.5376910911966264, + "grad_norm": 6.875, + "learning_rate": 7.297777777777778e-06, + "loss": 0.5668179988861084, + "mean_token_accuracy": 0.8106696806848049, + "num_tokens": 8902311.0, + "step": 1680 + }, + { + "entropy": 0.5610478781163692, + "epoch": 3.558777016341592, + "grad_norm": 6.96875, + "learning_rate": 7.20888888888889e-06, + "loss": 0.5528025150299072, + "mean_token_accuracy": 0.8120494581758976, + "num_tokens": 8957569.0, + "step": 1690 + }, + { + "entropy": 0.5370181340724229, + "epoch": 3.5798629414865575, + "grad_norm": 7.28125, + "learning_rate": 7.1200000000000004e-06, + "loss": 0.5477783203125, + "mean_token_accuracy": 0.8162322834134101, + "num_tokens": 9009083.0, + "step": 1700 + }, + { + "entropy": 0.5545388571918011, + "epoch": 3.6009488666315237, + "grad_norm": 6.5, + "learning_rate": 7.031111111111112e-06, + "loss": 0.5430552005767822, + "mean_token_accuracy": 0.8191993653774261, + "num_tokens": 9061980.0, + "step": 1710 + }, + { + "entropy": 0.5662489753216505, + "epoch": 3.622034791776489, + "grad_norm": 7.3125, + "learning_rate": 6.942222222222222e-06, + "loss": 0.5623671531677246, + "mean_token_accuracy": 0.8118648871779441, + "num_tokens": 9117653.0, + "step": 1720 + }, + { + "entropy": 0.5656806524842978, + "epoch": 3.643120716921455, + "grad_norm": 7.40625, + "learning_rate": 6.853333333333334e-06, + "loss": 0.5693060398101807, + "mean_token_accuracy": 0.8121738277375699, + "num_tokens": 9168884.0, + "step": 1730 + }, + { + "entropy": 0.5669469233602286, + "epoch": 3.6642066420664205, + "grad_norm": 6.875, + "learning_rate": 6.764444444444445e-06, + "loss": 0.5708917617797852, + "mean_token_accuracy": 0.8130646869540215, + "num_tokens": 9221794.0, + "step": 1740 + }, + { + "entropy": 0.5680123705416917, + "epoch": 3.6852925672113863, + "grad_norm": 7.3125, + "learning_rate": 6.675555555555556e-06, + "loss": 0.565749979019165, + "mean_token_accuracy": 0.8114100478589534, + "num_tokens": 9277320.0, + "step": 1750 + }, + { + "entropy": 0.5670568168163299, + "epoch": 3.706378492356352, + "grad_norm": 6.8125, + "learning_rate": 6.5866666666666666e-06, + "loss": 0.5531346797943115, + "mean_token_accuracy": 0.815721620619297, + "num_tokens": 9328654.0, + "step": 1760 + }, + { + "entropy": 0.5794487085193396, + "epoch": 3.727464417501318, + "grad_norm": 7.25, + "learning_rate": 6.497777777777779e-06, + "loss": 0.5695443630218506, + "mean_token_accuracy": 0.8125023365020752, + "num_tokens": 9382781.0, + "step": 1770 + }, + { + "entropy": 0.5529596284031868, + "epoch": 3.7485503426462836, + "grad_norm": 7.21875, + "learning_rate": 6.408888888888889e-06, + "loss": 0.5538197040557862, + "mean_token_accuracy": 0.8132442943751812, + "num_tokens": 9434446.0, + "step": 1780 + }, + { + "entropy": 0.554557716101408, + "epoch": 3.7696362677912494, + "grad_norm": 7.34375, + "learning_rate": 6.3200000000000005e-06, + "loss": 0.5493638515472412, + "mean_token_accuracy": 0.8154775679111481, + "num_tokens": 9488504.0, + "step": 1790 + }, + { + "entropy": 0.5696977652609349, + "epoch": 3.790722192936215, + "grad_norm": 6.28125, + "learning_rate": 6.231111111111111e-06, + "loss": 0.5688616275787354, + "mean_token_accuracy": 0.8128006041049958, + "num_tokens": 9545943.0, + "step": 1800 + }, + { + "entropy": 0.5715777482837439, + "epoch": 3.811808118081181, + "grad_norm": 6.78125, + "learning_rate": 6.142222222222223e-06, + "loss": 0.5713305473327637, + "mean_token_accuracy": 0.811994157731533, + "num_tokens": 9597820.0, + "step": 1810 + }, + { + "entropy": 0.5638129018247128, + "epoch": 3.8328940432261467, + "grad_norm": 7.25, + "learning_rate": 6.0533333333333335e-06, + "loss": 0.5548198699951172, + "mean_token_accuracy": 0.8136275447905064, + "num_tokens": 9651480.0, + "step": 1820 + }, + { + "entropy": 0.5706254750490188, + "epoch": 3.853979968371112, + "grad_norm": 6.8125, + "learning_rate": 5.964444444444445e-06, + "loss": 0.5752524852752685, + "mean_token_accuracy": 0.8053847931325435, + "num_tokens": 9706114.0, + "step": 1830 + }, + { + "entropy": 0.5612337995320559, + "epoch": 3.875065893516078, + "grad_norm": 7.0, + "learning_rate": 5.875555555555556e-06, + "loss": 0.5460340499877929, + "mean_token_accuracy": 0.8159381657838821, + "num_tokens": 9760141.0, + "step": 1840 + }, + { + "entropy": 0.5514645285904407, + "epoch": 3.8961518186610435, + "grad_norm": 8.0, + "learning_rate": 5.7866666666666674e-06, + "loss": 0.5530869960784912, + "mean_token_accuracy": 0.8169759206473828, + "num_tokens": 9812466.0, + "step": 1850 + }, + { + "entropy": 0.5574501186609269, + "epoch": 3.9172377438060098, + "grad_norm": 7.5, + "learning_rate": 5.697777777777779e-06, + "loss": 0.5557053089141846, + "mean_token_accuracy": 0.8142473429441452, + "num_tokens": 9863547.0, + "step": 1860 + }, + { + "entropy": 0.560401076450944, + "epoch": 3.938323668950975, + "grad_norm": 6.53125, + "learning_rate": 5.608888888888889e-06, + "loss": 0.5598348140716553, + "mean_token_accuracy": 0.810575645416975, + "num_tokens": 9917210.0, + "step": 1870 + }, + { + "entropy": 0.5611382886767388, + "epoch": 3.959409594095941, + "grad_norm": 6.875, + "learning_rate": 5.5200000000000005e-06, + "loss": 0.5510398387908936, + "mean_token_accuracy": 0.8139252245426178, + "num_tokens": 9971753.0, + "step": 1880 + }, + { + "entropy": 0.571844469755888, + "epoch": 3.9804955192409066, + "grad_norm": 7.15625, + "learning_rate": 5.431111111111112e-06, + "loss": 0.5825337409973145, + "mean_token_accuracy": 0.8051016978919506, + "num_tokens": 10024121.0, + "step": 1890 + }, + { + "entropy": 0.5610769511880102, + "epoch": 4.0, + "grad_norm": 15.75, + "learning_rate": 5.342222222222223e-06, + "loss": 0.5544355392456055, + "mean_token_accuracy": 0.8145432029221509, + "num_tokens": 10072608.0, + "step": 1900 + }, + { + "entropy": 0.5582417894154788, + "epoch": 4.021085925144965, + "grad_norm": 6.5625, + "learning_rate": 5.2533333333333336e-06, + "loss": 0.5517781734466553, + "mean_token_accuracy": 0.8166630238294601, + "num_tokens": 10127149.0, + "step": 1910 + }, + { + "entropy": 0.5561550367623568, + "epoch": 4.0421718502899315, + "grad_norm": 6.5, + "learning_rate": 5.164444444444445e-06, + "loss": 0.5391493797302246, + "mean_token_accuracy": 0.8172842212021351, + "num_tokens": 10178921.0, + "step": 1920 + }, + { + "entropy": 0.5536927301436663, + "epoch": 4.063257775434897, + "grad_norm": 6.8125, + "learning_rate": 5.075555555555556e-06, + "loss": 0.5435996532440186, + "mean_token_accuracy": 0.8193235926330089, + "num_tokens": 10230953.0, + "step": 1930 + }, + { + "entropy": 0.5522860966622829, + "epoch": 4.084343700579863, + "grad_norm": 6.65625, + "learning_rate": 4.986666666666667e-06, + "loss": 0.5417564868927002, + "mean_token_accuracy": 0.8215342923998833, + "num_tokens": 10285145.0, + "step": 1940 + }, + { + "entropy": 0.558988232910633, + "epoch": 4.105429625724828, + "grad_norm": 8.25, + "learning_rate": 4.897777777777778e-06, + "loss": 0.5513458251953125, + "mean_token_accuracy": 0.8168663889169693, + "num_tokens": 10340678.0, + "step": 1950 + }, + { + "entropy": 0.5479944169521331, + "epoch": 4.126515550869795, + "grad_norm": 7.6875, + "learning_rate": 4.808888888888889e-06, + "loss": 0.5411903858184814, + "mean_token_accuracy": 0.8173033416271209, + "num_tokens": 10396265.0, + "step": 1960 + }, + { + "entropy": 0.5479635439813137, + "epoch": 4.14760147601476, + "grad_norm": 6.6875, + "learning_rate": 4.7200000000000005e-06, + "loss": 0.5325856685638428, + "mean_token_accuracy": 0.8196660950779915, + "num_tokens": 10448162.0, + "step": 1970 + }, + { + "entropy": 0.5631415072828532, + "epoch": 4.168687401159726, + "grad_norm": 6.96875, + "learning_rate": 4.631111111111111e-06, + "loss": 0.5563766956329346, + "mean_token_accuracy": 0.8139985457062722, + "num_tokens": 10500711.0, + "step": 1980 + }, + { + "entropy": 0.5530602999031544, + "epoch": 4.1897733263046915, + "grad_norm": 6.78125, + "learning_rate": 4.542222222222223e-06, + "loss": 0.5429882049560547, + "mean_token_accuracy": 0.815962927043438, + "num_tokens": 10554919.0, + "step": 1990 + }, + { + "entropy": 0.5508173421025276, + "epoch": 4.210859251449658, + "grad_norm": 6.90625, + "learning_rate": 4.453333333333334e-06, + "loss": 0.5390869140625, + "mean_token_accuracy": 0.8184089511632919, + "num_tokens": 10606269.0, + "step": 2000 + }, + { + "entropy": 0.5496918559074402, + "epoch": 4.231945176594623, + "grad_norm": 7.0, + "learning_rate": 4.364444444444445e-06, + "loss": 0.5486071109771729, + "mean_token_accuracy": 0.8196936115622521, + "num_tokens": 10660269.0, + "step": 2010 + }, + { + "entropy": 0.5489282257854938, + "epoch": 4.253031101739589, + "grad_norm": 7.4375, + "learning_rate": 4.275555555555556e-06, + "loss": 0.5514675617218018, + "mean_token_accuracy": 0.8145210847258568, + "num_tokens": 10711759.0, + "step": 2020 + }, + { + "entropy": 0.5529059153050184, + "epoch": 4.2741170268845545, + "grad_norm": 6.78125, + "learning_rate": 4.1866666666666675e-06, + "loss": 0.5444664001464844, + "mean_token_accuracy": 0.8184945359826088, + "num_tokens": 10763911.0, + "step": 2030 + }, + { + "entropy": 0.551506045460701, + "epoch": 4.29520295202952, + "grad_norm": 7.34375, + "learning_rate": 4.097777777777778e-06, + "loss": 0.548325777053833, + "mean_token_accuracy": 0.8162619814276695, + "num_tokens": 10817880.0, + "step": 2040 + }, + { + "entropy": 0.5396009232848883, + "epoch": 4.316288877174486, + "grad_norm": 6.8125, + "learning_rate": 4.008888888888889e-06, + "loss": 0.5268721103668212, + "mean_token_accuracy": 0.8218203999102116, + "num_tokens": 10870365.0, + "step": 2050 + }, + { + "entropy": 0.5537472911179065, + "epoch": 4.337374802319451, + "grad_norm": 6.65625, + "learning_rate": 3.920000000000001e-06, + "loss": 0.5522951126098633, + "mean_token_accuracy": 0.8159066334366798, + "num_tokens": 10922363.0, + "step": 2060 + }, + { + "entropy": 0.5330416887998581, + "epoch": 4.358460727464418, + "grad_norm": 7.21875, + "learning_rate": 3.831111111111112e-06, + "loss": 0.5207313537597656, + "mean_token_accuracy": 0.8252225182950497, + "num_tokens": 10972251.0, + "step": 2070 + }, + { + "entropy": 0.5454271405935287, + "epoch": 4.379546652609383, + "grad_norm": 7.125, + "learning_rate": 3.7422222222222228e-06, + "loss": 0.5341221809387207, + "mean_token_accuracy": 0.8211991436779499, + "num_tokens": 11025983.0, + "step": 2080 + }, + { + "entropy": 0.5519603937864304, + "epoch": 4.400632577754349, + "grad_norm": 7.5625, + "learning_rate": 3.6533333333333336e-06, + "loss": 0.5493678569793701, + "mean_token_accuracy": 0.8145092785358429, + "num_tokens": 11081248.0, + "step": 2090 + }, + { + "entropy": 0.5668473180383444, + "epoch": 4.4217185028993145, + "grad_norm": 7.21875, + "learning_rate": 3.564444444444445e-06, + "loss": 0.5566862106323243, + "mean_token_accuracy": 0.813294805586338, + "num_tokens": 11135369.0, + "step": 2100 + }, + { + "entropy": 0.5516544353216887, + "epoch": 4.442804428044281, + "grad_norm": 7.21875, + "learning_rate": 3.475555555555556e-06, + "loss": 0.5478980541229248, + "mean_token_accuracy": 0.8169082552194595, + "num_tokens": 11190765.0, + "step": 2110 + }, + { + "entropy": 0.5469749603420496, + "epoch": 4.463890353189246, + "grad_norm": 8.1875, + "learning_rate": 3.386666666666667e-06, + "loss": 0.5484558582305908, + "mean_token_accuracy": 0.8153667166829109, + "num_tokens": 11242129.0, + "step": 2120 + }, + { + "entropy": 0.5624320391565562, + "epoch": 4.484976278334212, + "grad_norm": 7.15625, + "learning_rate": 3.297777777777778e-06, + "loss": 0.5431473255157471, + "mean_token_accuracy": 0.8167137958109378, + "num_tokens": 11296409.0, + "step": 2130 + }, + { + "entropy": 0.5575876057147979, + "epoch": 4.5060622034791775, + "grad_norm": 6.625, + "learning_rate": 3.2088888888888893e-06, + "loss": 0.546560525894165, + "mean_token_accuracy": 0.8190927796065808, + "num_tokens": 11351788.0, + "step": 2140 + }, + { + "entropy": 0.5531029619276524, + "epoch": 4.527148128624144, + "grad_norm": 6.6875, + "learning_rate": 3.12e-06, + "loss": 0.5480860233306885, + "mean_token_accuracy": 0.8185123346745968, + "num_tokens": 11405274.0, + "step": 2150 + }, + { + "entropy": 0.5629217986017465, + "epoch": 4.548234053769109, + "grad_norm": 7.40625, + "learning_rate": 3.0311111111111115e-06, + "loss": 0.5423421859741211, + "mean_token_accuracy": 0.8194163963198662, + "num_tokens": 11459305.0, + "step": 2160 + }, + { + "entropy": 0.5574934646487236, + "epoch": 4.569319978914075, + "grad_norm": 7.40625, + "learning_rate": 2.9422222222222224e-06, + "loss": 0.5479726791381836, + "mean_token_accuracy": 0.8106288813054562, + "num_tokens": 11512317.0, + "step": 2170 + }, + { + "entropy": 0.5610745966434478, + "epoch": 4.590405904059041, + "grad_norm": 7.96875, + "learning_rate": 2.8533333333333337e-06, + "loss": 0.5556921005249024, + "mean_token_accuracy": 0.8137760147452354, + "num_tokens": 11566146.0, + "step": 2180 + }, + { + "entropy": 0.5264041472226382, + "epoch": 4.611491829204006, + "grad_norm": 6.71875, + "learning_rate": 2.7644444444444446e-06, + "loss": 0.535123348236084, + "mean_token_accuracy": 0.8231086760759354, + "num_tokens": 11619757.0, + "step": 2190 + }, + { + "entropy": 0.5513654734939337, + "epoch": 4.632577754348972, + "grad_norm": 7.28125, + "learning_rate": 2.675555555555556e-06, + "loss": 0.5477256774902344, + "mean_token_accuracy": 0.8165041409432888, + "num_tokens": 11673528.0, + "step": 2200 + }, + { + "entropy": 0.5440514117479325, + "epoch": 4.653663679493937, + "grad_norm": 7.15625, + "learning_rate": 2.5866666666666667e-06, + "loss": 0.5353331565856934, + "mean_token_accuracy": 0.8190776027739048, + "num_tokens": 11728320.0, + "step": 2210 + }, + { + "entropy": 0.5514211151748896, + "epoch": 4.674749604638904, + "grad_norm": 6.6875, + "learning_rate": 2.497777777777778e-06, + "loss": 0.5415604114532471, + "mean_token_accuracy": 0.8205962501466274, + "num_tokens": 11780028.0, + "step": 2220 + }, + { + "entropy": 0.5382947374135256, + "epoch": 4.695835529783869, + "grad_norm": 8.375, + "learning_rate": 2.4088888888888894e-06, + "loss": 0.5256577491760254, + "mean_token_accuracy": 0.8235763169825077, + "num_tokens": 11830895.0, + "step": 2230 + }, + { + "entropy": 0.54562176913023, + "epoch": 4.716921454928835, + "grad_norm": 7.1875, + "learning_rate": 2.3200000000000002e-06, + "loss": 0.5267295837402344, + "mean_token_accuracy": 0.8207360960543155, + "num_tokens": 11882155.0, + "step": 2240 + }, + { + "entropy": 0.5469693087041378, + "epoch": 4.7380073800738005, + "grad_norm": 7.28125, + "learning_rate": 2.2311111111111115e-06, + "loss": 0.5502583026885987, + "mean_token_accuracy": 0.812764473259449, + "num_tokens": 11936554.0, + "step": 2250 + }, + { + "entropy": 0.5447085045278073, + "epoch": 4.759093305218767, + "grad_norm": 6.5, + "learning_rate": 2.1422222222222224e-06, + "loss": 0.5389336585998535, + "mean_token_accuracy": 0.820573166757822, + "num_tokens": 11988156.0, + "step": 2260 + }, + { + "entropy": 0.5513703927397728, + "epoch": 4.780179230363732, + "grad_norm": 7.59375, + "learning_rate": 2.0533333333333337e-06, + "loss": 0.5426520347595215, + "mean_token_accuracy": 0.8161885485053062, + "num_tokens": 12041543.0, + "step": 2270 + }, + { + "entropy": 0.5453945960849523, + "epoch": 4.801265155508698, + "grad_norm": 8.3125, + "learning_rate": 1.9644444444444446e-06, + "loss": 0.5514724731445313, + "mean_token_accuracy": 0.8171669006347656, + "num_tokens": 12092858.0, + "step": 2280 + }, + { + "entropy": 0.5482615809887648, + "epoch": 4.822351080653664, + "grad_norm": 8.1875, + "learning_rate": 1.8755555555555557e-06, + "loss": 0.5525861263275147, + "mean_token_accuracy": 0.8130425401031971, + "num_tokens": 12144862.0, + "step": 2290 + }, + { + "entropy": 0.5636871963739395, + "epoch": 4.84343700579863, + "grad_norm": 7.34375, + "learning_rate": 1.7866666666666668e-06, + "loss": 0.5567370414733886, + "mean_token_accuracy": 0.8114767424762249, + "num_tokens": 12197932.0, + "step": 2300 + }, + { + "entropy": 0.5445570610463619, + "epoch": 4.864522930943595, + "grad_norm": 6.875, + "learning_rate": 1.6977777777777779e-06, + "loss": 0.5326248645782471, + "mean_token_accuracy": 0.8205311246216297, + "num_tokens": 12249536.0, + "step": 2310 + }, + { + "entropy": 0.567267632111907, + "epoch": 4.885608856088561, + "grad_norm": 7.125, + "learning_rate": 1.608888888888889e-06, + "loss": 0.5614665031433106, + "mean_token_accuracy": 0.8123429246246815, + "num_tokens": 12302328.0, + "step": 2320 + }, + { + "entropy": 0.5418668888509274, + "epoch": 4.906694781233527, + "grad_norm": 7.09375, + "learning_rate": 1.52e-06, + "loss": 0.5339171409606933, + "mean_token_accuracy": 0.821372539550066, + "num_tokens": 12354907.0, + "step": 2330 + }, + { + "entropy": 0.5500333663076162, + "epoch": 4.927780706378492, + "grad_norm": 7.4375, + "learning_rate": 1.4311111111111111e-06, + "loss": 0.5362411499023437, + "mean_token_accuracy": 0.8219335168600083, + "num_tokens": 12409864.0, + "step": 2340 + }, + { + "entropy": 0.551620377600193, + "epoch": 4.948866631523458, + "grad_norm": 7.5, + "learning_rate": 1.3422222222222222e-06, + "loss": 0.5413876056671143, + "mean_token_accuracy": 0.8180868111550808, + "num_tokens": 12463400.0, + "step": 2350 + }, + { + "entropy": 0.5467831913381815, + "epoch": 4.9699525566684235, + "grad_norm": 7.0, + "learning_rate": 1.2533333333333333e-06, + "loss": 0.5441878318786622, + "mean_token_accuracy": 0.8187509059906006, + "num_tokens": 12515674.0, + "step": 2360 + }, + { + "entropy": 0.5409926638007164, + "epoch": 4.99103848181339, + "grad_norm": 6.65625, + "learning_rate": 1.1644444444444446e-06, + "loss": 0.5396465301513672, + "mean_token_accuracy": 0.8193774409592152, + "num_tokens": 12568384.0, + "step": 2370 + } + ], + "logging_steps": 10, + "max_steps": 2500, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2884493756162662e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2375/training_args.bin b/checkpoint-2375/training_args.bin new file mode 100644 index 0000000..f138a50 --- /dev/null +++ b/checkpoint-2375/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7103b0ce3340ea41aca3b7441d01d7c38791e3d3ce8c9b28c906c9b1d49bbee9 +size 5713 diff --git a/checkpoint-2500/chat_template.jinja b/checkpoint-2500/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-2500/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-2500/config.json b/checkpoint-2500/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-2500/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-2500/generation_config.json b/checkpoint-2500/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-2500/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-2500/model.safetensors b/checkpoint-2500/model.safetensors new file mode 100644 index 0000000..d426085 --- /dev/null +++ b/checkpoint-2500/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9d2b9b9a2cc160bae62741936f9ecd9d8068706b9bada8256cffe2a4853804 +size 3441185608 diff --git a/checkpoint-2500/optimizer.pt b/checkpoint-2500/optimizer.pt new file mode 100644 index 0000000..ef916b8 --- /dev/null +++ b/checkpoint-2500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:244f607b2565089bd7e7020696d29e3bf0e6424c6f35a2fda54ec76d01ec59a9 +size 6882572207 diff --git a/checkpoint-2500/rng_state.pth b/checkpoint-2500/rng_state.pth new file mode 100644 index 0000000..3ab6836 --- /dev/null +++ b/checkpoint-2500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01f9a0f7843a37be87edd23f4e88aa93b38b95cc2c07503eeb1cf2e4632453a2 +size 14645 diff --git a/checkpoint-2500/scheduler.pt b/checkpoint-2500/scheduler.pt new file mode 100644 index 0000000..4f428a2 --- /dev/null +++ b/checkpoint-2500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7452baf421186c7ecb17aca0bed967e1dda02ffe91de0d6aa55e68a1398920e2 +size 1465 diff --git a/checkpoint-2500/tokenizer.json b/checkpoint-2500/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-2500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-2500/tokenizer_config.json b/checkpoint-2500/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-2500/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-2500/trainer_state.json b/checkpoint-2500/trainer_state.json new file mode 100644 index 0000000..cbc570c --- /dev/null +++ b/checkpoint-2500/trainer_state.json @@ -0,0 +1,2534 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.263574064312071, + "eval_steps": 500, + "global_step": 2500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.8691443920135498, + "epoch": 0.021085925144965736, + "grad_norm": 47.5, + "learning_rate": 7.2e-07, + "loss": 2.1268569946289064, + "mean_token_accuracy": 0.5856592752039432, + "num_tokens": 53385.0, + "step": 10 + }, + { + "entropy": 0.9018128983676433, + "epoch": 0.04217185028993147, + "grad_norm": 49.25, + "learning_rate": 1.52e-06, + "loss": 2.1235822677612304, + "mean_token_accuracy": 0.5826416034251451, + "num_tokens": 108949.0, + "step": 20 + }, + { + "entropy": 0.9313402399420738, + "epoch": 0.0632577754348972, + "grad_norm": 38.25, + "learning_rate": 2.3200000000000002e-06, + "loss": 1.9997358322143555, + "mean_token_accuracy": 0.5817345686256885, + "num_tokens": 162034.0, + "step": 30 + }, + { + "entropy": 0.9819381944835186, + "epoch": 0.08434370057986294, + "grad_norm": 28.25, + "learning_rate": 3.12e-06, + "loss": 1.8760208129882812, + "mean_token_accuracy": 0.5874959267675877, + "num_tokens": 214165.0, + "step": 40 + }, + { + "entropy": 1.0624087303876877, + "epoch": 0.10542962572482868, + "grad_norm": 14.1875, + "learning_rate": 3.920000000000001e-06, + "loss": 1.6169570922851562, + "mean_token_accuracy": 0.601640059798956, + "num_tokens": 265527.0, + "step": 50 + }, + { + "entropy": 1.1721510097384453, + "epoch": 0.1265155508697944, + "grad_norm": 11.3125, + "learning_rate": 4.7200000000000005e-06, + "loss": 1.4622878074645995, + "mean_token_accuracy": 0.6209174185991287, + "num_tokens": 319341.0, + "step": 60 + }, + { + "entropy": 1.2009219758212566, + "epoch": 0.14760147601476015, + "grad_norm": 9.0625, + "learning_rate": 5.5200000000000005e-06, + "loss": 1.2933337211608886, + "mean_token_accuracy": 0.6467074818909169, + "num_tokens": 373709.0, + "step": 70 + }, + { + "entropy": 1.1439887225627898, + "epoch": 0.16868740115972589, + "grad_norm": 9.3125, + "learning_rate": 6.3200000000000005e-06, + "loss": 1.2084498405456543, + "mean_token_accuracy": 0.6660686552524566, + "num_tokens": 425923.0, + "step": 80 + }, + { + "entropy": 1.0713621236383915, + "epoch": 0.18977332630469163, + "grad_norm": 7.84375, + "learning_rate": 7.1200000000000004e-06, + "loss": 1.1175435066223145, + "mean_token_accuracy": 0.6812180563807487, + "num_tokens": 480012.0, + "step": 90 + }, + { + "entropy": 1.0369222231209279, + "epoch": 0.21085925144965736, + "grad_norm": 7.09375, + "learning_rate": 7.92e-06, + "loss": 1.062850570678711, + "mean_token_accuracy": 0.6899570092558861, + "num_tokens": 533890.0, + "step": 100 + }, + { + "entropy": 0.9894454136490822, + "epoch": 0.23194517659462308, + "grad_norm": 7.75, + "learning_rate": 8.720000000000001e-06, + "loss": 1.0150799751281738, + "mean_token_accuracy": 0.7009239844977856, + "num_tokens": 590654.0, + "step": 110 + }, + { + "entropy": 0.9410129584372043, + "epoch": 0.2530311017395888, + "grad_norm": 7.28125, + "learning_rate": 9.52e-06, + "loss": 0.9810515403747558, + "mean_token_accuracy": 0.709618617594242, + "num_tokens": 643080.0, + "step": 120 + }, + { + "entropy": 0.9265115469694137, + "epoch": 0.2741170268845546, + "grad_norm": 7.46875, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.9485825538635254, + "mean_token_accuracy": 0.7162063598632813, + "num_tokens": 697254.0, + "step": 130 + }, + { + "entropy": 0.9009825974702835, + "epoch": 0.2952029520295203, + "grad_norm": 6.96875, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.9119878768920898, + "mean_token_accuracy": 0.7238413639366627, + "num_tokens": 751774.0, + "step": 140 + }, + { + "entropy": 0.9040703341364861, + "epoch": 0.316288877174486, + "grad_norm": 7.21875, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.9283308029174805, + "mean_token_accuracy": 0.7166316144168376, + "num_tokens": 808927.0, + "step": 150 + }, + { + "entropy": 0.8878034949302673, + "epoch": 0.33737480231945177, + "grad_norm": 7.375, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.9068665504455566, + "mean_token_accuracy": 0.7254431843757629, + "num_tokens": 858559.0, + "step": 160 + }, + { + "entropy": 0.887480016052723, + "epoch": 0.3584607274644175, + "grad_norm": 7.65625, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.8971159934997559, + "mean_token_accuracy": 0.7258390121161937, + "num_tokens": 912285.0, + "step": 170 + }, + { + "entropy": 0.8527634374797344, + "epoch": 0.37954665260938325, + "grad_norm": 7.6875, + "learning_rate": 1.432e-05, + "loss": 0.8586221694946289, + "mean_token_accuracy": 0.7359919100999832, + "num_tokens": 969385.0, + "step": 180 + }, + { + "entropy": 0.8445627830922604, + "epoch": 0.40063257775434896, + "grad_norm": 7.03125, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.855714225769043, + "mean_token_accuracy": 0.7341729424893856, + "num_tokens": 1026106.0, + "step": 190 + }, + { + "entropy": 0.841130618005991, + "epoch": 0.42171850289931473, + "grad_norm": 7.21875, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.8620488166809082, + "mean_token_accuracy": 0.7361006513237953, + "num_tokens": 1079743.0, + "step": 200 + }, + { + "entropy": 0.831258487701416, + "epoch": 0.44280442804428044, + "grad_norm": 7.0, + "learning_rate": 1.672e-05, + "loss": 0.8441590309143067, + "mean_token_accuracy": 0.7379129216074943, + "num_tokens": 1131453.0, + "step": 210 + }, + { + "entropy": 0.79731200709939, + "epoch": 0.46389035318924615, + "grad_norm": 7.875, + "learning_rate": 1.752e-05, + "loss": 0.8135190963745117, + "mean_token_accuracy": 0.7497627973556519, + "num_tokens": 1182486.0, + "step": 220 + }, + { + "entropy": 0.8269097700715065, + "epoch": 0.4849762783342119, + "grad_norm": 7.5625, + "learning_rate": 1.832e-05, + "loss": 0.8399410247802734, + "mean_token_accuracy": 0.7389348305761814, + "num_tokens": 1238136.0, + "step": 230 + }, + { + "entropy": 0.8093138128519058, + "epoch": 0.5060622034791776, + "grad_norm": 7.40625, + "learning_rate": 1.912e-05, + "loss": 0.810025691986084, + "mean_token_accuracy": 0.7460841409862041, + "num_tokens": 1294939.0, + "step": 240 + }, + { + "entropy": 0.8069386176764966, + "epoch": 0.5271481286241434, + "grad_norm": 6.84375, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8215888023376465, + "mean_token_accuracy": 0.7438369721174241, + "num_tokens": 1346462.0, + "step": 250 + }, + { + "entropy": 0.8088659413158894, + "epoch": 0.5482340537691092, + "grad_norm": 6.75, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8201449394226075, + "mean_token_accuracy": 0.7393839374184609, + "num_tokens": 1397444.0, + "step": 260 + }, + { + "entropy": 0.7969542622566224, + "epoch": 0.5693199789140748, + "grad_norm": 6.59375, + "learning_rate": 1.9831111111111113e-05, + "loss": 0.8009759902954101, + "mean_token_accuracy": 0.7512488305568695, + "num_tokens": 1453900.0, + "step": 270 + }, + { + "entropy": 0.780416651815176, + "epoch": 0.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.9742222222222223e-05, + "loss": 0.7874263763427735, + "mean_token_accuracy": 0.7526524804532528, + "num_tokens": 1505529.0, + "step": 280 + }, + { + "entropy": 0.7752278268337249, + "epoch": 0.6114918292040064, + "grad_norm": 6.875, + "learning_rate": 1.9653333333333334e-05, + "loss": 0.8075678825378418, + "mean_token_accuracy": 0.746569037437439, + "num_tokens": 1562473.0, + "step": 290 + }, + { + "entropy": 0.7895354829728604, + "epoch": 0.632577754348972, + "grad_norm": 6.6875, + "learning_rate": 1.9564444444444444e-05, + "loss": 0.7880948543548584, + "mean_token_accuracy": 0.7494045957922936, + "num_tokens": 1613320.0, + "step": 300 + }, + { + "entropy": 0.7653828673064709, + "epoch": 0.6536636794939378, + "grad_norm": 6.78125, + "learning_rate": 1.9475555555555558e-05, + "loss": 0.7780828475952148, + "mean_token_accuracy": 0.7556171745061875, + "num_tokens": 1662847.0, + "step": 310 + }, + { + "entropy": 0.7652887552976608, + "epoch": 0.6747496046389035, + "grad_norm": 6.375, + "learning_rate": 1.938666666666667e-05, + "loss": 0.7539787292480469, + "mean_token_accuracy": 0.7582654722034932, + "num_tokens": 1718553.0, + "step": 320 + }, + { + "entropy": 0.7076033115386963, + "epoch": 0.6958355297838693, + "grad_norm": 7.15625, + "learning_rate": 1.929777777777778e-05, + "loss": 0.7396605491638184, + "mean_token_accuracy": 0.7659220136702061, + "num_tokens": 1771024.0, + "step": 330 + }, + { + "entropy": 0.7509547933936119, + "epoch": 0.716921454928835, + "grad_norm": 6.15625, + "learning_rate": 1.920888888888889e-05, + "loss": 0.7459388256072998, + "mean_token_accuracy": 0.7644543245434761, + "num_tokens": 1821303.0, + "step": 340 + }, + { + "entropy": 0.7591827407479286, + "epoch": 0.7380073800738007, + "grad_norm": 7.03125, + "learning_rate": 1.912e-05, + "loss": 0.7627717971801757, + "mean_token_accuracy": 0.7567056350409984, + "num_tokens": 1868674.0, + "step": 350 + }, + { + "entropy": 0.7316406652331352, + "epoch": 0.7590933052187665, + "grad_norm": 6.59375, + "learning_rate": 1.9031111111111114e-05, + "loss": 0.7425707340240478, + "mean_token_accuracy": 0.7653280735015869, + "num_tokens": 1924619.0, + "step": 360 + }, + { + "entropy": 0.7539067789912224, + "epoch": 0.7801792303637322, + "grad_norm": 7.0625, + "learning_rate": 1.8942222222222224e-05, + "loss": 0.7716987609863282, + "mean_token_accuracy": 0.7524559155106545, + "num_tokens": 1978795.0, + "step": 370 + }, + { + "entropy": 0.7687754213809967, + "epoch": 0.8012651555086979, + "grad_norm": 7.1875, + "learning_rate": 1.8853333333333335e-05, + "loss": 0.7734373569488525, + "mean_token_accuracy": 0.7497503250837326, + "num_tokens": 2029994.0, + "step": 380 + }, + { + "entropy": 0.7508607231080532, + "epoch": 0.8223510806536637, + "grad_norm": 7.09375, + "learning_rate": 1.8764444444444445e-05, + "loss": 0.7575721740722656, + "mean_token_accuracy": 0.7572125047445297, + "num_tokens": 2082653.0, + "step": 390 + }, + { + "entropy": 0.7252861328423024, + "epoch": 0.8434370057986295, + "grad_norm": 6.5625, + "learning_rate": 1.867555555555556e-05, + "loss": 0.7381186485290527, + "mean_token_accuracy": 0.7612978041172027, + "num_tokens": 2135276.0, + "step": 400 + }, + { + "entropy": 0.7339485622942448, + "epoch": 0.8645229309435951, + "grad_norm": 6.625, + "learning_rate": 1.858666666666667e-05, + "loss": 0.7348906993865967, + "mean_token_accuracy": 0.7663561321794987, + "num_tokens": 2187466.0, + "step": 410 + }, + { + "entropy": 0.7272624362260103, + "epoch": 0.8856088560885609, + "grad_norm": 6.90625, + "learning_rate": 1.849777777777778e-05, + "loss": 0.749025821685791, + "mean_token_accuracy": 0.7617379955947399, + "num_tokens": 2237434.0, + "step": 420 + }, + { + "entropy": 0.7477380312979222, + "epoch": 0.9066947812335266, + "grad_norm": 6.34375, + "learning_rate": 1.840888888888889e-05, + "loss": 0.7498275756835937, + "mean_token_accuracy": 0.7604753144085408, + "num_tokens": 2288985.0, + "step": 430 + }, + { + "entropy": 0.7346535548567772, + "epoch": 0.9277807063784923, + "grad_norm": 6.9375, + "learning_rate": 1.832e-05, + "loss": 0.739972448348999, + "mean_token_accuracy": 0.7658773727715016, + "num_tokens": 2341187.0, + "step": 440 + }, + { + "entropy": 0.7167556572705507, + "epoch": 0.9488666315234581, + "grad_norm": 6.4375, + "learning_rate": 1.823111111111111e-05, + "loss": 0.7350784778594971, + "mean_token_accuracy": 0.7644179269671441, + "num_tokens": 2393385.0, + "step": 450 + }, + { + "entropy": 0.7537519961595536, + "epoch": 0.9699525566684238, + "grad_norm": 7.28125, + "learning_rate": 1.814222222222222e-05, + "loss": 0.757652759552002, + "mean_token_accuracy": 0.7587068513035774, + "num_tokens": 2442248.0, + "step": 460 + }, + { + "entropy": 0.72038439437747, + "epoch": 0.9910384818133896, + "grad_norm": 7.40625, + "learning_rate": 1.8053333333333332e-05, + "loss": 0.7361824035644531, + "mean_token_accuracy": 0.7609050884842873, + "num_tokens": 2498394.0, + "step": 470 + }, + { + "entropy": 0.735570513718837, + "epoch": 1.0105429625724829, + "grad_norm": 6.875, + "learning_rate": 1.7964444444444446e-05, + "loss": 0.730868148803711, + "mean_token_accuracy": 0.7710521068121936, + "num_tokens": 2545422.0, + "step": 480 + }, + { + "entropy": 0.6775223582983017, + "epoch": 1.0316288877174487, + "grad_norm": 7.125, + "learning_rate": 1.7875555555555556e-05, + "loss": 0.6740085124969483, + "mean_token_accuracy": 0.7795436836779117, + "num_tokens": 2597144.0, + "step": 490 + }, + { + "entropy": 0.6628748029470444, + "epoch": 1.0527148128624144, + "grad_norm": 6.84375, + "learning_rate": 1.7786666666666667e-05, + "loss": 0.6774313926696778, + "mean_token_accuracy": 0.7789938881993294, + "num_tokens": 2651938.0, + "step": 500 + }, + { + "entropy": 0.6695188779383898, + "epoch": 1.07380073800738, + "grad_norm": 7.0, + "learning_rate": 1.7697777777777777e-05, + "loss": 0.664512825012207, + "mean_token_accuracy": 0.7832319870591163, + "num_tokens": 2704948.0, + "step": 510 + }, + { + "entropy": 0.6812492057681083, + "epoch": 1.0948866631523457, + "grad_norm": 7.0, + "learning_rate": 1.760888888888889e-05, + "loss": 0.7036191940307617, + "mean_token_accuracy": 0.7707728065550328, + "num_tokens": 2757576.0, + "step": 520 + }, + { + "entropy": 0.6861097872257232, + "epoch": 1.1159725882973115, + "grad_norm": 7.3125, + "learning_rate": 1.752e-05, + "loss": 0.6651991844177246, + "mean_token_accuracy": 0.7841649115085602, + "num_tokens": 2808989.0, + "step": 530 + }, + { + "entropy": 0.6694958411157131, + "epoch": 1.1370585134422773, + "grad_norm": 6.78125, + "learning_rate": 1.7431111111111112e-05, + "loss": 0.6833772659301758, + "mean_token_accuracy": 0.7775424145162105, + "num_tokens": 2864507.0, + "step": 540 + }, + { + "entropy": 0.6568067740648985, + "epoch": 1.158144438587243, + "grad_norm": 6.21875, + "learning_rate": 1.7342222222222226e-05, + "loss": 0.6552294254302978, + "mean_token_accuracy": 0.7832994520664215, + "num_tokens": 2919201.0, + "step": 550 + }, + { + "entropy": 0.6432577680796385, + "epoch": 1.1792303637322088, + "grad_norm": 6.65625, + "learning_rate": 1.7253333333333336e-05, + "loss": 0.6560451030731201, + "mean_token_accuracy": 0.7863748580217361, + "num_tokens": 2972434.0, + "step": 560 + }, + { + "entropy": 0.6655397050082683, + "epoch": 1.2003162888771746, + "grad_norm": 6.09375, + "learning_rate": 1.7164444444444447e-05, + "loss": 0.6878513336181641, + "mean_token_accuracy": 0.7753379993140698, + "num_tokens": 3025099.0, + "step": 570 + }, + { + "entropy": 0.686093359068036, + "epoch": 1.2214022140221403, + "grad_norm": 7.09375, + "learning_rate": 1.7075555555555557e-05, + "loss": 0.6911579132080078, + "mean_token_accuracy": 0.7787310972809791, + "num_tokens": 3076307.0, + "step": 580 + }, + { + "entropy": 0.6812980778515338, + "epoch": 1.2424881391671059, + "grad_norm": 7.125, + "learning_rate": 1.6986666666666668e-05, + "loss": 0.6704156398773193, + "mean_token_accuracy": 0.7793935135006904, + "num_tokens": 3130053.0, + "step": 590 + }, + { + "entropy": 0.6409769468009472, + "epoch": 1.2635740643120716, + "grad_norm": 7.4375, + "learning_rate": 1.6897777777777778e-05, + "loss": 0.661864185333252, + "mean_token_accuracy": 0.7865670621395111, + "num_tokens": 3183331.0, + "step": 600 + }, + { + "entropy": 0.6799454756081105, + "epoch": 1.2846599894570374, + "grad_norm": 6.65625, + "learning_rate": 1.680888888888889e-05, + "loss": 0.6818383693695068, + "mean_token_accuracy": 0.778287123888731, + "num_tokens": 3239561.0, + "step": 610 + }, + { + "entropy": 0.6814555354416371, + "epoch": 1.3057459146020032, + "grad_norm": 6.3125, + "learning_rate": 1.672e-05, + "loss": 0.679498815536499, + "mean_token_accuracy": 0.7788635492324829, + "num_tokens": 3292996.0, + "step": 620 + }, + { + "entropy": 0.667108029127121, + "epoch": 1.326831839746969, + "grad_norm": 6.90625, + "learning_rate": 1.6631111111111113e-05, + "loss": 0.6758651733398438, + "mean_token_accuracy": 0.7792007848620415, + "num_tokens": 3345736.0, + "step": 630 + }, + { + "entropy": 0.6692709110677242, + "epoch": 1.3479177648919347, + "grad_norm": 6.96875, + "learning_rate": 1.6542222222222223e-05, + "loss": 0.6796128273010253, + "mean_token_accuracy": 0.7766051791608334, + "num_tokens": 3398076.0, + "step": 640 + }, + { + "entropy": 0.6564048878848553, + "epoch": 1.3690036900369003, + "grad_norm": 6.9375, + "learning_rate": 1.6453333333333334e-05, + "loss": 0.6633264064788819, + "mean_token_accuracy": 0.7854946702718735, + "num_tokens": 3450796.0, + "step": 650 + }, + { + "entropy": 0.6611979272216558, + "epoch": 1.390089615181866, + "grad_norm": 6.65625, + "learning_rate": 1.6364444444444444e-05, + "loss": 0.6661429405212402, + "mean_token_accuracy": 0.7835663430392742, + "num_tokens": 3503863.0, + "step": 660 + }, + { + "entropy": 0.6534961331635714, + "epoch": 1.4111755403268318, + "grad_norm": 6.9375, + "learning_rate": 1.6275555555555558e-05, + "loss": 0.6586530208587646, + "mean_token_accuracy": 0.7844067431986332, + "num_tokens": 3556265.0, + "step": 670 + }, + { + "entropy": 0.6516873501241207, + "epoch": 1.4322614654717976, + "grad_norm": 6.40625, + "learning_rate": 1.618666666666667e-05, + "loss": 0.6522529125213623, + "mean_token_accuracy": 0.7855845786631107, + "num_tokens": 3611423.0, + "step": 680 + }, + { + "entropy": 0.6764013767242432, + "epoch": 1.4533473906167633, + "grad_norm": 6.71875, + "learning_rate": 1.609777777777778e-05, + "loss": 0.6884442806243897, + "mean_token_accuracy": 0.7756352722644806, + "num_tokens": 3666630.0, + "step": 690 + }, + { + "entropy": 0.6554486058652401, + "epoch": 1.474433315761729, + "grad_norm": 7.40625, + "learning_rate": 1.600888888888889e-05, + "loss": 0.6563316822052002, + "mean_token_accuracy": 0.7854672558605671, + "num_tokens": 3718719.0, + "step": 700 + }, + { + "entropy": 0.6385185327380896, + "epoch": 1.4955192409066949, + "grad_norm": 6.59375, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.6513954639434815, + "mean_token_accuracy": 0.7851097173988819, + "num_tokens": 3771822.0, + "step": 710 + }, + { + "entropy": 0.654766371101141, + "epoch": 1.5166051660516606, + "grad_norm": 6.5, + "learning_rate": 1.5831111111111114e-05, + "loss": 0.6622748851776123, + "mean_token_accuracy": 0.7847663886845112, + "num_tokens": 3823795.0, + "step": 720 + }, + { + "entropy": 0.6597310703247785, + "epoch": 1.5376910911966264, + "grad_norm": 6.375, + "learning_rate": 1.5742222222222224e-05, + "loss": 0.6634318351745605, + "mean_token_accuracy": 0.7861979939043522, + "num_tokens": 3875784.0, + "step": 730 + }, + { + "entropy": 0.6637851595878601, + "epoch": 1.558777016341592, + "grad_norm": 6.9375, + "learning_rate": 1.5653333333333335e-05, + "loss": 0.6621220111846924, + "mean_token_accuracy": 0.7886320613324642, + "num_tokens": 3929412.0, + "step": 740 + }, + { + "entropy": 0.637692479044199, + "epoch": 1.5798629414865577, + "grad_norm": 6.59375, + "learning_rate": 1.5564444444444445e-05, + "loss": 0.6402386665344239, + "mean_token_accuracy": 0.7895873270928859, + "num_tokens": 3982472.0, + "step": 750 + }, + { + "entropy": 0.6578179821372032, + "epoch": 1.6009488666315235, + "grad_norm": 7.25, + "learning_rate": 1.5475555555555556e-05, + "loss": 0.6751936435699463, + "mean_token_accuracy": 0.7776105716824532, + "num_tokens": 4037829.0, + "step": 760 + }, + { + "entropy": 0.6653612300753593, + "epoch": 1.6220347917764892, + "grad_norm": 6.9375, + "learning_rate": 1.5386666666666666e-05, + "loss": 0.6717413425445556, + "mean_token_accuracy": 0.777862798422575, + "num_tokens": 4093735.0, + "step": 770 + }, + { + "entropy": 0.6561707239598036, + "epoch": 1.6431207169214548, + "grad_norm": 7.03125, + "learning_rate": 1.5297777777777777e-05, + "loss": 0.6501868724822998, + "mean_token_accuracy": 0.7863908343017101, + "num_tokens": 4148410.0, + "step": 780 + }, + { + "entropy": 0.6655665650963783, + "epoch": 1.6642066420664205, + "grad_norm": 6.59375, + "learning_rate": 1.520888888888889e-05, + "loss": 0.679684829711914, + "mean_token_accuracy": 0.778567586839199, + "num_tokens": 4202907.0, + "step": 790 + }, + { + "entropy": 0.6781473312526941, + "epoch": 1.6852925672113863, + "grad_norm": 6.875, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.6722404479980468, + "mean_token_accuracy": 0.7763243250548839, + "num_tokens": 4256213.0, + "step": 800 + }, + { + "entropy": 0.6321199681609869, + "epoch": 1.706378492356352, + "grad_norm": 6.78125, + "learning_rate": 1.5031111111111111e-05, + "loss": 0.6404708385467529, + "mean_token_accuracy": 0.7903632469475269, + "num_tokens": 4309268.0, + "step": 810 + }, + { + "entropy": 0.6424700867384672, + "epoch": 1.7274644175013179, + "grad_norm": 7.15625, + "learning_rate": 1.4942222222222222e-05, + "loss": 0.6444578170776367, + "mean_token_accuracy": 0.7881530813872815, + "num_tokens": 4360761.0, + "step": 820 + }, + { + "entropy": 0.6615507304668427, + "epoch": 1.7485503426462836, + "grad_norm": 6.0625, + "learning_rate": 1.4853333333333336e-05, + "loss": 0.6604718208312989, + "mean_token_accuracy": 0.782805722206831, + "num_tokens": 4411678.0, + "step": 830 + }, + { + "entropy": 0.6196760050952435, + "epoch": 1.7696362677912494, + "grad_norm": 6.6875, + "learning_rate": 1.4764444444444446e-05, + "loss": 0.6261836051940918, + "mean_token_accuracy": 0.7926589228212834, + "num_tokens": 4467254.0, + "step": 840 + }, + { + "entropy": 0.6139154363423586, + "epoch": 1.7907221929362152, + "grad_norm": 7.21875, + "learning_rate": 1.4675555555555557e-05, + "loss": 0.6301932811737061, + "mean_token_accuracy": 0.7918383464217186, + "num_tokens": 4517383.0, + "step": 850 + }, + { + "entropy": 0.6410171519964933, + "epoch": 1.811808118081181, + "grad_norm": 7.1875, + "learning_rate": 1.4586666666666667e-05, + "loss": 0.655810022354126, + "mean_token_accuracy": 0.7856725618243218, + "num_tokens": 4569980.0, + "step": 860 + }, + { + "entropy": 0.6666022066026926, + "epoch": 1.8328940432261467, + "grad_norm": 6.5625, + "learning_rate": 1.449777777777778e-05, + "loss": 0.6677482604980469, + "mean_token_accuracy": 0.7798060193657875, + "num_tokens": 4622612.0, + "step": 870 + }, + { + "entropy": 0.6398684818297624, + "epoch": 1.8539799683711122, + "grad_norm": 7.84375, + "learning_rate": 1.440888888888889e-05, + "loss": 0.6501501083374024, + "mean_token_accuracy": 0.7857734531164169, + "num_tokens": 4676525.0, + "step": 880 + }, + { + "entropy": 0.6285852700471878, + "epoch": 1.875065893516078, + "grad_norm": 7.0625, + "learning_rate": 1.432e-05, + "loss": 0.6422877311706543, + "mean_token_accuracy": 0.7884316690266132, + "num_tokens": 4734090.0, + "step": 890 + }, + { + "entropy": 0.6409385818988085, + "epoch": 1.8961518186610438, + "grad_norm": 7.9375, + "learning_rate": 1.4231111111111114e-05, + "loss": 0.6352181434631348, + "mean_token_accuracy": 0.7952412232756615, + "num_tokens": 4788537.0, + "step": 900 + }, + { + "entropy": 0.6456746399402619, + "epoch": 1.9172377438060095, + "grad_norm": 6.90625, + "learning_rate": 1.4142222222222224e-05, + "loss": 0.6564863204956055, + "mean_token_accuracy": 0.789187741279602, + "num_tokens": 4840342.0, + "step": 910 + }, + { + "entropy": 0.6360713161528111, + "epoch": 1.938323668950975, + "grad_norm": 6.3125, + "learning_rate": 1.4053333333333335e-05, + "loss": 0.6300373554229737, + "mean_token_accuracy": 0.7876299776136875, + "num_tokens": 4891524.0, + "step": 920 + }, + { + "entropy": 0.6285211775451899, + "epoch": 1.9594095940959408, + "grad_norm": 7.78125, + "learning_rate": 1.3964444444444445e-05, + "loss": 0.6337003231048584, + "mean_token_accuracy": 0.7889726974070073, + "num_tokens": 4939573.0, + "step": 930 + }, + { + "entropy": 0.6301535632461309, + "epoch": 1.9804955192409066, + "grad_norm": 6.84375, + "learning_rate": 1.3875555555555557e-05, + "loss": 0.6374162673950196, + "mean_token_accuracy": 0.7898242607712745, + "num_tokens": 4991267.0, + "step": 940 + }, + { + "entropy": 0.6537058566873138, + "epoch": 2.0, + "grad_norm": 13.3125, + "learning_rate": 1.3786666666666668e-05, + "loss": 0.641156530380249, + "mean_token_accuracy": 0.7822844466647586, + "num_tokens": 5036304.0, + "step": 950 + }, + { + "entropy": 0.6411334682255984, + "epoch": 2.0210859251449658, + "grad_norm": 7.1875, + "learning_rate": 1.3697777777777778e-05, + "loss": 0.6083747386932373, + "mean_token_accuracy": 0.8002627611160278, + "num_tokens": 5089737.0, + "step": 960 + }, + { + "entropy": 0.5841103594750165, + "epoch": 2.0421718502899315, + "grad_norm": 6.53125, + "learning_rate": 1.3608888888888889e-05, + "loss": 0.6012133121490478, + "mean_token_accuracy": 0.8027123242616654, + "num_tokens": 5143330.0, + "step": 970 + }, + { + "entropy": 0.590392829477787, + "epoch": 2.0632577754348973, + "grad_norm": 6.46875, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.5829256534576416, + "mean_token_accuracy": 0.8068650290369987, + "num_tokens": 5198614.0, + "step": 980 + }, + { + "entropy": 0.6148912344127894, + "epoch": 2.084343700579863, + "grad_norm": 6.59375, + "learning_rate": 1.3431111111111113e-05, + "loss": 0.6117685317993165, + "mean_token_accuracy": 0.7951427385210991, + "num_tokens": 5252755.0, + "step": 990 + }, + { + "entropy": 0.5893940377980471, + "epoch": 2.105429625724829, + "grad_norm": 6.625, + "learning_rate": 1.3342222222222224e-05, + "loss": 0.5790496349334717, + "mean_token_accuracy": 0.809234394133091, + "num_tokens": 5304859.0, + "step": 1000 + }, + { + "entropy": 0.587013079598546, + "epoch": 2.1265155508697946, + "grad_norm": 6.9375, + "learning_rate": 1.3253333333333334e-05, + "loss": 0.5892822265625, + "mean_token_accuracy": 0.8040889374911785, + "num_tokens": 5357877.0, + "step": 1010 + }, + { + "entropy": 0.5877181258052587, + "epoch": 2.14760147601476, + "grad_norm": 6.09375, + "learning_rate": 1.3164444444444446e-05, + "loss": 0.595665693283081, + "mean_token_accuracy": 0.8036225430667401, + "num_tokens": 5413504.0, + "step": 1020 + }, + { + "entropy": 0.5848948258906603, + "epoch": 2.1686874011597257, + "grad_norm": 6.4375, + "learning_rate": 1.3075555555555557e-05, + "loss": 0.5771171092987061, + "mean_token_accuracy": 0.8047491163015366, + "num_tokens": 5466549.0, + "step": 1030 + }, + { + "entropy": 0.5815035384148359, + "epoch": 2.1897733263046915, + "grad_norm": 6.96875, + "learning_rate": 1.2986666666666667e-05, + "loss": 0.5768617153167724, + "mean_token_accuracy": 0.8057669088244438, + "num_tokens": 5519515.0, + "step": 1040 + }, + { + "entropy": 0.58335498906672, + "epoch": 2.2108592514496572, + "grad_norm": 7.6875, + "learning_rate": 1.2897777777777778e-05, + "loss": 0.6007503509521485, + "mean_token_accuracy": 0.8005498602986336, + "num_tokens": 5570188.0, + "step": 1050 + }, + { + "entropy": 0.6068603809922933, + "epoch": 2.231945176594623, + "grad_norm": 6.5625, + "learning_rate": 1.2808888888888891e-05, + "loss": 0.6024115562438965, + "mean_token_accuracy": 0.7973541341722011, + "num_tokens": 5624190.0, + "step": 1060 + }, + { + "entropy": 0.5774707745760679, + "epoch": 2.2530311017395888, + "grad_norm": 6.53125, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.5705479145050049, + "mean_token_accuracy": 0.8100002221763134, + "num_tokens": 5675656.0, + "step": 1070 + }, + { + "entropy": 0.5851722996681928, + "epoch": 2.2741170268845545, + "grad_norm": 7.3125, + "learning_rate": 1.2631111111111112e-05, + "loss": 0.5905722141265869, + "mean_token_accuracy": 0.8069308035075664, + "num_tokens": 5728878.0, + "step": 1080 + }, + { + "entropy": 0.594813584536314, + "epoch": 2.2952029520295203, + "grad_norm": 6.875, + "learning_rate": 1.2542222222222223e-05, + "loss": 0.5923901557922363, + "mean_token_accuracy": 0.8034852437675, + "num_tokens": 5783257.0, + "step": 1090 + }, + { + "entropy": 0.5841789487749338, + "epoch": 2.316288877174486, + "grad_norm": 7.0, + "learning_rate": 1.2453333333333335e-05, + "loss": 0.5864346504211426, + "mean_token_accuracy": 0.8033530995249748, + "num_tokens": 5834581.0, + "step": 1100 + }, + { + "entropy": 0.6010521166026592, + "epoch": 2.337374802319452, + "grad_norm": 6.9375, + "learning_rate": 1.2364444444444445e-05, + "loss": 0.5941137313842774, + "mean_token_accuracy": 0.8033333070576191, + "num_tokens": 5887458.0, + "step": 1110 + }, + { + "entropy": 0.5919341973960399, + "epoch": 2.3584607274644176, + "grad_norm": 8.75, + "learning_rate": 1.2275555555555556e-05, + "loss": 0.6103182315826416, + "mean_token_accuracy": 0.7999237082898617, + "num_tokens": 5941284.0, + "step": 1120 + }, + { + "entropy": 0.5942621320486069, + "epoch": 2.3795466526093834, + "grad_norm": 7.375, + "learning_rate": 1.2186666666666666e-05, + "loss": 0.5983363151550293, + "mean_token_accuracy": 0.8016222022473812, + "num_tokens": 5995817.0, + "step": 1130 + }, + { + "entropy": 0.6046227600425482, + "epoch": 2.400632577754349, + "grad_norm": 6.40625, + "learning_rate": 1.209777777777778e-05, + "loss": 0.6026669025421143, + "mean_token_accuracy": 0.8024684950709343, + "num_tokens": 6050783.0, + "step": 1140 + }, + { + "entropy": 0.5998306684195995, + "epoch": 2.421718502899315, + "grad_norm": 6.59375, + "learning_rate": 1.200888888888889e-05, + "loss": 0.5893473625183105, + "mean_token_accuracy": 0.803314708918333, + "num_tokens": 6103371.0, + "step": 1150 + }, + { + "entropy": 0.593693134188652, + "epoch": 2.4428044280442807, + "grad_norm": 6.5, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.6076583862304688, + "mean_token_accuracy": 0.7995239071547985, + "num_tokens": 6155502.0, + "step": 1160 + }, + { + "entropy": 0.5827030725777149, + "epoch": 2.463890353189246, + "grad_norm": 6.84375, + "learning_rate": 1.1831111111111112e-05, + "loss": 0.5882821559906006, + "mean_token_accuracy": 0.7929977595806121, + "num_tokens": 6208060.0, + "step": 1170 + }, + { + "entropy": 0.5910260166972876, + "epoch": 2.4849762783342118, + "grad_norm": 7.53125, + "learning_rate": 1.1742222222222224e-05, + "loss": 0.5860947132110595, + "mean_token_accuracy": 0.8049415498971939, + "num_tokens": 6261588.0, + "step": 1180 + }, + { + "entropy": 0.5704694345593453, + "epoch": 2.5060622034791775, + "grad_norm": 6.75, + "learning_rate": 1.1653333333333334e-05, + "loss": 0.5749064445495605, + "mean_token_accuracy": 0.8084491945803165, + "num_tokens": 6314963.0, + "step": 1190 + }, + { + "entropy": 0.597440430521965, + "epoch": 2.5271481286241433, + "grad_norm": 6.5, + "learning_rate": 1.1564444444444445e-05, + "loss": 0.5958696842193604, + "mean_token_accuracy": 0.8057812102138996, + "num_tokens": 6368741.0, + "step": 1200 + }, + { + "entropy": 0.6070981964468956, + "epoch": 2.548234053769109, + "grad_norm": 7.96875, + "learning_rate": 1.1475555555555557e-05, + "loss": 0.6143896102905273, + "mean_token_accuracy": 0.7980599500238895, + "num_tokens": 6425424.0, + "step": 1210 + }, + { + "entropy": 0.5948727998882533, + "epoch": 2.569319978914075, + "grad_norm": 6.5, + "learning_rate": 1.1386666666666669e-05, + "loss": 0.5917119026184082, + "mean_token_accuracy": 0.8025051333010197, + "num_tokens": 6476263.0, + "step": 1220 + }, + { + "entropy": 0.5811738461256027, + "epoch": 2.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.129777777777778e-05, + "loss": 0.5865864753723145, + "mean_token_accuracy": 0.8044181451201439, + "num_tokens": 6528012.0, + "step": 1230 + }, + { + "entropy": 0.5937653455883265, + "epoch": 2.6114918292040064, + "grad_norm": 6.59375, + "learning_rate": 1.120888888888889e-05, + "loss": 0.5843213558197021, + "mean_token_accuracy": 0.8025668740272522, + "num_tokens": 6583421.0, + "step": 1240 + }, + { + "entropy": 0.573565949127078, + "epoch": 2.632577754348972, + "grad_norm": 6.5625, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.5826557159423829, + "mean_token_accuracy": 0.8070438347756863, + "num_tokens": 6636858.0, + "step": 1250 + }, + { + "entropy": 0.6216809719800949, + "epoch": 2.653663679493938, + "grad_norm": 7.09375, + "learning_rate": 1.1031111111111112e-05, + "loss": 0.6125890254974365, + "mean_token_accuracy": 0.7970895677804947, + "num_tokens": 6692168.0, + "step": 1260 + }, + { + "entropy": 0.5998566258698702, + "epoch": 2.6747496046389037, + "grad_norm": 7.0625, + "learning_rate": 1.0942222222222223e-05, + "loss": 0.5921674728393554, + "mean_token_accuracy": 0.8061520479619503, + "num_tokens": 6745834.0, + "step": 1270 + }, + { + "entropy": 0.5880189586430788, + "epoch": 2.6958355297838694, + "grad_norm": 7.59375, + "learning_rate": 1.0853333333333333e-05, + "loss": 0.5986497402191162, + "mean_token_accuracy": 0.8006219923496246, + "num_tokens": 6800244.0, + "step": 1280 + }, + { + "entropy": 0.5981763809919357, + "epoch": 2.716921454928835, + "grad_norm": 6.6875, + "learning_rate": 1.0764444444444445e-05, + "loss": 0.6020579814910889, + "mean_token_accuracy": 0.8032843962311744, + "num_tokens": 6853454.0, + "step": 1290 + }, + { + "entropy": 0.5963539965450764, + "epoch": 2.7380073800738005, + "grad_norm": 6.59375, + "learning_rate": 1.0675555555555558e-05, + "loss": 0.5908561706542969, + "mean_token_accuracy": 0.8011696465313435, + "num_tokens": 6904473.0, + "step": 1300 + }, + { + "entropy": 0.5899917531758547, + "epoch": 2.7590933052187667, + "grad_norm": 6.6875, + "learning_rate": 1.0586666666666668e-05, + "loss": 0.5744058132171631, + "mean_token_accuracy": 0.8053683824837208, + "num_tokens": 6956756.0, + "step": 1310 + }, + { + "entropy": 0.582448274642229, + "epoch": 2.780179230363732, + "grad_norm": 6.78125, + "learning_rate": 1.0497777777777779e-05, + "loss": 0.5848373889923095, + "mean_token_accuracy": 0.8054254874587059, + "num_tokens": 7008611.0, + "step": 1320 + }, + { + "entropy": 0.5906594522297383, + "epoch": 2.801265155508698, + "grad_norm": 7.75, + "learning_rate": 1.040888888888889e-05, + "loss": 0.6006948471069335, + "mean_token_accuracy": 0.7971956506371498, + "num_tokens": 7062235.0, + "step": 1330 + }, + { + "entropy": 0.5976275954395532, + "epoch": 2.8223510806536636, + "grad_norm": 6.34375, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.5997600078582763, + "mean_token_accuracy": 0.7998531877994537, + "num_tokens": 7115737.0, + "step": 1340 + }, + { + "entropy": 0.6079881630837918, + "epoch": 2.8434370057986293, + "grad_norm": 7.0, + "learning_rate": 1.0231111111111112e-05, + "loss": 0.6024625778198243, + "mean_token_accuracy": 0.7988371469080449, + "num_tokens": 7166870.0, + "step": 1350 + }, + { + "entropy": 0.6036868289113044, + "epoch": 2.864522930943595, + "grad_norm": 6.6875, + "learning_rate": 1.0142222222222222e-05, + "loss": 0.5962271213531494, + "mean_token_accuracy": 0.8011133573949337, + "num_tokens": 7219785.0, + "step": 1360 + }, + { + "entropy": 0.5746289525181055, + "epoch": 2.885608856088561, + "grad_norm": 7.5625, + "learning_rate": 1.0053333333333334e-05, + "loss": 0.5769401073455811, + "mean_token_accuracy": 0.809252617508173, + "num_tokens": 7275411.0, + "step": 1370 + }, + { + "entropy": 0.5846736520528794, + "epoch": 2.9066947812335266, + "grad_norm": 7.125, + "learning_rate": 9.964444444444445e-06, + "loss": 0.5899319171905517, + "mean_token_accuracy": 0.8028473742306232, + "num_tokens": 7326752.0, + "step": 1380 + }, + { + "entropy": 0.5813163112848997, + "epoch": 2.9277807063784924, + "grad_norm": 7.46875, + "learning_rate": 9.875555555555557e-06, + "loss": 0.5799898147583008, + "mean_token_accuracy": 0.8026903085410595, + "num_tokens": 7375608.0, + "step": 1390 + }, + { + "entropy": 0.5724870227277279, + "epoch": 2.948866631523458, + "grad_norm": 6.90625, + "learning_rate": 9.786666666666667e-06, + "loss": 0.5783823490142822, + "mean_token_accuracy": 0.8059644535183906, + "num_tokens": 7430145.0, + "step": 1400 + }, + { + "entropy": 0.5778653729707003, + "epoch": 2.969952556668424, + "grad_norm": 6.84375, + "learning_rate": 9.697777777777778e-06, + "loss": 0.5640594005584717, + "mean_token_accuracy": 0.8100298568606377, + "num_tokens": 7482976.0, + "step": 1410 + }, + { + "entropy": 0.5823766525834799, + "epoch": 2.9910384818133897, + "grad_norm": 6.5625, + "learning_rate": 9.60888888888889e-06, + "loss": 0.5931865692138671, + "mean_token_accuracy": 0.8023995622992516, + "num_tokens": 7533043.0, + "step": 1420 + }, + { + "entropy": 0.5924574036050487, + "epoch": 3.0105429625724827, + "grad_norm": 6.53125, + "learning_rate": 9.52e-06, + "loss": 0.5867043972015381, + "mean_token_accuracy": 0.803342911842707, + "num_tokens": 7581993.0, + "step": 1430 + }, + { + "entropy": 0.579814862459898, + "epoch": 3.0316288877174484, + "grad_norm": 6.34375, + "learning_rate": 9.431111111111112e-06, + "loss": 0.5730488300323486, + "mean_token_accuracy": 0.8101175747811794, + "num_tokens": 7639343.0, + "step": 1440 + }, + { + "entropy": 0.560778708383441, + "epoch": 3.052714812862414, + "grad_norm": 7.40625, + "learning_rate": 9.342222222222223e-06, + "loss": 0.5500640392303466, + "mean_token_accuracy": 0.8135757654905319, + "num_tokens": 7690695.0, + "step": 1450 + }, + { + "entropy": 0.5529176875948906, + "epoch": 3.07380073800738, + "grad_norm": 7.21875, + "learning_rate": 9.253333333333333e-06, + "loss": 0.549665117263794, + "mean_token_accuracy": 0.81509620398283, + "num_tokens": 7746705.0, + "step": 1460 + }, + { + "entropy": 0.5382925808429718, + "epoch": 3.0948866631523457, + "grad_norm": 7.5, + "learning_rate": 9.164444444444446e-06, + "loss": 0.54415283203125, + "mean_token_accuracy": 0.819620954990387, + "num_tokens": 7799255.0, + "step": 1470 + }, + { + "entropy": 0.551479097828269, + "epoch": 3.1159725882973115, + "grad_norm": 7.125, + "learning_rate": 9.075555555555556e-06, + "loss": 0.5520545959472656, + "mean_token_accuracy": 0.8147840730845928, + "num_tokens": 7852753.0, + "step": 1480 + }, + { + "entropy": 0.571834359690547, + "epoch": 3.1370585134422773, + "grad_norm": 7.59375, + "learning_rate": 8.986666666666666e-06, + "loss": 0.5553510189056396, + "mean_token_accuracy": 0.8117231354117393, + "num_tokens": 7904298.0, + "step": 1490 + }, + { + "entropy": 0.5676855646073818, + "epoch": 3.158144438587243, + "grad_norm": 7.75, + "learning_rate": 8.897777777777779e-06, + "loss": 0.5595914840698242, + "mean_token_accuracy": 0.8130396403372288, + "num_tokens": 7956705.0, + "step": 1500 + }, + { + "entropy": 0.5613869089633227, + "epoch": 3.179230363732209, + "grad_norm": 7.78125, + "learning_rate": 8.80888888888889e-06, + "loss": 0.544979190826416, + "mean_token_accuracy": 0.8170888110995292, + "num_tokens": 8008023.0, + "step": 1510 + }, + { + "entropy": 0.570792680233717, + "epoch": 3.2003162888771746, + "grad_norm": 6.75, + "learning_rate": 8.720000000000001e-06, + "loss": 0.560499906539917, + "mean_token_accuracy": 0.8146887816488743, + "num_tokens": 8058344.0, + "step": 1520 + }, + { + "entropy": 0.5498528245836496, + "epoch": 3.2214022140221403, + "grad_norm": 7.15625, + "learning_rate": 8.631111111111112e-06, + "loss": 0.5479342937469482, + "mean_token_accuracy": 0.8137361407279968, + "num_tokens": 8110587.0, + "step": 1530 + }, + { + "entropy": 0.5499754682183265, + "epoch": 3.242488139167106, + "grad_norm": 6.9375, + "learning_rate": 8.542222222222222e-06, + "loss": 0.5445993423461915, + "mean_token_accuracy": 0.8179522223770619, + "num_tokens": 8160497.0, + "step": 1540 + }, + { + "entropy": 0.572133193910122, + "epoch": 3.263574064312072, + "grad_norm": 6.84375, + "learning_rate": 8.453333333333334e-06, + "loss": 0.5740565776824951, + "mean_token_accuracy": 0.8101011097431183, + "num_tokens": 8214878.0, + "step": 1550 + }, + { + "entropy": 0.5666386324912309, + "epoch": 3.2846599894570376, + "grad_norm": 7.59375, + "learning_rate": 8.364444444444445e-06, + "loss": 0.5723002433776856, + "mean_token_accuracy": 0.8091295748949051, + "num_tokens": 8267314.0, + "step": 1560 + }, + { + "entropy": 0.5659929499030113, + "epoch": 3.305745914602003, + "grad_norm": 6.96875, + "learning_rate": 8.275555555555557e-06, + "loss": 0.5623570442199707, + "mean_token_accuracy": 0.8093601904809475, + "num_tokens": 8320145.0, + "step": 1570 + }, + { + "entropy": 0.5665242202579975, + "epoch": 3.3268318397469687, + "grad_norm": 7.0625, + "learning_rate": 8.186666666666667e-06, + "loss": 0.5534502983093261, + "mean_token_accuracy": 0.8129527278244495, + "num_tokens": 8370852.0, + "step": 1580 + }, + { + "entropy": 0.5829939197748899, + "epoch": 3.3479177648919345, + "grad_norm": 7.40625, + "learning_rate": 8.09777777777778e-06, + "loss": 0.5778002738952637, + "mean_token_accuracy": 0.8072332389652729, + "num_tokens": 8424209.0, + "step": 1590 + }, + { + "entropy": 0.5479087311774492, + "epoch": 3.3690036900369003, + "grad_norm": 6.15625, + "learning_rate": 8.00888888888889e-06, + "loss": 0.5359113216400146, + "mean_token_accuracy": 0.819999934732914, + "num_tokens": 8476617.0, + "step": 1600 + }, + { + "entropy": 0.5520685013383627, + "epoch": 3.390089615181866, + "grad_norm": 6.53125, + "learning_rate": 7.92e-06, + "loss": 0.540053415298462, + "mean_token_accuracy": 0.817080944776535, + "num_tokens": 8529074.0, + "step": 1610 + }, + { + "entropy": 0.5533283669501543, + "epoch": 3.411175540326832, + "grad_norm": 7.25, + "learning_rate": 7.831111111111111e-06, + "loss": 0.5461609363555908, + "mean_token_accuracy": 0.8130916491150856, + "num_tokens": 8582416.0, + "step": 1620 + }, + { + "entropy": 0.5606811255216598, + "epoch": 3.4322614654717976, + "grad_norm": 8.25, + "learning_rate": 7.742222222222223e-06, + "loss": 0.565118408203125, + "mean_token_accuracy": 0.813844607770443, + "num_tokens": 8634249.0, + "step": 1630 + }, + { + "entropy": 0.5458527609705925, + "epoch": 3.4533473906167633, + "grad_norm": 7.6875, + "learning_rate": 7.653333333333333e-06, + "loss": 0.5410670757293701, + "mean_token_accuracy": 0.8188620381057262, + "num_tokens": 8685168.0, + "step": 1640 + }, + { + "entropy": 0.5717878874391318, + "epoch": 3.474433315761729, + "grad_norm": 7.6875, + "learning_rate": 7.564444444444446e-06, + "loss": 0.5788788795471191, + "mean_token_accuracy": 0.8078153640031814, + "num_tokens": 8740562.0, + "step": 1650 + }, + { + "entropy": 0.5646973233669996, + "epoch": 3.495519240906695, + "grad_norm": 6.75, + "learning_rate": 7.475555555555556e-06, + "loss": 0.5595187187194824, + "mean_token_accuracy": 0.8135132856667042, + "num_tokens": 8794411.0, + "step": 1660 + }, + { + "entropy": 0.5641634043306112, + "epoch": 3.5166051660516606, + "grad_norm": 6.6875, + "learning_rate": 7.386666666666667e-06, + "loss": 0.5511392116546631, + "mean_token_accuracy": 0.8159678690135479, + "num_tokens": 8844547.0, + "step": 1670 + }, + { + "entropy": 0.5724293068051338, + "epoch": 3.5376910911966264, + "grad_norm": 6.875, + "learning_rate": 7.297777777777778e-06, + "loss": 0.5668179988861084, + "mean_token_accuracy": 0.8106696806848049, + "num_tokens": 8902311.0, + "step": 1680 + }, + { + "entropy": 0.5610478781163692, + "epoch": 3.558777016341592, + "grad_norm": 6.96875, + "learning_rate": 7.20888888888889e-06, + "loss": 0.5528025150299072, + "mean_token_accuracy": 0.8120494581758976, + "num_tokens": 8957569.0, + "step": 1690 + }, + { + "entropy": 0.5370181340724229, + "epoch": 3.5798629414865575, + "grad_norm": 7.28125, + "learning_rate": 7.1200000000000004e-06, + "loss": 0.5477783203125, + "mean_token_accuracy": 0.8162322834134101, + "num_tokens": 9009083.0, + "step": 1700 + }, + { + "entropy": 0.5545388571918011, + "epoch": 3.6009488666315237, + "grad_norm": 6.5, + "learning_rate": 7.031111111111112e-06, + "loss": 0.5430552005767822, + "mean_token_accuracy": 0.8191993653774261, + "num_tokens": 9061980.0, + "step": 1710 + }, + { + "entropy": 0.5662489753216505, + "epoch": 3.622034791776489, + "grad_norm": 7.3125, + "learning_rate": 6.942222222222222e-06, + "loss": 0.5623671531677246, + "mean_token_accuracy": 0.8118648871779441, + "num_tokens": 9117653.0, + "step": 1720 + }, + { + "entropy": 0.5656806524842978, + "epoch": 3.643120716921455, + "grad_norm": 7.40625, + "learning_rate": 6.853333333333334e-06, + "loss": 0.5693060398101807, + "mean_token_accuracy": 0.8121738277375699, + "num_tokens": 9168884.0, + "step": 1730 + }, + { + "entropy": 0.5669469233602286, + "epoch": 3.6642066420664205, + "grad_norm": 6.875, + "learning_rate": 6.764444444444445e-06, + "loss": 0.5708917617797852, + "mean_token_accuracy": 0.8130646869540215, + "num_tokens": 9221794.0, + "step": 1740 + }, + { + "entropy": 0.5680123705416917, + "epoch": 3.6852925672113863, + "grad_norm": 7.3125, + "learning_rate": 6.675555555555556e-06, + "loss": 0.565749979019165, + "mean_token_accuracy": 0.8114100478589534, + "num_tokens": 9277320.0, + "step": 1750 + }, + { + "entropy": 0.5670568168163299, + "epoch": 3.706378492356352, + "grad_norm": 6.8125, + "learning_rate": 6.5866666666666666e-06, + "loss": 0.5531346797943115, + "mean_token_accuracy": 0.815721620619297, + "num_tokens": 9328654.0, + "step": 1760 + }, + { + "entropy": 0.5794487085193396, + "epoch": 3.727464417501318, + "grad_norm": 7.25, + "learning_rate": 6.497777777777779e-06, + "loss": 0.5695443630218506, + "mean_token_accuracy": 0.8125023365020752, + "num_tokens": 9382781.0, + "step": 1770 + }, + { + "entropy": 0.5529596284031868, + "epoch": 3.7485503426462836, + "grad_norm": 7.21875, + "learning_rate": 6.408888888888889e-06, + "loss": 0.5538197040557862, + "mean_token_accuracy": 0.8132442943751812, + "num_tokens": 9434446.0, + "step": 1780 + }, + { + "entropy": 0.554557716101408, + "epoch": 3.7696362677912494, + "grad_norm": 7.34375, + "learning_rate": 6.3200000000000005e-06, + "loss": 0.5493638515472412, + "mean_token_accuracy": 0.8154775679111481, + "num_tokens": 9488504.0, + "step": 1790 + }, + { + "entropy": 0.5696977652609349, + "epoch": 3.790722192936215, + "grad_norm": 6.28125, + "learning_rate": 6.231111111111111e-06, + "loss": 0.5688616275787354, + "mean_token_accuracy": 0.8128006041049958, + "num_tokens": 9545943.0, + "step": 1800 + }, + { + "entropy": 0.5715777482837439, + "epoch": 3.811808118081181, + "grad_norm": 6.78125, + "learning_rate": 6.142222222222223e-06, + "loss": 0.5713305473327637, + "mean_token_accuracy": 0.811994157731533, + "num_tokens": 9597820.0, + "step": 1810 + }, + { + "entropy": 0.5638129018247128, + "epoch": 3.8328940432261467, + "grad_norm": 7.25, + "learning_rate": 6.0533333333333335e-06, + "loss": 0.5548198699951172, + "mean_token_accuracy": 0.8136275447905064, + "num_tokens": 9651480.0, + "step": 1820 + }, + { + "entropy": 0.5706254750490188, + "epoch": 3.853979968371112, + "grad_norm": 6.8125, + "learning_rate": 5.964444444444445e-06, + "loss": 0.5752524852752685, + "mean_token_accuracy": 0.8053847931325435, + "num_tokens": 9706114.0, + "step": 1830 + }, + { + "entropy": 0.5612337995320559, + "epoch": 3.875065893516078, + "grad_norm": 7.0, + "learning_rate": 5.875555555555556e-06, + "loss": 0.5460340499877929, + "mean_token_accuracy": 0.8159381657838821, + "num_tokens": 9760141.0, + "step": 1840 + }, + { + "entropy": 0.5514645285904407, + "epoch": 3.8961518186610435, + "grad_norm": 8.0, + "learning_rate": 5.7866666666666674e-06, + "loss": 0.5530869960784912, + "mean_token_accuracy": 0.8169759206473828, + "num_tokens": 9812466.0, + "step": 1850 + }, + { + "entropy": 0.5574501186609269, + "epoch": 3.9172377438060098, + "grad_norm": 7.5, + "learning_rate": 5.697777777777779e-06, + "loss": 0.5557053089141846, + "mean_token_accuracy": 0.8142473429441452, + "num_tokens": 9863547.0, + "step": 1860 + }, + { + "entropy": 0.560401076450944, + "epoch": 3.938323668950975, + "grad_norm": 6.53125, + "learning_rate": 5.608888888888889e-06, + "loss": 0.5598348140716553, + "mean_token_accuracy": 0.810575645416975, + "num_tokens": 9917210.0, + "step": 1870 + }, + { + "entropy": 0.5611382886767388, + "epoch": 3.959409594095941, + "grad_norm": 6.875, + "learning_rate": 5.5200000000000005e-06, + "loss": 0.5510398387908936, + "mean_token_accuracy": 0.8139252245426178, + "num_tokens": 9971753.0, + "step": 1880 + }, + { + "entropy": 0.571844469755888, + "epoch": 3.9804955192409066, + "grad_norm": 7.15625, + "learning_rate": 5.431111111111112e-06, + "loss": 0.5825337409973145, + "mean_token_accuracy": 0.8051016978919506, + "num_tokens": 10024121.0, + "step": 1890 + }, + { + "entropy": 0.5610769511880102, + "epoch": 4.0, + "grad_norm": 15.75, + "learning_rate": 5.342222222222223e-06, + "loss": 0.5544355392456055, + "mean_token_accuracy": 0.8145432029221509, + "num_tokens": 10072608.0, + "step": 1900 + }, + { + "entropy": 0.5582417894154788, + "epoch": 4.021085925144965, + "grad_norm": 6.5625, + "learning_rate": 5.2533333333333336e-06, + "loss": 0.5517781734466553, + "mean_token_accuracy": 0.8166630238294601, + "num_tokens": 10127149.0, + "step": 1910 + }, + { + "entropy": 0.5561550367623568, + "epoch": 4.0421718502899315, + "grad_norm": 6.5, + "learning_rate": 5.164444444444445e-06, + "loss": 0.5391493797302246, + "mean_token_accuracy": 0.8172842212021351, + "num_tokens": 10178921.0, + "step": 1920 + }, + { + "entropy": 0.5536927301436663, + "epoch": 4.063257775434897, + "grad_norm": 6.8125, + "learning_rate": 5.075555555555556e-06, + "loss": 0.5435996532440186, + "mean_token_accuracy": 0.8193235926330089, + "num_tokens": 10230953.0, + "step": 1930 + }, + { + "entropy": 0.5522860966622829, + "epoch": 4.084343700579863, + "grad_norm": 6.65625, + "learning_rate": 4.986666666666667e-06, + "loss": 0.5417564868927002, + "mean_token_accuracy": 0.8215342923998833, + "num_tokens": 10285145.0, + "step": 1940 + }, + { + "entropy": 0.558988232910633, + "epoch": 4.105429625724828, + "grad_norm": 8.25, + "learning_rate": 4.897777777777778e-06, + "loss": 0.5513458251953125, + "mean_token_accuracy": 0.8168663889169693, + "num_tokens": 10340678.0, + "step": 1950 + }, + { + "entropy": 0.5479944169521331, + "epoch": 4.126515550869795, + "grad_norm": 7.6875, + "learning_rate": 4.808888888888889e-06, + "loss": 0.5411903858184814, + "mean_token_accuracy": 0.8173033416271209, + "num_tokens": 10396265.0, + "step": 1960 + }, + { + "entropy": 0.5479635439813137, + "epoch": 4.14760147601476, + "grad_norm": 6.6875, + "learning_rate": 4.7200000000000005e-06, + "loss": 0.5325856685638428, + "mean_token_accuracy": 0.8196660950779915, + "num_tokens": 10448162.0, + "step": 1970 + }, + { + "entropy": 0.5631415072828532, + "epoch": 4.168687401159726, + "grad_norm": 6.96875, + "learning_rate": 4.631111111111111e-06, + "loss": 0.5563766956329346, + "mean_token_accuracy": 0.8139985457062722, + "num_tokens": 10500711.0, + "step": 1980 + }, + { + "entropy": 0.5530602999031544, + "epoch": 4.1897733263046915, + "grad_norm": 6.78125, + "learning_rate": 4.542222222222223e-06, + "loss": 0.5429882049560547, + "mean_token_accuracy": 0.815962927043438, + "num_tokens": 10554919.0, + "step": 1990 + }, + { + "entropy": 0.5508173421025276, + "epoch": 4.210859251449658, + "grad_norm": 6.90625, + "learning_rate": 4.453333333333334e-06, + "loss": 0.5390869140625, + "mean_token_accuracy": 0.8184089511632919, + "num_tokens": 10606269.0, + "step": 2000 + }, + { + "entropy": 0.5496918559074402, + "epoch": 4.231945176594623, + "grad_norm": 7.0, + "learning_rate": 4.364444444444445e-06, + "loss": 0.5486071109771729, + "mean_token_accuracy": 0.8196936115622521, + "num_tokens": 10660269.0, + "step": 2010 + }, + { + "entropy": 0.5489282257854938, + "epoch": 4.253031101739589, + "grad_norm": 7.4375, + "learning_rate": 4.275555555555556e-06, + "loss": 0.5514675617218018, + "mean_token_accuracy": 0.8145210847258568, + "num_tokens": 10711759.0, + "step": 2020 + }, + { + "entropy": 0.5529059153050184, + "epoch": 4.2741170268845545, + "grad_norm": 6.78125, + "learning_rate": 4.1866666666666675e-06, + "loss": 0.5444664001464844, + "mean_token_accuracy": 0.8184945359826088, + "num_tokens": 10763911.0, + "step": 2030 + }, + { + "entropy": 0.551506045460701, + "epoch": 4.29520295202952, + "grad_norm": 7.34375, + "learning_rate": 4.097777777777778e-06, + "loss": 0.548325777053833, + "mean_token_accuracy": 0.8162619814276695, + "num_tokens": 10817880.0, + "step": 2040 + }, + { + "entropy": 0.5396009232848883, + "epoch": 4.316288877174486, + "grad_norm": 6.8125, + "learning_rate": 4.008888888888889e-06, + "loss": 0.5268721103668212, + "mean_token_accuracy": 0.8218203999102116, + "num_tokens": 10870365.0, + "step": 2050 + }, + { + "entropy": 0.5537472911179065, + "epoch": 4.337374802319451, + "grad_norm": 6.65625, + "learning_rate": 3.920000000000001e-06, + "loss": 0.5522951126098633, + "mean_token_accuracy": 0.8159066334366798, + "num_tokens": 10922363.0, + "step": 2060 + }, + { + "entropy": 0.5330416887998581, + "epoch": 4.358460727464418, + "grad_norm": 7.21875, + "learning_rate": 3.831111111111112e-06, + "loss": 0.5207313537597656, + "mean_token_accuracy": 0.8252225182950497, + "num_tokens": 10972251.0, + "step": 2070 + }, + { + "entropy": 0.5454271405935287, + "epoch": 4.379546652609383, + "grad_norm": 7.125, + "learning_rate": 3.7422222222222228e-06, + "loss": 0.5341221809387207, + "mean_token_accuracy": 0.8211991436779499, + "num_tokens": 11025983.0, + "step": 2080 + }, + { + "entropy": 0.5519603937864304, + "epoch": 4.400632577754349, + "grad_norm": 7.5625, + "learning_rate": 3.6533333333333336e-06, + "loss": 0.5493678569793701, + "mean_token_accuracy": 0.8145092785358429, + "num_tokens": 11081248.0, + "step": 2090 + }, + { + "entropy": 0.5668473180383444, + "epoch": 4.4217185028993145, + "grad_norm": 7.21875, + "learning_rate": 3.564444444444445e-06, + "loss": 0.5566862106323243, + "mean_token_accuracy": 0.813294805586338, + "num_tokens": 11135369.0, + "step": 2100 + }, + { + "entropy": 0.5516544353216887, + "epoch": 4.442804428044281, + "grad_norm": 7.21875, + "learning_rate": 3.475555555555556e-06, + "loss": 0.5478980541229248, + "mean_token_accuracy": 0.8169082552194595, + "num_tokens": 11190765.0, + "step": 2110 + }, + { + "entropy": 0.5469749603420496, + "epoch": 4.463890353189246, + "grad_norm": 8.1875, + "learning_rate": 3.386666666666667e-06, + "loss": 0.5484558582305908, + "mean_token_accuracy": 0.8153667166829109, + "num_tokens": 11242129.0, + "step": 2120 + }, + { + "entropy": 0.5624320391565562, + "epoch": 4.484976278334212, + "grad_norm": 7.15625, + "learning_rate": 3.297777777777778e-06, + "loss": 0.5431473255157471, + "mean_token_accuracy": 0.8167137958109378, + "num_tokens": 11296409.0, + "step": 2130 + }, + { + "entropy": 0.5575876057147979, + "epoch": 4.5060622034791775, + "grad_norm": 6.625, + "learning_rate": 3.2088888888888893e-06, + "loss": 0.546560525894165, + "mean_token_accuracy": 0.8190927796065808, + "num_tokens": 11351788.0, + "step": 2140 + }, + { + "entropy": 0.5531029619276524, + "epoch": 4.527148128624144, + "grad_norm": 6.6875, + "learning_rate": 3.12e-06, + "loss": 0.5480860233306885, + "mean_token_accuracy": 0.8185123346745968, + "num_tokens": 11405274.0, + "step": 2150 + }, + { + "entropy": 0.5629217986017465, + "epoch": 4.548234053769109, + "grad_norm": 7.40625, + "learning_rate": 3.0311111111111115e-06, + "loss": 0.5423421859741211, + "mean_token_accuracy": 0.8194163963198662, + "num_tokens": 11459305.0, + "step": 2160 + }, + { + "entropy": 0.5574934646487236, + "epoch": 4.569319978914075, + "grad_norm": 7.40625, + "learning_rate": 2.9422222222222224e-06, + "loss": 0.5479726791381836, + "mean_token_accuracy": 0.8106288813054562, + "num_tokens": 11512317.0, + "step": 2170 + }, + { + "entropy": 0.5610745966434478, + "epoch": 4.590405904059041, + "grad_norm": 7.96875, + "learning_rate": 2.8533333333333337e-06, + "loss": 0.5556921005249024, + "mean_token_accuracy": 0.8137760147452354, + "num_tokens": 11566146.0, + "step": 2180 + }, + { + "entropy": 0.5264041472226382, + "epoch": 4.611491829204006, + "grad_norm": 6.71875, + "learning_rate": 2.7644444444444446e-06, + "loss": 0.535123348236084, + "mean_token_accuracy": 0.8231086760759354, + "num_tokens": 11619757.0, + "step": 2190 + }, + { + "entropy": 0.5513654734939337, + "epoch": 4.632577754348972, + "grad_norm": 7.28125, + "learning_rate": 2.675555555555556e-06, + "loss": 0.5477256774902344, + "mean_token_accuracy": 0.8165041409432888, + "num_tokens": 11673528.0, + "step": 2200 + }, + { + "entropy": 0.5440514117479325, + "epoch": 4.653663679493937, + "grad_norm": 7.15625, + "learning_rate": 2.5866666666666667e-06, + "loss": 0.5353331565856934, + "mean_token_accuracy": 0.8190776027739048, + "num_tokens": 11728320.0, + "step": 2210 + }, + { + "entropy": 0.5514211151748896, + "epoch": 4.674749604638904, + "grad_norm": 6.6875, + "learning_rate": 2.497777777777778e-06, + "loss": 0.5415604114532471, + "mean_token_accuracy": 0.8205962501466274, + "num_tokens": 11780028.0, + "step": 2220 + }, + { + "entropy": 0.5382947374135256, + "epoch": 4.695835529783869, + "grad_norm": 8.375, + "learning_rate": 2.4088888888888894e-06, + "loss": 0.5256577491760254, + "mean_token_accuracy": 0.8235763169825077, + "num_tokens": 11830895.0, + "step": 2230 + }, + { + "entropy": 0.54562176913023, + "epoch": 4.716921454928835, + "grad_norm": 7.1875, + "learning_rate": 2.3200000000000002e-06, + "loss": 0.5267295837402344, + "mean_token_accuracy": 0.8207360960543155, + "num_tokens": 11882155.0, + "step": 2240 + }, + { + "entropy": 0.5469693087041378, + "epoch": 4.7380073800738005, + "grad_norm": 7.28125, + "learning_rate": 2.2311111111111115e-06, + "loss": 0.5502583026885987, + "mean_token_accuracy": 0.812764473259449, + "num_tokens": 11936554.0, + "step": 2250 + }, + { + "entropy": 0.5447085045278073, + "epoch": 4.759093305218767, + "grad_norm": 6.5, + "learning_rate": 2.1422222222222224e-06, + "loss": 0.5389336585998535, + "mean_token_accuracy": 0.820573166757822, + "num_tokens": 11988156.0, + "step": 2260 + }, + { + "entropy": 0.5513703927397728, + "epoch": 4.780179230363732, + "grad_norm": 7.59375, + "learning_rate": 2.0533333333333337e-06, + "loss": 0.5426520347595215, + "mean_token_accuracy": 0.8161885485053062, + "num_tokens": 12041543.0, + "step": 2270 + }, + { + "entropy": 0.5453945960849523, + "epoch": 4.801265155508698, + "grad_norm": 8.3125, + "learning_rate": 1.9644444444444446e-06, + "loss": 0.5514724731445313, + "mean_token_accuracy": 0.8171669006347656, + "num_tokens": 12092858.0, + "step": 2280 + }, + { + "entropy": 0.5482615809887648, + "epoch": 4.822351080653664, + "grad_norm": 8.1875, + "learning_rate": 1.8755555555555557e-06, + "loss": 0.5525861263275147, + "mean_token_accuracy": 0.8130425401031971, + "num_tokens": 12144862.0, + "step": 2290 + }, + { + "entropy": 0.5636871963739395, + "epoch": 4.84343700579863, + "grad_norm": 7.34375, + "learning_rate": 1.7866666666666668e-06, + "loss": 0.5567370414733886, + "mean_token_accuracy": 0.8114767424762249, + "num_tokens": 12197932.0, + "step": 2300 + }, + { + "entropy": 0.5445570610463619, + "epoch": 4.864522930943595, + "grad_norm": 6.875, + "learning_rate": 1.6977777777777779e-06, + "loss": 0.5326248645782471, + "mean_token_accuracy": 0.8205311246216297, + "num_tokens": 12249536.0, + "step": 2310 + }, + { + "entropy": 0.567267632111907, + "epoch": 4.885608856088561, + "grad_norm": 7.125, + "learning_rate": 1.608888888888889e-06, + "loss": 0.5614665031433106, + "mean_token_accuracy": 0.8123429246246815, + "num_tokens": 12302328.0, + "step": 2320 + }, + { + "entropy": 0.5418668888509274, + "epoch": 4.906694781233527, + "grad_norm": 7.09375, + "learning_rate": 1.52e-06, + "loss": 0.5339171409606933, + "mean_token_accuracy": 0.821372539550066, + "num_tokens": 12354907.0, + "step": 2330 + }, + { + "entropy": 0.5500333663076162, + "epoch": 4.927780706378492, + "grad_norm": 7.4375, + "learning_rate": 1.4311111111111111e-06, + "loss": 0.5362411499023437, + "mean_token_accuracy": 0.8219335168600083, + "num_tokens": 12409864.0, + "step": 2340 + }, + { + "entropy": 0.551620377600193, + "epoch": 4.948866631523458, + "grad_norm": 7.5, + "learning_rate": 1.3422222222222222e-06, + "loss": 0.5413876056671143, + "mean_token_accuracy": 0.8180868111550808, + "num_tokens": 12463400.0, + "step": 2350 + }, + { + "entropy": 0.5467831913381815, + "epoch": 4.9699525566684235, + "grad_norm": 7.0, + "learning_rate": 1.2533333333333333e-06, + "loss": 0.5441878318786622, + "mean_token_accuracy": 0.8187509059906006, + "num_tokens": 12515674.0, + "step": 2360 + }, + { + "entropy": 0.5409926638007164, + "epoch": 4.99103848181339, + "grad_norm": 6.65625, + "learning_rate": 1.1644444444444446e-06, + "loss": 0.5396465301513672, + "mean_token_accuracy": 0.8193774409592152, + "num_tokens": 12568384.0, + "step": 2370 + }, + { + "entropy": 0.5556026499819111, + "epoch": 5.010542962572483, + "grad_norm": 7.59375, + "learning_rate": 1.0755555555555557e-06, + "loss": 0.549100399017334, + "mean_token_accuracy": 0.8160284652903274, + "num_tokens": 12616599.0, + "step": 2380 + }, + { + "entropy": 0.5403375245630742, + "epoch": 5.031628887717448, + "grad_norm": 6.6875, + "learning_rate": 9.866666666666668e-07, + "loss": 0.5360908508300781, + "mean_token_accuracy": 0.8230746820569038, + "num_tokens": 12670346.0, + "step": 2390 + }, + { + "entropy": 0.5554473079741001, + "epoch": 5.052714812862415, + "grad_norm": 6.84375, + "learning_rate": 8.977777777777778e-07, + "loss": 0.5354344844818115, + "mean_token_accuracy": 0.82228574603796, + "num_tokens": 12722166.0, + "step": 2400 + }, + { + "entropy": 0.5599666737020016, + "epoch": 5.07380073800738, + "grad_norm": 6.5, + "learning_rate": 8.088888888888889e-07, + "loss": 0.5506783962249756, + "mean_token_accuracy": 0.8167558930814266, + "num_tokens": 12772478.0, + "step": 2410 + }, + { + "entropy": 0.5462040238082408, + "epoch": 5.094886663152346, + "grad_norm": 7.03125, + "learning_rate": 7.2e-07, + "loss": 0.5262123584747315, + "mean_token_accuracy": 0.8223468787968159, + "num_tokens": 12828399.0, + "step": 2420 + }, + { + "entropy": 0.5505207426846027, + "epoch": 5.1159725882973115, + "grad_norm": 7.6875, + "learning_rate": 6.311111111111112e-07, + "loss": 0.5430339813232422, + "mean_token_accuracy": 0.8183211967349052, + "num_tokens": 12882883.0, + "step": 2430 + }, + { + "entropy": 0.5480441506952047, + "epoch": 5.137058513442278, + "grad_norm": 7.4375, + "learning_rate": 5.422222222222223e-07, + "loss": 0.5462999820709229, + "mean_token_accuracy": 0.817756624519825, + "num_tokens": 12931900.0, + "step": 2440 + }, + { + "entropy": 0.5532797310501337, + "epoch": 5.158144438587243, + "grad_norm": 7.625, + "learning_rate": 4.533333333333334e-07, + "loss": 0.5435749530792237, + "mean_token_accuracy": 0.816430265456438, + "num_tokens": 12983728.0, + "step": 2450 + }, + { + "entropy": 0.5532240316271781, + "epoch": 5.179230363732208, + "grad_norm": 7.4375, + "learning_rate": 3.644444444444445e-07, + "loss": 0.556332015991211, + "mean_token_accuracy": 0.8138340756297111, + "num_tokens": 13033055.0, + "step": 2460 + }, + { + "entropy": 0.5663877837359905, + "epoch": 5.200316288877175, + "grad_norm": 7.5, + "learning_rate": 2.7555555555555555e-07, + "loss": 0.5551712036132812, + "mean_token_accuracy": 0.8142496928572655, + "num_tokens": 13088474.0, + "step": 2470 + }, + { + "entropy": 0.540856496989727, + "epoch": 5.22140221402214, + "grad_norm": 6.65625, + "learning_rate": 1.866666666666667e-07, + "loss": 0.5255017280578613, + "mean_token_accuracy": 0.8260384507477283, + "num_tokens": 13143575.0, + "step": 2480 + }, + { + "entropy": 0.5562540978193283, + "epoch": 5.242488139167106, + "grad_norm": 7.21875, + "learning_rate": 9.777777777777779e-08, + "loss": 0.5427064895629883, + "mean_token_accuracy": 0.8161332339048386, + "num_tokens": 13194995.0, + "step": 2490 + }, + { + "entropy": 0.5543773144483566, + "epoch": 5.263574064312071, + "grad_norm": 7.40625, + "learning_rate": 8.88888888888889e-09, + "loss": 0.5387593269348144, + "mean_token_accuracy": 0.8183573611080647, + "num_tokens": 13248069.0, + "step": 2500 + } + ], + "logging_steps": 10, + "max_steps": 2500, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.3554811993811558e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2500/training_args.bin b/checkpoint-2500/training_args.bin new file mode 100644 index 0000000..f138a50 --- /dev/null +++ b/checkpoint-2500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7103b0ce3340ea41aca3b7441d01d7c38791e3d3ce8c9b28c906c9b1d49bbee9 +size 5713 diff --git a/checkpoint-313/chat_template.jinja b/checkpoint-313/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-313/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-313/config.json b/checkpoint-313/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-313/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-313/generation_config.json b/checkpoint-313/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-313/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-313/model.safetensors b/checkpoint-313/model.safetensors new file mode 100644 index 0000000..c6dacf0 --- /dev/null +++ b/checkpoint-313/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10cecf8b5a7da85f2b9fe0e264241fbdae415c2513c18811fcff956007adbe8f +size 3441185608 diff --git a/checkpoint-313/optimizer.pt b/checkpoint-313/optimizer.pt new file mode 100644 index 0000000..0058c0c --- /dev/null +++ b/checkpoint-313/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:454520e51a030e9e7ba240f9118130044116e416f3025d463460f956e4b4a1bc +size 6882572207 diff --git a/checkpoint-313/rng_state.pth b/checkpoint-313/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-313/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-313/scheduler.pt b/checkpoint-313/scheduler.pt new file mode 100644 index 0000000..eecab6e --- /dev/null +++ b/checkpoint-313/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fc7bf16140c6649b98c5629dea8eb8bbb945385e512dcb8ed617e88aee71cf5 +size 1465 diff --git a/checkpoint-313/tokenizer.json b/checkpoint-313/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-313/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-313/tokenizer_config.json b/checkpoint-313/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-313/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-313/trainer_state.json b/checkpoint-313/trainer_state.json new file mode 100644 index 0000000..3d785a9 --- /dev/null +++ b/checkpoint-313/trainer_state.json @@ -0,0 +1,344 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 313, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4538299292325974, + "epoch": 0.03201280512204882, + "grad_norm": 22.75, + "learning_rate": 5.806451612903226e-06, + "loss": 3.070576477050781, + "mean_token_accuracy": 0.4463906198740005, + "num_tokens": 39534.0, + "step": 10 + }, + { + "entropy": 1.7499498546123504, + "epoch": 0.06402561024409764, + "grad_norm": 10.5, + "learning_rate": 1.2258064516129034e-05, + "loss": 2.6519420623779295, + "mean_token_accuracy": 0.46584000922739505, + "num_tokens": 79078.0, + "step": 20 + }, + { + "entropy": 2.058299073576927, + "epoch": 0.09603841536614646, + "grad_norm": 6.53125, + "learning_rate": 1.870967741935484e-05, + "loss": 2.1608741760253904, + "mean_token_accuracy": 0.5163775213062763, + "num_tokens": 118225.0, + "step": 30 + }, + { + "entropy": 1.9837611675262452, + "epoch": 0.12805122048819528, + "grad_norm": 4.3125, + "learning_rate": 1.9432624113475178e-05, + "loss": 2.000233268737793, + "mean_token_accuracy": 0.5434215664863586, + "num_tokens": 158198.0, + "step": 40 + }, + { + "entropy": 1.8247230246663093, + "epoch": 0.1600640256102441, + "grad_norm": 3.9375, + "learning_rate": 1.872340425531915e-05, + "loss": 1.885169219970703, + "mean_token_accuracy": 0.5518886391073465, + "num_tokens": 197695.0, + "step": 50 + }, + { + "entropy": 1.820909820497036, + "epoch": 0.19207683073229292, + "grad_norm": 3.46875, + "learning_rate": 1.801418439716312e-05, + "loss": 1.8881864547729492, + "mean_token_accuracy": 0.5554269794374704, + "num_tokens": 238556.0, + "step": 60 + }, + { + "entropy": 1.8308037787675857, + "epoch": 0.22408963585434175, + "grad_norm": 3.46875, + "learning_rate": 1.7304964539007094e-05, + "loss": 1.8693714141845703, + "mean_token_accuracy": 0.5529689989984036, + "num_tokens": 278868.0, + "step": 70 + }, + { + "entropy": 1.7878538712859153, + "epoch": 0.25610244097639057, + "grad_norm": 3.6875, + "learning_rate": 1.6595744680851064e-05, + "loss": 1.847772216796875, + "mean_token_accuracy": 0.5583747569471598, + "num_tokens": 319510.0, + "step": 80 + }, + { + "entropy": 1.806380895525217, + "epoch": 0.28811524609843936, + "grad_norm": 3.578125, + "learning_rate": 1.5886524822695038e-05, + "loss": 1.8553556442260741, + "mean_token_accuracy": 0.562133489176631, + "num_tokens": 360080.0, + "step": 90 + }, + { + "entropy": 1.7894016727805138, + "epoch": 0.3201280512204882, + "grad_norm": 3.46875, + "learning_rate": 1.5177304964539008e-05, + "loss": 1.8526411056518555, + "mean_token_accuracy": 0.5623987793922425, + "num_tokens": 400106.0, + "step": 100 + }, + { + "entropy": 1.863182120025158, + "epoch": 0.352140856342537, + "grad_norm": 3.34375, + "learning_rate": 1.4468085106382981e-05, + "loss": 1.8998908996582031, + "mean_token_accuracy": 0.5487093910574913, + "num_tokens": 442713.0, + "step": 110 + }, + { + "entropy": 1.8063384279608727, + "epoch": 0.38415366146458585, + "grad_norm": 3.453125, + "learning_rate": 1.3758865248226951e-05, + "loss": 1.8471385955810546, + "mean_token_accuracy": 0.557015884667635, + "num_tokens": 483468.0, + "step": 120 + }, + { + "entropy": 1.8770785540342332, + "epoch": 0.41616646658663464, + "grad_norm": 3.609375, + "learning_rate": 1.3049645390070925e-05, + "loss": 1.9237331390380858, + "mean_token_accuracy": 0.5443044405430555, + "num_tokens": 526232.0, + "step": 130 + }, + { + "entropy": 1.8102150335907936, + "epoch": 0.4481792717086835, + "grad_norm": 3.6875, + "learning_rate": 1.2340425531914895e-05, + "loss": 1.8544883728027344, + "mean_token_accuracy": 0.558148629963398, + "num_tokens": 568427.0, + "step": 140 + }, + { + "entropy": 1.7433957248926162, + "epoch": 0.4801920768307323, + "grad_norm": 3.28125, + "learning_rate": 1.1631205673758865e-05, + "loss": 1.7788505554199219, + "mean_token_accuracy": 0.5669799156486988, + "num_tokens": 608610.0, + "step": 150 + }, + { + "entropy": 1.788873042166233, + "epoch": 0.5122048819527811, + "grad_norm": 3.484375, + "learning_rate": 1.0921985815602838e-05, + "loss": 1.8689699172973633, + "mean_token_accuracy": 0.5592933889478445, + "num_tokens": 649517.0, + "step": 160 + }, + { + "entropy": 1.784371581673622, + "epoch": 0.54421768707483, + "grad_norm": 3.46875, + "learning_rate": 1.0212765957446808e-05, + "loss": 1.8266977310180663, + "mean_token_accuracy": 0.5599350105971098, + "num_tokens": 690274.0, + "step": 170 + }, + { + "entropy": 1.770761302113533, + "epoch": 0.5762304921968787, + "grad_norm": 3.359375, + "learning_rate": 9.503546099290782e-06, + "loss": 1.8199588775634765, + "mean_token_accuracy": 0.5642654895782471, + "num_tokens": 732104.0, + "step": 180 + }, + { + "entropy": 1.785500568151474, + "epoch": 0.6082432973189276, + "grad_norm": 3.359375, + "learning_rate": 8.794326241134753e-06, + "loss": 1.8203857421875, + "mean_token_accuracy": 0.5628614298999309, + "num_tokens": 772388.0, + "step": 190 + }, + { + "entropy": 1.7996786698698997, + "epoch": 0.6402561024409764, + "grad_norm": 3.234375, + "learning_rate": 8.085106382978723e-06, + "loss": 1.838487434387207, + "mean_token_accuracy": 0.5601454164832831, + "num_tokens": 814417.0, + "step": 200 + }, + { + "entropy": 1.7707496628165245, + "epoch": 0.6722689075630253, + "grad_norm": 3.5625, + "learning_rate": 7.375886524822695e-06, + "loss": 1.8333843231201172, + "mean_token_accuracy": 0.5659072320908308, + "num_tokens": 855220.0, + "step": 210 + }, + { + "entropy": 1.849867083132267, + "epoch": 0.704281712685074, + "grad_norm": 3.25, + "learning_rate": 6.666666666666667e-06, + "loss": 1.9260578155517578, + "mean_token_accuracy": 0.5531776797026395, + "num_tokens": 898972.0, + "step": 220 + }, + { + "entropy": 1.7605908274650575, + "epoch": 0.7362945178071229, + "grad_norm": 3.4375, + "learning_rate": 5.957446808510638e-06, + "loss": 1.7972305297851563, + "mean_token_accuracy": 0.5678223472088575, + "num_tokens": 938284.0, + "step": 230 + }, + { + "entropy": 1.7362026132643222, + "epoch": 0.7683073229291717, + "grad_norm": 3.484375, + "learning_rate": 5.24822695035461e-06, + "loss": 1.7981218338012694, + "mean_token_accuracy": 0.5710738152265549, + "num_tokens": 977880.0, + "step": 240 + }, + { + "entropy": 1.79102204144001, + "epoch": 0.8003201280512204, + "grad_norm": 3.421875, + "learning_rate": 4.539007092198582e-06, + "loss": 1.8328744888305664, + "mean_token_accuracy": 0.5625536382198334, + "num_tokens": 1019549.0, + "step": 250 + }, + { + "entropy": 1.788681410253048, + "epoch": 0.8323329331732693, + "grad_norm": 3.375, + "learning_rate": 3.8297872340425535e-06, + "loss": 1.8353548049926758, + "mean_token_accuracy": 0.5642140224575997, + "num_tokens": 1060145.0, + "step": 260 + }, + { + "entropy": 1.8368026711046697, + "epoch": 0.8643457382953181, + "grad_norm": 3.28125, + "learning_rate": 3.120567375886525e-06, + "loss": 1.9220146179199218, + "mean_token_accuracy": 0.5521392654627562, + "num_tokens": 1102818.0, + "step": 270 + }, + { + "entropy": 1.7728961274027824, + "epoch": 0.896358543417367, + "grad_norm": 3.546875, + "learning_rate": 2.4113475177304965e-06, + "loss": 1.8067329406738282, + "mean_token_accuracy": 0.5586034987121821, + "num_tokens": 1143300.0, + "step": 280 + }, + { + "entropy": 1.7760732486844062, + "epoch": 0.9283713485394157, + "grad_norm": 3.328125, + "learning_rate": 1.7021276595744682e-06, + "loss": 1.8068740844726563, + "mean_token_accuracy": 0.5682683397084475, + "num_tokens": 1183341.0, + "step": 290 + }, + { + "entropy": 1.7398153685033322, + "epoch": 0.9603841536614646, + "grad_norm": 3.75, + "learning_rate": 9.929078014184399e-07, + "loss": 1.7799579620361328, + "mean_token_accuracy": 0.5732921231538057, + "num_tokens": 1223298.0, + "step": 300 + }, + { + "entropy": 1.8165026500821113, + "epoch": 0.9923969587835134, + "grad_norm": 3.265625, + "learning_rate": 2.8368794326241136e-07, + "loss": 1.875440788269043, + "mean_token_accuracy": 0.5596334230154753, + "num_tokens": 1265400.0, + "step": 310 + } + ], + "logging_steps": 10, + "max_steps": 313, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.2815731740622848e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-313/training_args.bin b/checkpoint-313/training_args.bin new file mode 100644 index 0000000..8a0a0d2 --- /dev/null +++ b/checkpoint-313/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaf9e6725ef566099e3d00b3940e51a6fb4e35f9a95932e9fe932512dfd8044d +size 5713 diff --git a/checkpoint-32/chat_template.jinja b/checkpoint-32/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-32/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-32/config.json b/checkpoint-32/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-32/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-32/generation_config.json b/checkpoint-32/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-32/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-32/model.safetensors b/checkpoint-32/model.safetensors new file mode 100644 index 0000000..40a4268 --- /dev/null +++ b/checkpoint-32/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa6088ebce294b696349ce6fd8b85740a1ee100440c1c003b221396a618aebb5 +size 3441185608 diff --git a/checkpoint-32/optimizer.pt b/checkpoint-32/optimizer.pt new file mode 100644 index 0000000..5d5ad28 --- /dev/null +++ b/checkpoint-32/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64065e7b30c2ac7efd0acf1ee72d144ab0fe368c19910d9776907e1193168bf0 +size 6882572207 diff --git a/checkpoint-32/rng_state.pth b/checkpoint-32/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-32/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-32/scheduler.pt b/checkpoint-32/scheduler.pt new file mode 100644 index 0000000..152fe92 --- /dev/null +++ b/checkpoint-32/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08c138eb062a03f4d8ee078d0df4767cf6607757a5547a83bd0aafe7e70d4e04 +size 1465 diff --git a/checkpoint-32/tokenizer.json b/checkpoint-32/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-32/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-32/tokenizer_config.json b/checkpoint-32/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-32/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-32/trainer_state.json b/checkpoint-32/trainer_state.json new file mode 100644 index 0000000..c8de08c --- /dev/null +++ b/checkpoint-32/trainer_state.json @@ -0,0 +1,64 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 32, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4525227755308152, + "epoch": 0.32, + "grad_norm": 11.375, + "learning_rate": 1.586206896551724e-05, + "loss": 3.044580841064453, + "mean_token_accuracy": 0.5073798138648271, + "num_tokens": 34236.0, + "step": 10 + }, + { + "entropy": 1.3969161085784436, + "epoch": 0.64, + "grad_norm": 6.78125, + "learning_rate": 8.965517241379312e-06, + "loss": 1.4662676811218263, + "mean_token_accuracy": 0.6803411647677422, + "num_tokens": 69411.0, + "step": 20 + }, + { + "entropy": 1.2149690441787242, + "epoch": 0.96, + "grad_norm": 7.21875, + "learning_rate": 2.0689655172413796e-06, + "loss": 1.3216065406799316, + "mean_token_accuracy": 0.7025610633194447, + "num_tokens": 104303.0, + "step": 30 + } + ], + "logging_steps": 10, + "max_steps": 32, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1038690104254464.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-32/training_args.bin b/checkpoint-32/training_args.bin new file mode 100644 index 0000000..99599ce --- /dev/null +++ b/checkpoint-32/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bef5ac8f7dc7e70d45d97875e64c2ce84fdf5ced9b88c53205552ee0e85f8903 +size 5713 diff --git a/checkpoint-378/chat_template.jinja b/checkpoint-378/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-378/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-378/config.json b/checkpoint-378/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-378/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-378/generation_config.json b/checkpoint-378/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-378/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-378/model.safetensors b/checkpoint-378/model.safetensors new file mode 100644 index 0000000..be3336d --- /dev/null +++ b/checkpoint-378/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53d9d0b3554758146b2b7b4701c0c7d8cb9f561498bf840324dd6592b6e3a648 +size 3441185608 diff --git a/checkpoint-378/optimizer.pt b/checkpoint-378/optimizer.pt new file mode 100644 index 0000000..6536f96 --- /dev/null +++ b/checkpoint-378/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1f7b95e7cef31fe83ab76c41bbffb096ebebf563d74dfb44b15d685eb3f7976 +size 6882572207 diff --git a/checkpoint-378/rng_state.pth b/checkpoint-378/rng_state.pth new file mode 100644 index 0000000..1feba1a --- /dev/null +++ b/checkpoint-378/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878 +size 14645 diff --git a/checkpoint-378/scheduler.pt b/checkpoint-378/scheduler.pt new file mode 100644 index 0000000..311a09d --- /dev/null +++ b/checkpoint-378/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c127f395b7227eea2bb4925cd661a8c770b8f49fb02a217376a2299885ecbe7 +size 1465 diff --git a/checkpoint-378/tokenizer.json b/checkpoint-378/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-378/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-378/tokenizer_config.json b/checkpoint-378/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-378/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-378/trainer_state.json b/checkpoint-378/trainer_state.json new file mode 100644 index 0000000..0476b8c --- /dev/null +++ b/checkpoint-378/trainer_state.json @@ -0,0 +1,404 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 378, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.9081946484744549, + "epoch": 0.053015241882041084, + "grad_norm": 51.5, + "learning_rate": 3e-06, + "loss": 2.320839500427246, + "mean_token_accuracy": 0.5708254482597113, + "num_tokens": 30761.0, + "step": 10 + }, + { + "entropy": 1.032745758444071, + "epoch": 0.10603048376408217, + "grad_norm": 21.125, + "learning_rate": 6.333333333333333e-06, + "loss": 1.9631355285644532, + "mean_token_accuracy": 0.5809337079524994, + "num_tokens": 61277.0, + "step": 20 + }, + { + "entropy": 1.2370933540165425, + "epoch": 0.15904572564612326, + "grad_norm": 11.0625, + "learning_rate": 9.666666666666667e-06, + "loss": 1.4653306961059571, + "mean_token_accuracy": 0.6311864696443081, + "num_tokens": 91730.0, + "step": 30 + }, + { + "entropy": 1.1646109029650689, + "epoch": 0.21206096752816433, + "grad_norm": 8.5625, + "learning_rate": 1.3000000000000001e-05, + "loss": 1.200753402709961, + "mean_token_accuracy": 0.6730783767998219, + "num_tokens": 122896.0, + "step": 40 + }, + { + "entropy": 1.0138436160981654, + "epoch": 0.26507620941020543, + "grad_norm": 7.4375, + "learning_rate": 1.6333333333333335e-05, + "loss": 1.0109354972839355, + "mean_token_accuracy": 0.7080431424081326, + "num_tokens": 153323.0, + "step": 50 + }, + { + "entropy": 0.8669569574296474, + "epoch": 0.31809145129224653, + "grad_norm": 6.75, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8859419822692871, + "mean_token_accuracy": 0.7356305047869682, + "num_tokens": 183928.0, + "step": 60 + }, + { + "entropy": 0.856675973534584, + "epoch": 0.3711066931742876, + "grad_norm": 7.6875, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8721119880676269, + "mean_token_accuracy": 0.7318881675601006, + "num_tokens": 215054.0, + "step": 70 + }, + { + "entropy": 0.7872001446783543, + "epoch": 0.42412193505632867, + "grad_norm": 7.46875, + "learning_rate": 1.92962962962963e-05, + "loss": 0.7920477390289307, + "mean_token_accuracy": 0.7553018122911453, + "num_tokens": 245614.0, + "step": 80 + }, + { + "entropy": 0.7739131577312947, + "epoch": 0.47713717693836977, + "grad_norm": 7.09375, + "learning_rate": 1.8925925925925928e-05, + "loss": 0.7949441432952881, + "mean_token_accuracy": 0.7514781631529331, + "num_tokens": 276651.0, + "step": 90 + }, + { + "entropy": 0.7763891488313674, + "epoch": 0.5301524188204109, + "grad_norm": 7.8125, + "learning_rate": 1.8555555555555557e-05, + "loss": 0.7867375373840332, + "mean_token_accuracy": 0.7523738898336887, + "num_tokens": 308561.0, + "step": 100 + }, + { + "entropy": 0.7757978036999702, + "epoch": 0.583167660702452, + "grad_norm": 7.21875, + "learning_rate": 1.8185185185185186e-05, + "loss": 0.7776416301727295, + "mean_token_accuracy": 0.7548527352511882, + "num_tokens": 339537.0, + "step": 110 + }, + { + "entropy": 0.7528716944158077, + "epoch": 0.6361829025844931, + "grad_norm": 6.28125, + "learning_rate": 1.7814814814814815e-05, + "loss": 0.7566296100616455, + "mean_token_accuracy": 0.7637521453201771, + "num_tokens": 370938.0, + "step": 120 + }, + { + "entropy": 0.7483146995306015, + "epoch": 0.6891981444665342, + "grad_norm": 7.0, + "learning_rate": 1.7444444444444448e-05, + "loss": 0.7508650302886963, + "mean_token_accuracy": 0.758705198019743, + "num_tokens": 402708.0, + "step": 130 + }, + { + "entropy": 0.7206243231892586, + "epoch": 0.7422133863485753, + "grad_norm": 6.65625, + "learning_rate": 1.7074074074074077e-05, + "loss": 0.7531259536743165, + "mean_token_accuracy": 0.7641572073101998, + "num_tokens": 433645.0, + "step": 140 + }, + { + "entropy": 0.7596748031675815, + "epoch": 0.7952286282306164, + "grad_norm": 6.03125, + "learning_rate": 1.6703703703703703e-05, + "loss": 0.7591583728790283, + "mean_token_accuracy": 0.7570627965033054, + "num_tokens": 465791.0, + "step": 150 + }, + { + "entropy": 0.7133973561227321, + "epoch": 0.8482438701126573, + "grad_norm": 7.125, + "learning_rate": 1.6333333333333335e-05, + "loss": 0.7458220005035401, + "mean_token_accuracy": 0.7626615799963474, + "num_tokens": 497296.0, + "step": 160 + }, + { + "entropy": 0.7134327009320259, + "epoch": 0.9012591119946984, + "grad_norm": 7.0, + "learning_rate": 1.5962962962962964e-05, + "loss": 0.7290266990661621, + "mean_token_accuracy": 0.7681502625346184, + "num_tokens": 528181.0, + "step": 170 + }, + { + "entropy": 0.7324420973658562, + "epoch": 0.9542743538767395, + "grad_norm": 7.5, + "learning_rate": 1.5592592592592593e-05, + "loss": 0.731742525100708, + "mean_token_accuracy": 0.7631207950413227, + "num_tokens": 558915.0, + "step": 180 + }, + { + "entropy": 0.7100333733992144, + "epoch": 1.0053015241882042, + "grad_norm": 8.3125, + "learning_rate": 1.5222222222222223e-05, + "loss": 0.7303330898284912, + "mean_token_accuracy": 0.7633489200047084, + "num_tokens": 588567.0, + "step": 190 + }, + { + "entropy": 0.6628884050995112, + "epoch": 1.0583167660702453, + "grad_norm": 7.0625, + "learning_rate": 1.4851851851851853e-05, + "loss": 0.6481011867523193, + "mean_token_accuracy": 0.7933171860873699, + "num_tokens": 619081.0, + "step": 200 + }, + { + "entropy": 0.6257969986647367, + "epoch": 1.1113320079522864, + "grad_norm": 7.25, + "learning_rate": 1.4481481481481483e-05, + "loss": 0.642666244506836, + "mean_token_accuracy": 0.7921255677938461, + "num_tokens": 650402.0, + "step": 210 + }, + { + "entropy": 0.6399037420749665, + "epoch": 1.1643472498343272, + "grad_norm": 6.03125, + "learning_rate": 1.4111111111111113e-05, + "loss": 0.6525338172912598, + "mean_token_accuracy": 0.7887163788080216, + "num_tokens": 681837.0, + "step": 220 + }, + { + "entropy": 0.6367121540009976, + "epoch": 1.2173624917163686, + "grad_norm": 7.59375, + "learning_rate": 1.3740740740740741e-05, + "loss": 0.6356833457946778, + "mean_token_accuracy": 0.7893597863614559, + "num_tokens": 712571.0, + "step": 230 + }, + { + "entropy": 0.6334792256355286, + "epoch": 1.2703777335984094, + "grad_norm": 6.6875, + "learning_rate": 1.3370370370370372e-05, + "loss": 0.6391499996185303, + "mean_token_accuracy": 0.7947010740637779, + "num_tokens": 743396.0, + "step": 240 + }, + { + "entropy": 0.6417076423764229, + "epoch": 1.3233929754804508, + "grad_norm": 7.4375, + "learning_rate": 1.3000000000000001e-05, + "loss": 0.6563210010528564, + "mean_token_accuracy": 0.787147381901741, + "num_tokens": 774883.0, + "step": 250 + }, + { + "entropy": 0.6213644035160542, + "epoch": 1.3764082173624916, + "grad_norm": 7.1875, + "learning_rate": 1.2629629629629632e-05, + "loss": 0.6295370578765869, + "mean_token_accuracy": 0.7955153584480286, + "num_tokens": 805520.0, + "step": 260 + }, + { + "entropy": 0.6552363462746144, + "epoch": 1.4294234592445327, + "grad_norm": 6.84375, + "learning_rate": 1.225925925925926e-05, + "loss": 0.6632986068725586, + "mean_token_accuracy": 0.7826757229864597, + "num_tokens": 836758.0, + "step": 270 + }, + { + "entropy": 0.6550090927630663, + "epoch": 1.4824387011265738, + "grad_norm": 7.25, + "learning_rate": 1.188888888888889e-05, + "loss": 0.6479949474334716, + "mean_token_accuracy": 0.78950335085392, + "num_tokens": 868056.0, + "step": 280 + }, + { + "entropy": 0.6275094959884882, + "epoch": 1.535453943008615, + "grad_norm": 6.4375, + "learning_rate": 1.151851851851852e-05, + "loss": 0.6384372234344482, + "mean_token_accuracy": 0.7930210866034031, + "num_tokens": 899420.0, + "step": 290 + }, + { + "entropy": 0.6052290312945843, + "epoch": 1.588469184890656, + "grad_norm": 6.875, + "learning_rate": 1.114814814814815e-05, + "loss": 0.609743309020996, + "mean_token_accuracy": 0.7990770071744919, + "num_tokens": 929571.0, + "step": 300 + }, + { + "entropy": 0.6174360021948815, + "epoch": 1.6414844267726971, + "grad_norm": 7.0625, + "learning_rate": 1.0777777777777778e-05, + "loss": 0.6313478946685791, + "mean_token_accuracy": 0.7964761383831501, + "num_tokens": 960928.0, + "step": 310 + }, + { + "entropy": 0.6127162992954254, + "epoch": 1.6944996686547382, + "grad_norm": 6.5625, + "learning_rate": 1.0407407407407408e-05, + "loss": 0.6150648593902588, + "mean_token_accuracy": 0.8008938111364842, + "num_tokens": 991961.0, + "step": 320 + }, + { + "entropy": 0.5947048265486956, + "epoch": 1.7475149105367793, + "grad_norm": 7.65625, + "learning_rate": 1.003703703703704e-05, + "loss": 0.6053301334381104, + "mean_token_accuracy": 0.801536674797535, + "num_tokens": 1022013.0, + "step": 330 + }, + { + "entropy": 0.6275908622890711, + "epoch": 1.8005301524188204, + "grad_norm": 6.9375, + "learning_rate": 9.666666666666667e-06, + "loss": 0.6307861804962158, + "mean_token_accuracy": 0.7918523952364922, + "num_tokens": 1053061.0, + "step": 340 + }, + { + "entropy": 0.6406017865985631, + "epoch": 1.8535453943008615, + "grad_norm": 7.0, + "learning_rate": 9.296296296296296e-06, + "loss": 0.6534273147583007, + "mean_token_accuracy": 0.789715650677681, + "num_tokens": 1085392.0, + "step": 350 + }, + { + "entropy": 0.6285702683031559, + "epoch": 1.9065606361829026, + "grad_norm": 7.8125, + "learning_rate": 8.925925925925927e-06, + "loss": 0.6331883907318115, + "mean_token_accuracy": 0.7924636773765087, + "num_tokens": 1117204.0, + "step": 360 + }, + { + "entropy": 0.615465309098363, + "epoch": 1.9595758780649437, + "grad_norm": 7.25, + "learning_rate": 8.555555555555556e-06, + "loss": 0.6278977870941163, + "mean_token_accuracy": 0.7965366452932358, + "num_tokens": 1147711.0, + "step": 370 + } + ], + "logging_steps": 10, + "max_steps": 600, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.131034650599424e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-378/training_args.bin b/checkpoint-378/training_args.bin new file mode 100644 index 0000000..6618e60 --- /dev/null +++ b/checkpoint-378/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c0557c0612cd65ef8abf2a6c3d767d5aa1d732f9c315d62b560fc3d0f5beafe +size 5713 diff --git a/checkpoint-475/chat_template.jinja b/checkpoint-475/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-475/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-475/config.json b/checkpoint-475/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-475/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-475/generation_config.json b/checkpoint-475/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-475/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-475/model.safetensors b/checkpoint-475/model.safetensors new file mode 100644 index 0000000..083664c --- /dev/null +++ b/checkpoint-475/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:951f2b23ef2764a406e964232deb576dbdac03ecc12a915a7b6769a28bb6b0d3 +size 3441185608 diff --git a/checkpoint-475/optimizer.pt b/checkpoint-475/optimizer.pt new file mode 100644 index 0000000..3d377e5 --- /dev/null +++ b/checkpoint-475/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5c52d266003d1fcf6508385fbefb47376a16b44545120a5a24b22238218b763 +size 6882572207 diff --git a/checkpoint-475/rng_state.pth b/checkpoint-475/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-475/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-475/scheduler.pt b/checkpoint-475/scheduler.pt new file mode 100644 index 0000000..9e848e9 --- /dev/null +++ b/checkpoint-475/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44b16da41da49faf8d18f42bb521bad9cc5ae54d5e5510c0cebaefaeee8251fb +size 1465 diff --git a/checkpoint-475/tokenizer.json b/checkpoint-475/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-475/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-475/tokenizer_config.json b/checkpoint-475/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-475/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-475/trainer_state.json b/checkpoint-475/trainer_state.json new file mode 100644 index 0000000..7584852 --- /dev/null +++ b/checkpoint-475/trainer_state.json @@ -0,0 +1,504 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 475, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.8691443920135498, + "epoch": 0.021085925144965736, + "grad_norm": 47.5, + "learning_rate": 7.2e-07, + "loss": 2.1268569946289064, + "mean_token_accuracy": 0.5856592752039432, + "num_tokens": 53385.0, + "step": 10 + }, + { + "entropy": 0.9018128983676433, + "epoch": 0.04217185028993147, + "grad_norm": 49.25, + "learning_rate": 1.52e-06, + "loss": 2.1235822677612304, + "mean_token_accuracy": 0.5826416034251451, + "num_tokens": 108949.0, + "step": 20 + }, + { + "entropy": 0.9313402399420738, + "epoch": 0.0632577754348972, + "grad_norm": 38.25, + "learning_rate": 2.3200000000000002e-06, + "loss": 1.9997358322143555, + "mean_token_accuracy": 0.5817345686256885, + "num_tokens": 162034.0, + "step": 30 + }, + { + "entropy": 0.9819381944835186, + "epoch": 0.08434370057986294, + "grad_norm": 28.25, + "learning_rate": 3.12e-06, + "loss": 1.8760208129882812, + "mean_token_accuracy": 0.5874959267675877, + "num_tokens": 214165.0, + "step": 40 + }, + { + "entropy": 1.0624087303876877, + "epoch": 0.10542962572482868, + "grad_norm": 14.1875, + "learning_rate": 3.920000000000001e-06, + "loss": 1.6169570922851562, + "mean_token_accuracy": 0.601640059798956, + "num_tokens": 265527.0, + "step": 50 + }, + { + "entropy": 1.1721510097384453, + "epoch": 0.1265155508697944, + "grad_norm": 11.3125, + "learning_rate": 4.7200000000000005e-06, + "loss": 1.4622878074645995, + "mean_token_accuracy": 0.6209174185991287, + "num_tokens": 319341.0, + "step": 60 + }, + { + "entropy": 1.2009219758212566, + "epoch": 0.14760147601476015, + "grad_norm": 9.0625, + "learning_rate": 5.5200000000000005e-06, + "loss": 1.2933337211608886, + "mean_token_accuracy": 0.6467074818909169, + "num_tokens": 373709.0, + "step": 70 + }, + { + "entropy": 1.1439887225627898, + "epoch": 0.16868740115972589, + "grad_norm": 9.3125, + "learning_rate": 6.3200000000000005e-06, + "loss": 1.2084498405456543, + "mean_token_accuracy": 0.6660686552524566, + "num_tokens": 425923.0, + "step": 80 + }, + { + "entropy": 1.0713621236383915, + "epoch": 0.18977332630469163, + "grad_norm": 7.84375, + "learning_rate": 7.1200000000000004e-06, + "loss": 1.1175435066223145, + "mean_token_accuracy": 0.6812180563807487, + "num_tokens": 480012.0, + "step": 90 + }, + { + "entropy": 1.0369222231209279, + "epoch": 0.21085925144965736, + "grad_norm": 7.09375, + "learning_rate": 7.92e-06, + "loss": 1.062850570678711, + "mean_token_accuracy": 0.6899570092558861, + "num_tokens": 533890.0, + "step": 100 + }, + { + "entropy": 0.9894454136490822, + "epoch": 0.23194517659462308, + "grad_norm": 7.75, + "learning_rate": 8.720000000000001e-06, + "loss": 1.0150799751281738, + "mean_token_accuracy": 0.7009239844977856, + "num_tokens": 590654.0, + "step": 110 + }, + { + "entropy": 0.9410129584372043, + "epoch": 0.2530311017395888, + "grad_norm": 7.28125, + "learning_rate": 9.52e-06, + "loss": 0.9810515403747558, + "mean_token_accuracy": 0.709618617594242, + "num_tokens": 643080.0, + "step": 120 + }, + { + "entropy": 0.9265115469694137, + "epoch": 0.2741170268845546, + "grad_norm": 7.46875, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.9485825538635254, + "mean_token_accuracy": 0.7162063598632813, + "num_tokens": 697254.0, + "step": 130 + }, + { + "entropy": 0.9009825974702835, + "epoch": 0.2952029520295203, + "grad_norm": 6.96875, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.9119878768920898, + "mean_token_accuracy": 0.7238413639366627, + "num_tokens": 751774.0, + "step": 140 + }, + { + "entropy": 0.9040703341364861, + "epoch": 0.316288877174486, + "grad_norm": 7.21875, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.9283308029174805, + "mean_token_accuracy": 0.7166316144168376, + "num_tokens": 808927.0, + "step": 150 + }, + { + "entropy": 0.8878034949302673, + "epoch": 0.33737480231945177, + "grad_norm": 7.375, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.9068665504455566, + "mean_token_accuracy": 0.7254431843757629, + "num_tokens": 858559.0, + "step": 160 + }, + { + "entropy": 0.887480016052723, + "epoch": 0.3584607274644175, + "grad_norm": 7.65625, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.8971159934997559, + "mean_token_accuracy": 0.7258390121161937, + "num_tokens": 912285.0, + "step": 170 + }, + { + "entropy": 0.8527634374797344, + "epoch": 0.37954665260938325, + "grad_norm": 7.6875, + "learning_rate": 1.432e-05, + "loss": 0.8586221694946289, + "mean_token_accuracy": 0.7359919100999832, + "num_tokens": 969385.0, + "step": 180 + }, + { + "entropy": 0.8445627830922604, + "epoch": 0.40063257775434896, + "grad_norm": 7.03125, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.855714225769043, + "mean_token_accuracy": 0.7341729424893856, + "num_tokens": 1026106.0, + "step": 190 + }, + { + "entropy": 0.841130618005991, + "epoch": 0.42171850289931473, + "grad_norm": 7.21875, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.8620488166809082, + "mean_token_accuracy": 0.7361006513237953, + "num_tokens": 1079743.0, + "step": 200 + }, + { + "entropy": 0.831258487701416, + "epoch": 0.44280442804428044, + "grad_norm": 7.0, + "learning_rate": 1.672e-05, + "loss": 0.8441590309143067, + "mean_token_accuracy": 0.7379129216074943, + "num_tokens": 1131453.0, + "step": 210 + }, + { + "entropy": 0.79731200709939, + "epoch": 0.46389035318924615, + "grad_norm": 7.875, + "learning_rate": 1.752e-05, + "loss": 0.8135190963745117, + "mean_token_accuracy": 0.7497627973556519, + "num_tokens": 1182486.0, + "step": 220 + }, + { + "entropy": 0.8269097700715065, + "epoch": 0.4849762783342119, + "grad_norm": 7.5625, + "learning_rate": 1.832e-05, + "loss": 0.8399410247802734, + "mean_token_accuracy": 0.7389348305761814, + "num_tokens": 1238136.0, + "step": 230 + }, + { + "entropy": 0.8093138128519058, + "epoch": 0.5060622034791776, + "grad_norm": 7.40625, + "learning_rate": 1.912e-05, + "loss": 0.810025691986084, + "mean_token_accuracy": 0.7460841409862041, + "num_tokens": 1294939.0, + "step": 240 + }, + { + "entropy": 0.8069386176764966, + "epoch": 0.5271481286241434, + "grad_norm": 6.84375, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8215888023376465, + "mean_token_accuracy": 0.7438369721174241, + "num_tokens": 1346462.0, + "step": 250 + }, + { + "entropy": 0.8088659413158894, + "epoch": 0.5482340537691092, + "grad_norm": 6.75, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8201449394226075, + "mean_token_accuracy": 0.7393839374184609, + "num_tokens": 1397444.0, + "step": 260 + }, + { + "entropy": 0.7969542622566224, + "epoch": 0.5693199789140748, + "grad_norm": 6.59375, + "learning_rate": 1.9831111111111113e-05, + "loss": 0.8009759902954101, + "mean_token_accuracy": 0.7512488305568695, + "num_tokens": 1453900.0, + "step": 270 + }, + { + "entropy": 0.780416651815176, + "epoch": 0.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.9742222222222223e-05, + "loss": 0.7874263763427735, + "mean_token_accuracy": 0.7526524804532528, + "num_tokens": 1505529.0, + "step": 280 + }, + { + "entropy": 0.7752278268337249, + "epoch": 0.6114918292040064, + "grad_norm": 6.875, + "learning_rate": 1.9653333333333334e-05, + "loss": 0.8075678825378418, + "mean_token_accuracy": 0.746569037437439, + "num_tokens": 1562473.0, + "step": 290 + }, + { + "entropy": 0.7895354829728604, + "epoch": 0.632577754348972, + "grad_norm": 6.6875, + "learning_rate": 1.9564444444444444e-05, + "loss": 0.7880948543548584, + "mean_token_accuracy": 0.7494045957922936, + "num_tokens": 1613320.0, + "step": 300 + }, + { + "entropy": 0.7653828673064709, + "epoch": 0.6536636794939378, + "grad_norm": 6.78125, + "learning_rate": 1.9475555555555558e-05, + "loss": 0.7780828475952148, + "mean_token_accuracy": 0.7556171745061875, + "num_tokens": 1662847.0, + "step": 310 + }, + { + "entropy": 0.7652887552976608, + "epoch": 0.6747496046389035, + "grad_norm": 6.375, + "learning_rate": 1.938666666666667e-05, + "loss": 0.7539787292480469, + "mean_token_accuracy": 0.7582654722034932, + "num_tokens": 1718553.0, + "step": 320 + }, + { + "entropy": 0.7076033115386963, + "epoch": 0.6958355297838693, + "grad_norm": 7.15625, + "learning_rate": 1.929777777777778e-05, + "loss": 0.7396605491638184, + "mean_token_accuracy": 0.7659220136702061, + "num_tokens": 1771024.0, + "step": 330 + }, + { + "entropy": 0.7509547933936119, + "epoch": 0.716921454928835, + "grad_norm": 6.15625, + "learning_rate": 1.920888888888889e-05, + "loss": 0.7459388256072998, + "mean_token_accuracy": 0.7644543245434761, + "num_tokens": 1821303.0, + "step": 340 + }, + { + "entropy": 0.7591827407479286, + "epoch": 0.7380073800738007, + "grad_norm": 7.03125, + "learning_rate": 1.912e-05, + "loss": 0.7627717971801757, + "mean_token_accuracy": 0.7567056350409984, + "num_tokens": 1868674.0, + "step": 350 + }, + { + "entropy": 0.7316406652331352, + "epoch": 0.7590933052187665, + "grad_norm": 6.59375, + "learning_rate": 1.9031111111111114e-05, + "loss": 0.7425707340240478, + "mean_token_accuracy": 0.7653280735015869, + "num_tokens": 1924619.0, + "step": 360 + }, + { + "entropy": 0.7539067789912224, + "epoch": 0.7801792303637322, + "grad_norm": 7.0625, + "learning_rate": 1.8942222222222224e-05, + "loss": 0.7716987609863282, + "mean_token_accuracy": 0.7524559155106545, + "num_tokens": 1978795.0, + "step": 370 + }, + { + "entropy": 0.7687754213809967, + "epoch": 0.8012651555086979, + "grad_norm": 7.1875, + "learning_rate": 1.8853333333333335e-05, + "loss": 0.7734373569488525, + "mean_token_accuracy": 0.7497503250837326, + "num_tokens": 2029994.0, + "step": 380 + }, + { + "entropy": 0.7508607231080532, + "epoch": 0.8223510806536637, + "grad_norm": 7.09375, + "learning_rate": 1.8764444444444445e-05, + "loss": 0.7575721740722656, + "mean_token_accuracy": 0.7572125047445297, + "num_tokens": 2082653.0, + "step": 390 + }, + { + "entropy": 0.7252861328423024, + "epoch": 0.8434370057986295, + "grad_norm": 6.5625, + "learning_rate": 1.867555555555556e-05, + "loss": 0.7381186485290527, + "mean_token_accuracy": 0.7612978041172027, + "num_tokens": 2135276.0, + "step": 400 + }, + { + "entropy": 0.7339485622942448, + "epoch": 0.8645229309435951, + "grad_norm": 6.625, + "learning_rate": 1.858666666666667e-05, + "loss": 0.7348906993865967, + "mean_token_accuracy": 0.7663561321794987, + "num_tokens": 2187466.0, + "step": 410 + }, + { + "entropy": 0.7272624362260103, + "epoch": 0.8856088560885609, + "grad_norm": 6.90625, + "learning_rate": 1.849777777777778e-05, + "loss": 0.749025821685791, + "mean_token_accuracy": 0.7617379955947399, + "num_tokens": 2237434.0, + "step": 420 + }, + { + "entropy": 0.7477380312979222, + "epoch": 0.9066947812335266, + "grad_norm": 6.34375, + "learning_rate": 1.840888888888889e-05, + "loss": 0.7498275756835937, + "mean_token_accuracy": 0.7604753144085408, + "num_tokens": 2288985.0, + "step": 430 + }, + { + "entropy": 0.7346535548567772, + "epoch": 0.9277807063784923, + "grad_norm": 6.9375, + "learning_rate": 1.832e-05, + "loss": 0.739972448348999, + "mean_token_accuracy": 0.7658773727715016, + "num_tokens": 2341187.0, + "step": 440 + }, + { + "entropy": 0.7167556572705507, + "epoch": 0.9488666315234581, + "grad_norm": 6.4375, + "learning_rate": 1.823111111111111e-05, + "loss": 0.7350784778594971, + "mean_token_accuracy": 0.7644179269671441, + "num_tokens": 2393385.0, + "step": 450 + }, + { + "entropy": 0.7537519961595536, + "epoch": 0.9699525566684238, + "grad_norm": 7.28125, + "learning_rate": 1.814222222222222e-05, + "loss": 0.757652759552002, + "mean_token_accuracy": 0.7587068513035774, + "num_tokens": 2442248.0, + "step": 460 + }, + { + "entropy": 0.72038439437747, + "epoch": 0.9910384818133896, + "grad_norm": 7.40625, + "learning_rate": 1.8053333333333332e-05, + "loss": 0.7361824035644531, + "mean_token_accuracy": 0.7609050884842873, + "num_tokens": 2498394.0, + "step": 470 + } + ], + "logging_steps": 10, + "max_steps": 2500, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.5798444746092544e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-475/training_args.bin b/checkpoint-475/training_args.bin new file mode 100644 index 0000000..f138a50 --- /dev/null +++ b/checkpoint-475/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7103b0ce3340ea41aca3b7441d01d7c38791e3d3ce8c9b28c906c9b1d49bbee9 +size 5713 diff --git a/checkpoint-567/chat_template.jinja b/checkpoint-567/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-567/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-567/config.json b/checkpoint-567/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-567/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-567/generation_config.json b/checkpoint-567/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-567/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-567/model.safetensors b/checkpoint-567/model.safetensors new file mode 100644 index 0000000..aadda1f --- /dev/null +++ b/checkpoint-567/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ebce5752220a4f4783428941084b299402df12b70d7b1ec028fd33522cef226 +size 3441185608 diff --git a/checkpoint-567/optimizer.pt b/checkpoint-567/optimizer.pt new file mode 100644 index 0000000..f1d5d36 --- /dev/null +++ b/checkpoint-567/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a4ba813dbf09fb5a0602bbdd0ba61d8c28898a85d484eaa742d6148b3191800 +size 6882572207 diff --git a/checkpoint-567/rng_state.pth b/checkpoint-567/rng_state.pth new file mode 100644 index 0000000..bd3c190 --- /dev/null +++ b/checkpoint-567/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934 +size 14645 diff --git a/checkpoint-567/scheduler.pt b/checkpoint-567/scheduler.pt new file mode 100644 index 0000000..fb0555f --- /dev/null +++ b/checkpoint-567/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e648dfc5857663e541012d5431aa66db99818c41c314bc081e9445f775c48dc +size 1465 diff --git a/checkpoint-567/tokenizer.json b/checkpoint-567/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-567/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-567/tokenizer_config.json b/checkpoint-567/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-567/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-567/trainer_state.json b/checkpoint-567/trainer_state.json new file mode 100644 index 0000000..496e1d6 --- /dev/null +++ b/checkpoint-567/trainer_state.json @@ -0,0 +1,594 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 567, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.9081946484744549, + "epoch": 0.053015241882041084, + "grad_norm": 51.5, + "learning_rate": 3e-06, + "loss": 2.320839500427246, + "mean_token_accuracy": 0.5708254482597113, + "num_tokens": 30761.0, + "step": 10 + }, + { + "entropy": 1.032745758444071, + "epoch": 0.10603048376408217, + "grad_norm": 21.125, + "learning_rate": 6.333333333333333e-06, + "loss": 1.9631355285644532, + "mean_token_accuracy": 0.5809337079524994, + "num_tokens": 61277.0, + "step": 20 + }, + { + "entropy": 1.2370933540165425, + "epoch": 0.15904572564612326, + "grad_norm": 11.0625, + "learning_rate": 9.666666666666667e-06, + "loss": 1.4653306961059571, + "mean_token_accuracy": 0.6311864696443081, + "num_tokens": 91730.0, + "step": 30 + }, + { + "entropy": 1.1646109029650689, + "epoch": 0.21206096752816433, + "grad_norm": 8.5625, + "learning_rate": 1.3000000000000001e-05, + "loss": 1.200753402709961, + "mean_token_accuracy": 0.6730783767998219, + "num_tokens": 122896.0, + "step": 40 + }, + { + "entropy": 1.0138436160981654, + "epoch": 0.26507620941020543, + "grad_norm": 7.4375, + "learning_rate": 1.6333333333333335e-05, + "loss": 1.0109354972839355, + "mean_token_accuracy": 0.7080431424081326, + "num_tokens": 153323.0, + "step": 50 + }, + { + "entropy": 0.8669569574296474, + "epoch": 0.31809145129224653, + "grad_norm": 6.75, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8859419822692871, + "mean_token_accuracy": 0.7356305047869682, + "num_tokens": 183928.0, + "step": 60 + }, + { + "entropy": 0.856675973534584, + "epoch": 0.3711066931742876, + "grad_norm": 7.6875, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8721119880676269, + "mean_token_accuracy": 0.7318881675601006, + "num_tokens": 215054.0, + "step": 70 + }, + { + "entropy": 0.7872001446783543, + "epoch": 0.42412193505632867, + "grad_norm": 7.46875, + "learning_rate": 1.92962962962963e-05, + "loss": 0.7920477390289307, + "mean_token_accuracy": 0.7553018122911453, + "num_tokens": 245614.0, + "step": 80 + }, + { + "entropy": 0.7739131577312947, + "epoch": 0.47713717693836977, + "grad_norm": 7.09375, + "learning_rate": 1.8925925925925928e-05, + "loss": 0.7949441432952881, + "mean_token_accuracy": 0.7514781631529331, + "num_tokens": 276651.0, + "step": 90 + }, + { + "entropy": 0.7763891488313674, + "epoch": 0.5301524188204109, + "grad_norm": 7.8125, + "learning_rate": 1.8555555555555557e-05, + "loss": 0.7867375373840332, + "mean_token_accuracy": 0.7523738898336887, + "num_tokens": 308561.0, + "step": 100 + }, + { + "entropy": 0.7757978036999702, + "epoch": 0.583167660702452, + "grad_norm": 7.21875, + "learning_rate": 1.8185185185185186e-05, + "loss": 0.7776416301727295, + "mean_token_accuracy": 0.7548527352511882, + "num_tokens": 339537.0, + "step": 110 + }, + { + "entropy": 0.7528716944158077, + "epoch": 0.6361829025844931, + "grad_norm": 6.28125, + "learning_rate": 1.7814814814814815e-05, + "loss": 0.7566296100616455, + "mean_token_accuracy": 0.7637521453201771, + "num_tokens": 370938.0, + "step": 120 + }, + { + "entropy": 0.7483146995306015, + "epoch": 0.6891981444665342, + "grad_norm": 7.0, + "learning_rate": 1.7444444444444448e-05, + "loss": 0.7508650302886963, + "mean_token_accuracy": 0.758705198019743, + "num_tokens": 402708.0, + "step": 130 + }, + { + "entropy": 0.7206243231892586, + "epoch": 0.7422133863485753, + "grad_norm": 6.65625, + "learning_rate": 1.7074074074074077e-05, + "loss": 0.7531259536743165, + "mean_token_accuracy": 0.7641572073101998, + "num_tokens": 433645.0, + "step": 140 + }, + { + "entropy": 0.7596748031675815, + "epoch": 0.7952286282306164, + "grad_norm": 6.03125, + "learning_rate": 1.6703703703703703e-05, + "loss": 0.7591583728790283, + "mean_token_accuracy": 0.7570627965033054, + "num_tokens": 465791.0, + "step": 150 + }, + { + "entropy": 0.7133973561227321, + "epoch": 0.8482438701126573, + "grad_norm": 7.125, + "learning_rate": 1.6333333333333335e-05, + "loss": 0.7458220005035401, + "mean_token_accuracy": 0.7626615799963474, + "num_tokens": 497296.0, + "step": 160 + }, + { + "entropy": 0.7134327009320259, + "epoch": 0.9012591119946984, + "grad_norm": 7.0, + "learning_rate": 1.5962962962962964e-05, + "loss": 0.7290266990661621, + "mean_token_accuracy": 0.7681502625346184, + "num_tokens": 528181.0, + "step": 170 + }, + { + "entropy": 0.7324420973658562, + "epoch": 0.9542743538767395, + "grad_norm": 7.5, + "learning_rate": 1.5592592592592593e-05, + "loss": 0.731742525100708, + "mean_token_accuracy": 0.7631207950413227, + "num_tokens": 558915.0, + "step": 180 + }, + { + "entropy": 0.7100333733992144, + "epoch": 1.0053015241882042, + "grad_norm": 8.3125, + "learning_rate": 1.5222222222222223e-05, + "loss": 0.7303330898284912, + "mean_token_accuracy": 0.7633489200047084, + "num_tokens": 588567.0, + "step": 190 + }, + { + "entropy": 0.6628884050995112, + "epoch": 1.0583167660702453, + "grad_norm": 7.0625, + "learning_rate": 1.4851851851851853e-05, + "loss": 0.6481011867523193, + "mean_token_accuracy": 0.7933171860873699, + "num_tokens": 619081.0, + "step": 200 + }, + { + "entropy": 0.6257969986647367, + "epoch": 1.1113320079522864, + "grad_norm": 7.25, + "learning_rate": 1.4481481481481483e-05, + "loss": 0.642666244506836, + "mean_token_accuracy": 0.7921255677938461, + "num_tokens": 650402.0, + "step": 210 + }, + { + "entropy": 0.6399037420749665, + "epoch": 1.1643472498343272, + "grad_norm": 6.03125, + "learning_rate": 1.4111111111111113e-05, + "loss": 0.6525338172912598, + "mean_token_accuracy": 0.7887163788080216, + "num_tokens": 681837.0, + "step": 220 + }, + { + "entropy": 0.6367121540009976, + "epoch": 1.2173624917163686, + "grad_norm": 7.59375, + "learning_rate": 1.3740740740740741e-05, + "loss": 0.6356833457946778, + "mean_token_accuracy": 0.7893597863614559, + "num_tokens": 712571.0, + "step": 230 + }, + { + "entropy": 0.6334792256355286, + "epoch": 1.2703777335984094, + "grad_norm": 6.6875, + "learning_rate": 1.3370370370370372e-05, + "loss": 0.6391499996185303, + "mean_token_accuracy": 0.7947010740637779, + "num_tokens": 743396.0, + "step": 240 + }, + { + "entropy": 0.6417076423764229, + "epoch": 1.3233929754804508, + "grad_norm": 7.4375, + "learning_rate": 1.3000000000000001e-05, + "loss": 0.6563210010528564, + "mean_token_accuracy": 0.787147381901741, + "num_tokens": 774883.0, + "step": 250 + }, + { + "entropy": 0.6213644035160542, + "epoch": 1.3764082173624916, + "grad_norm": 7.1875, + "learning_rate": 1.2629629629629632e-05, + "loss": 0.6295370578765869, + "mean_token_accuracy": 0.7955153584480286, + "num_tokens": 805520.0, + "step": 260 + }, + { + "entropy": 0.6552363462746144, + "epoch": 1.4294234592445327, + "grad_norm": 6.84375, + "learning_rate": 1.225925925925926e-05, + "loss": 0.6632986068725586, + "mean_token_accuracy": 0.7826757229864597, + "num_tokens": 836758.0, + "step": 270 + }, + { + "entropy": 0.6550090927630663, + "epoch": 1.4824387011265738, + "grad_norm": 7.25, + "learning_rate": 1.188888888888889e-05, + "loss": 0.6479949474334716, + "mean_token_accuracy": 0.78950335085392, + "num_tokens": 868056.0, + "step": 280 + }, + { + "entropy": 0.6275094959884882, + "epoch": 1.535453943008615, + "grad_norm": 6.4375, + "learning_rate": 1.151851851851852e-05, + "loss": 0.6384372234344482, + "mean_token_accuracy": 0.7930210866034031, + "num_tokens": 899420.0, + "step": 290 + }, + { + "entropy": 0.6052290312945843, + "epoch": 1.588469184890656, + "grad_norm": 6.875, + "learning_rate": 1.114814814814815e-05, + "loss": 0.609743309020996, + "mean_token_accuracy": 0.7990770071744919, + "num_tokens": 929571.0, + "step": 300 + }, + { + "entropy": 0.6174360021948815, + "epoch": 1.6414844267726971, + "grad_norm": 7.0625, + "learning_rate": 1.0777777777777778e-05, + "loss": 0.6313478946685791, + "mean_token_accuracy": 0.7964761383831501, + "num_tokens": 960928.0, + "step": 310 + }, + { + "entropy": 0.6127162992954254, + "epoch": 1.6944996686547382, + "grad_norm": 6.5625, + "learning_rate": 1.0407407407407408e-05, + "loss": 0.6150648593902588, + "mean_token_accuracy": 0.8008938111364842, + "num_tokens": 991961.0, + "step": 320 + }, + { + "entropy": 0.5947048265486956, + "epoch": 1.7475149105367793, + "grad_norm": 7.65625, + "learning_rate": 1.003703703703704e-05, + "loss": 0.6053301334381104, + "mean_token_accuracy": 0.801536674797535, + "num_tokens": 1022013.0, + "step": 330 + }, + { + "entropy": 0.6275908622890711, + "epoch": 1.8005301524188204, + "grad_norm": 6.9375, + "learning_rate": 9.666666666666667e-06, + "loss": 0.6307861804962158, + "mean_token_accuracy": 0.7918523952364922, + "num_tokens": 1053061.0, + "step": 340 + }, + { + "entropy": 0.6406017865985631, + "epoch": 1.8535453943008615, + "grad_norm": 7.0, + "learning_rate": 9.296296296296296e-06, + "loss": 0.6534273147583007, + "mean_token_accuracy": 0.789715650677681, + "num_tokens": 1085392.0, + "step": 350 + }, + { + "entropy": 0.6285702683031559, + "epoch": 1.9065606361829026, + "grad_norm": 7.8125, + "learning_rate": 8.925925925925927e-06, + "loss": 0.6331883907318115, + "mean_token_accuracy": 0.7924636773765087, + "num_tokens": 1117204.0, + "step": 360 + }, + { + "entropy": 0.615465309098363, + "epoch": 1.9595758780649437, + "grad_norm": 7.25, + "learning_rate": 8.555555555555556e-06, + "loss": 0.6278977870941163, + "mean_token_accuracy": 0.7965366452932358, + "num_tokens": 1147711.0, + "step": 370 + }, + { + "entropy": 0.6297557744886968, + "epoch": 2.0106030483764084, + "grad_norm": 6.46875, + "learning_rate": 8.185185185185187e-06, + "loss": 0.6424172878265381, + "mean_token_accuracy": 0.7955868112576472, + "num_tokens": 1177908.0, + "step": 380 + }, + { + "entropy": 0.599795724824071, + "epoch": 2.0636182902584492, + "grad_norm": 6.4375, + "learning_rate": 7.814814814814816e-06, + "loss": 0.5880236625671387, + "mean_token_accuracy": 0.805521210283041, + "num_tokens": 1209330.0, + "step": 390 + }, + { + "entropy": 0.5943233758211136, + "epoch": 2.1166335321404905, + "grad_norm": 6.9375, + "learning_rate": 7.444444444444445e-06, + "loss": 0.5809715270996094, + "mean_token_accuracy": 0.8090382158756256, + "num_tokens": 1241043.0, + "step": 400 + }, + { + "entropy": 0.583736015111208, + "epoch": 2.1696487740225314, + "grad_norm": 6.875, + "learning_rate": 7.074074074074074e-06, + "loss": 0.595209789276123, + "mean_token_accuracy": 0.8054515205323696, + "num_tokens": 1272875.0, + "step": 410 + }, + { + "entropy": 0.5891257628798485, + "epoch": 2.2226640159045727, + "grad_norm": 7.875, + "learning_rate": 6.703703703703704e-06, + "loss": 0.6067119598388672, + "mean_token_accuracy": 0.8018631525337696, + "num_tokens": 1303473.0, + "step": 420 + }, + { + "entropy": 0.5886123344302178, + "epoch": 2.2756792577866136, + "grad_norm": 7.65625, + "learning_rate": 6.333333333333333e-06, + "loss": 0.577965784072876, + "mean_token_accuracy": 0.8081556148827076, + "num_tokens": 1335327.0, + "step": 430 + }, + { + "entropy": 0.5796562805771828, + "epoch": 2.3286944996686545, + "grad_norm": 8.125, + "learning_rate": 5.962962962962963e-06, + "loss": 0.5881845951080322, + "mean_token_accuracy": 0.8047773309051991, + "num_tokens": 1366019.0, + "step": 440 + }, + { + "entropy": 0.590113490819931, + "epoch": 2.381709741550696, + "grad_norm": 6.96875, + "learning_rate": 5.5925925925925926e-06, + "loss": 0.593454647064209, + "mean_token_accuracy": 0.809637114405632, + "num_tokens": 1397101.0, + "step": 450 + }, + { + "entropy": 0.570321362093091, + "epoch": 2.434724983432737, + "grad_norm": 7.34375, + "learning_rate": 5.2222222222222226e-06, + "loss": 0.5790628910064697, + "mean_token_accuracy": 0.809483564645052, + "num_tokens": 1428115.0, + "step": 460 + }, + { + "entropy": 0.6001236628741026, + "epoch": 2.487740225314778, + "grad_norm": 7.21875, + "learning_rate": 4.851851851851852e-06, + "loss": 0.6089425086975098, + "mean_token_accuracy": 0.8031827084720135, + "num_tokens": 1460337.0, + "step": 470 + }, + { + "entropy": 0.5818770505487919, + "epoch": 2.540755467196819, + "grad_norm": 6.5625, + "learning_rate": 4.481481481481482e-06, + "loss": 0.5913994789123536, + "mean_token_accuracy": 0.8058472029864788, + "num_tokens": 1491465.0, + "step": 480 + }, + { + "entropy": 0.5761014901101589, + "epoch": 2.59377070907886, + "grad_norm": 7.25, + "learning_rate": 4.111111111111111e-06, + "loss": 0.573754072189331, + "mean_token_accuracy": 0.8119833193719387, + "num_tokens": 1522283.0, + "step": 490 + }, + { + "entropy": 0.5881835252046586, + "epoch": 2.6467859509609015, + "grad_norm": 7.8125, + "learning_rate": 3.740740740740741e-06, + "loss": 0.5855993270874024, + "mean_token_accuracy": 0.8061533674597741, + "num_tokens": 1553066.0, + "step": 500 + }, + { + "entropy": 0.5923898722976446, + "epoch": 2.6998011928429424, + "grad_norm": 7.25, + "learning_rate": 3.3703703703703705e-06, + "loss": 0.5783270835876465, + "mean_token_accuracy": 0.8092506147921086, + "num_tokens": 1583785.0, + "step": 510 + }, + { + "entropy": 0.5890947930514813, + "epoch": 2.7528164347249833, + "grad_norm": 7.09375, + "learning_rate": 3e-06, + "loss": 0.5847487449645996, + "mean_token_accuracy": 0.8107674546539784, + "num_tokens": 1615166.0, + "step": 520 + }, + { + "entropy": 0.5714535418897867, + "epoch": 2.8058316766070246, + "grad_norm": 7.0, + "learning_rate": 2.6296296296296297e-06, + "loss": 0.5737119197845459, + "mean_token_accuracy": 0.8139248043298721, + "num_tokens": 1646166.0, + "step": 530 + }, + { + "entropy": 0.5585082449018955, + "epoch": 2.8588469184890655, + "grad_norm": 8.75, + "learning_rate": 2.2592592592592592e-06, + "loss": 0.5831988334655762, + "mean_token_accuracy": 0.8078319534659386, + "num_tokens": 1675970.0, + "step": 540 + }, + { + "entropy": 0.5816058319061994, + "epoch": 2.911862160371107, + "grad_norm": 6.6875, + "learning_rate": 1.888888888888889e-06, + "loss": 0.585014820098877, + "mean_token_accuracy": 0.8070621207356453, + "num_tokens": 1707365.0, + "step": 550 + }, + { + "entropy": 0.5718538902699948, + "epoch": 2.9648774022531477, + "grad_norm": 7.4375, + "learning_rate": 1.5185185185185186e-06, + "loss": 0.5958520889282226, + "mean_token_accuracy": 0.8062198340892792, + "num_tokens": 1737306.0, + "step": 560 + } + ], + "logging_steps": 10, + "max_steps": 600, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.696568888819712e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-567/training_args.bin b/checkpoint-567/training_args.bin new file mode 100644 index 0000000..6618e60 --- /dev/null +++ b/checkpoint-567/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c0557c0612cd65ef8abf2a6c3d767d5aa1d732f9c315d62b560fc3d0f5beafe +size 5713 diff --git a/checkpoint-600/chat_template.jinja b/checkpoint-600/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-600/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-600/config.json b/checkpoint-600/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-600/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-600/generation_config.json b/checkpoint-600/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-600/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-600/model.safetensors b/checkpoint-600/model.safetensors new file mode 100644 index 0000000..28aa3ca --- /dev/null +++ b/checkpoint-600/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d570423c6df660c3be3ebf594f39525fef6823650a3eae38f742af2facc38dae +size 3441185608 diff --git a/checkpoint-600/optimizer.pt b/checkpoint-600/optimizer.pt new file mode 100644 index 0000000..8d06d02 --- /dev/null +++ b/checkpoint-600/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7642ab0f044ffbc43b564aa5d895a996678d5ac57f1f4d612efeccc8c8ca9d2a +size 6882572207 diff --git a/checkpoint-600/rng_state.pth b/checkpoint-600/rng_state.pth new file mode 100644 index 0000000..435e005 --- /dev/null +++ b/checkpoint-600/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e2011629d8bed3ef560fa11175cac55684c4e12a72634bb24abf767b6c7399 +size 14645 diff --git a/checkpoint-600/scheduler.pt b/checkpoint-600/scheduler.pt new file mode 100644 index 0000000..02f4078 --- /dev/null +++ b/checkpoint-600/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b91574c22d1224d524777b6e7aad61eaf8ad00b25f7f1baa8b9fd77fb34462 +size 1465 diff --git a/checkpoint-600/tokenizer.json b/checkpoint-600/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-600/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-600/tokenizer_config.json b/checkpoint-600/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-600/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-600/trainer_state.json b/checkpoint-600/trainer_state.json new file mode 100644 index 0000000..ae98e7f --- /dev/null +++ b/checkpoint-600/trainer_state.json @@ -0,0 +1,634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.1749502982107356, + "eval_steps": 500, + "global_step": 600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.9081946484744549, + "epoch": 0.053015241882041084, + "grad_norm": 51.5, + "learning_rate": 3e-06, + "loss": 2.320839500427246, + "mean_token_accuracy": 0.5708254482597113, + "num_tokens": 30761.0, + "step": 10 + }, + { + "entropy": 1.032745758444071, + "epoch": 0.10603048376408217, + "grad_norm": 21.125, + "learning_rate": 6.333333333333333e-06, + "loss": 1.9631355285644532, + "mean_token_accuracy": 0.5809337079524994, + "num_tokens": 61277.0, + "step": 20 + }, + { + "entropy": 1.2370933540165425, + "epoch": 0.15904572564612326, + "grad_norm": 11.0625, + "learning_rate": 9.666666666666667e-06, + "loss": 1.4653306961059571, + "mean_token_accuracy": 0.6311864696443081, + "num_tokens": 91730.0, + "step": 30 + }, + { + "entropy": 1.1646109029650689, + "epoch": 0.21206096752816433, + "grad_norm": 8.5625, + "learning_rate": 1.3000000000000001e-05, + "loss": 1.200753402709961, + "mean_token_accuracy": 0.6730783767998219, + "num_tokens": 122896.0, + "step": 40 + }, + { + "entropy": 1.0138436160981654, + "epoch": 0.26507620941020543, + "grad_norm": 7.4375, + "learning_rate": 1.6333333333333335e-05, + "loss": 1.0109354972839355, + "mean_token_accuracy": 0.7080431424081326, + "num_tokens": 153323.0, + "step": 50 + }, + { + "entropy": 0.8669569574296474, + "epoch": 0.31809145129224653, + "grad_norm": 6.75, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8859419822692871, + "mean_token_accuracy": 0.7356305047869682, + "num_tokens": 183928.0, + "step": 60 + }, + { + "entropy": 0.856675973534584, + "epoch": 0.3711066931742876, + "grad_norm": 7.6875, + "learning_rate": 1.9666666666666666e-05, + "loss": 0.8721119880676269, + "mean_token_accuracy": 0.7318881675601006, + "num_tokens": 215054.0, + "step": 70 + }, + { + "entropy": 0.7872001446783543, + "epoch": 0.42412193505632867, + "grad_norm": 7.46875, + "learning_rate": 1.92962962962963e-05, + "loss": 0.7920477390289307, + "mean_token_accuracy": 0.7553018122911453, + "num_tokens": 245614.0, + "step": 80 + }, + { + "entropy": 0.7739131577312947, + "epoch": 0.47713717693836977, + "grad_norm": 7.09375, + "learning_rate": 1.8925925925925928e-05, + "loss": 0.7949441432952881, + "mean_token_accuracy": 0.7514781631529331, + "num_tokens": 276651.0, + "step": 90 + }, + { + "entropy": 0.7763891488313674, + "epoch": 0.5301524188204109, + "grad_norm": 7.8125, + "learning_rate": 1.8555555555555557e-05, + "loss": 0.7867375373840332, + "mean_token_accuracy": 0.7523738898336887, + "num_tokens": 308561.0, + "step": 100 + }, + { + "entropy": 0.7757978036999702, + "epoch": 0.583167660702452, + "grad_norm": 7.21875, + "learning_rate": 1.8185185185185186e-05, + "loss": 0.7776416301727295, + "mean_token_accuracy": 0.7548527352511882, + "num_tokens": 339537.0, + "step": 110 + }, + { + "entropy": 0.7528716944158077, + "epoch": 0.6361829025844931, + "grad_norm": 6.28125, + "learning_rate": 1.7814814814814815e-05, + "loss": 0.7566296100616455, + "mean_token_accuracy": 0.7637521453201771, + "num_tokens": 370938.0, + "step": 120 + }, + { + "entropy": 0.7483146995306015, + "epoch": 0.6891981444665342, + "grad_norm": 7.0, + "learning_rate": 1.7444444444444448e-05, + "loss": 0.7508650302886963, + "mean_token_accuracy": 0.758705198019743, + "num_tokens": 402708.0, + "step": 130 + }, + { + "entropy": 0.7206243231892586, + "epoch": 0.7422133863485753, + "grad_norm": 6.65625, + "learning_rate": 1.7074074074074077e-05, + "loss": 0.7531259536743165, + "mean_token_accuracy": 0.7641572073101998, + "num_tokens": 433645.0, + "step": 140 + }, + { + "entropy": 0.7596748031675815, + "epoch": 0.7952286282306164, + "grad_norm": 6.03125, + "learning_rate": 1.6703703703703703e-05, + "loss": 0.7591583728790283, + "mean_token_accuracy": 0.7570627965033054, + "num_tokens": 465791.0, + "step": 150 + }, + { + "entropy": 0.7133973561227321, + "epoch": 0.8482438701126573, + "grad_norm": 7.125, + "learning_rate": 1.6333333333333335e-05, + "loss": 0.7458220005035401, + "mean_token_accuracy": 0.7626615799963474, + "num_tokens": 497296.0, + "step": 160 + }, + { + "entropy": 0.7134327009320259, + "epoch": 0.9012591119946984, + "grad_norm": 7.0, + "learning_rate": 1.5962962962962964e-05, + "loss": 0.7290266990661621, + "mean_token_accuracy": 0.7681502625346184, + "num_tokens": 528181.0, + "step": 170 + }, + { + "entropy": 0.7324420973658562, + "epoch": 0.9542743538767395, + "grad_norm": 7.5, + "learning_rate": 1.5592592592592593e-05, + "loss": 0.731742525100708, + "mean_token_accuracy": 0.7631207950413227, + "num_tokens": 558915.0, + "step": 180 + }, + { + "entropy": 0.7100333733992144, + "epoch": 1.0053015241882042, + "grad_norm": 8.3125, + "learning_rate": 1.5222222222222223e-05, + "loss": 0.7303330898284912, + "mean_token_accuracy": 0.7633489200047084, + "num_tokens": 588567.0, + "step": 190 + }, + { + "entropy": 0.6628884050995112, + "epoch": 1.0583167660702453, + "grad_norm": 7.0625, + "learning_rate": 1.4851851851851853e-05, + "loss": 0.6481011867523193, + "mean_token_accuracy": 0.7933171860873699, + "num_tokens": 619081.0, + "step": 200 + }, + { + "entropy": 0.6257969986647367, + "epoch": 1.1113320079522864, + "grad_norm": 7.25, + "learning_rate": 1.4481481481481483e-05, + "loss": 0.642666244506836, + "mean_token_accuracy": 0.7921255677938461, + "num_tokens": 650402.0, + "step": 210 + }, + { + "entropy": 0.6399037420749665, + "epoch": 1.1643472498343272, + "grad_norm": 6.03125, + "learning_rate": 1.4111111111111113e-05, + "loss": 0.6525338172912598, + "mean_token_accuracy": 0.7887163788080216, + "num_tokens": 681837.0, + "step": 220 + }, + { + "entropy": 0.6367121540009976, + "epoch": 1.2173624917163686, + "grad_norm": 7.59375, + "learning_rate": 1.3740740740740741e-05, + "loss": 0.6356833457946778, + "mean_token_accuracy": 0.7893597863614559, + "num_tokens": 712571.0, + "step": 230 + }, + { + "entropy": 0.6334792256355286, + "epoch": 1.2703777335984094, + "grad_norm": 6.6875, + "learning_rate": 1.3370370370370372e-05, + "loss": 0.6391499996185303, + "mean_token_accuracy": 0.7947010740637779, + "num_tokens": 743396.0, + "step": 240 + }, + { + "entropy": 0.6417076423764229, + "epoch": 1.3233929754804508, + "grad_norm": 7.4375, + "learning_rate": 1.3000000000000001e-05, + "loss": 0.6563210010528564, + "mean_token_accuracy": 0.787147381901741, + "num_tokens": 774883.0, + "step": 250 + }, + { + "entropy": 0.6213644035160542, + "epoch": 1.3764082173624916, + "grad_norm": 7.1875, + "learning_rate": 1.2629629629629632e-05, + "loss": 0.6295370578765869, + "mean_token_accuracy": 0.7955153584480286, + "num_tokens": 805520.0, + "step": 260 + }, + { + "entropy": 0.6552363462746144, + "epoch": 1.4294234592445327, + "grad_norm": 6.84375, + "learning_rate": 1.225925925925926e-05, + "loss": 0.6632986068725586, + "mean_token_accuracy": 0.7826757229864597, + "num_tokens": 836758.0, + "step": 270 + }, + { + "entropy": 0.6550090927630663, + "epoch": 1.4824387011265738, + "grad_norm": 7.25, + "learning_rate": 1.188888888888889e-05, + "loss": 0.6479949474334716, + "mean_token_accuracy": 0.78950335085392, + "num_tokens": 868056.0, + "step": 280 + }, + { + "entropy": 0.6275094959884882, + "epoch": 1.535453943008615, + "grad_norm": 6.4375, + "learning_rate": 1.151851851851852e-05, + "loss": 0.6384372234344482, + "mean_token_accuracy": 0.7930210866034031, + "num_tokens": 899420.0, + "step": 290 + }, + { + "entropy": 0.6052290312945843, + "epoch": 1.588469184890656, + "grad_norm": 6.875, + "learning_rate": 1.114814814814815e-05, + "loss": 0.609743309020996, + "mean_token_accuracy": 0.7990770071744919, + "num_tokens": 929571.0, + "step": 300 + }, + { + "entropy": 0.6174360021948815, + "epoch": 1.6414844267726971, + "grad_norm": 7.0625, + "learning_rate": 1.0777777777777778e-05, + "loss": 0.6313478946685791, + "mean_token_accuracy": 0.7964761383831501, + "num_tokens": 960928.0, + "step": 310 + }, + { + "entropy": 0.6127162992954254, + "epoch": 1.6944996686547382, + "grad_norm": 6.5625, + "learning_rate": 1.0407407407407408e-05, + "loss": 0.6150648593902588, + "mean_token_accuracy": 0.8008938111364842, + "num_tokens": 991961.0, + "step": 320 + }, + { + "entropy": 0.5947048265486956, + "epoch": 1.7475149105367793, + "grad_norm": 7.65625, + "learning_rate": 1.003703703703704e-05, + "loss": 0.6053301334381104, + "mean_token_accuracy": 0.801536674797535, + "num_tokens": 1022013.0, + "step": 330 + }, + { + "entropy": 0.6275908622890711, + "epoch": 1.8005301524188204, + "grad_norm": 6.9375, + "learning_rate": 9.666666666666667e-06, + "loss": 0.6307861804962158, + "mean_token_accuracy": 0.7918523952364922, + "num_tokens": 1053061.0, + "step": 340 + }, + { + "entropy": 0.6406017865985631, + "epoch": 1.8535453943008615, + "grad_norm": 7.0, + "learning_rate": 9.296296296296296e-06, + "loss": 0.6534273147583007, + "mean_token_accuracy": 0.789715650677681, + "num_tokens": 1085392.0, + "step": 350 + }, + { + "entropy": 0.6285702683031559, + "epoch": 1.9065606361829026, + "grad_norm": 7.8125, + "learning_rate": 8.925925925925927e-06, + "loss": 0.6331883907318115, + "mean_token_accuracy": 0.7924636773765087, + "num_tokens": 1117204.0, + "step": 360 + }, + { + "entropy": 0.615465309098363, + "epoch": 1.9595758780649437, + "grad_norm": 7.25, + "learning_rate": 8.555555555555556e-06, + "loss": 0.6278977870941163, + "mean_token_accuracy": 0.7965366452932358, + "num_tokens": 1147711.0, + "step": 370 + }, + { + "entropy": 0.6297557744886968, + "epoch": 2.0106030483764084, + "grad_norm": 6.46875, + "learning_rate": 8.185185185185187e-06, + "loss": 0.6424172878265381, + "mean_token_accuracy": 0.7955868112576472, + "num_tokens": 1177908.0, + "step": 380 + }, + { + "entropy": 0.599795724824071, + "epoch": 2.0636182902584492, + "grad_norm": 6.4375, + "learning_rate": 7.814814814814816e-06, + "loss": 0.5880236625671387, + "mean_token_accuracy": 0.805521210283041, + "num_tokens": 1209330.0, + "step": 390 + }, + { + "entropy": 0.5943233758211136, + "epoch": 2.1166335321404905, + "grad_norm": 6.9375, + "learning_rate": 7.444444444444445e-06, + "loss": 0.5809715270996094, + "mean_token_accuracy": 0.8090382158756256, + "num_tokens": 1241043.0, + "step": 400 + }, + { + "entropy": 0.583736015111208, + "epoch": 2.1696487740225314, + "grad_norm": 6.875, + "learning_rate": 7.074074074074074e-06, + "loss": 0.595209789276123, + "mean_token_accuracy": 0.8054515205323696, + "num_tokens": 1272875.0, + "step": 410 + }, + { + "entropy": 0.5891257628798485, + "epoch": 2.2226640159045727, + "grad_norm": 7.875, + "learning_rate": 6.703703703703704e-06, + "loss": 0.6067119598388672, + "mean_token_accuracy": 0.8018631525337696, + "num_tokens": 1303473.0, + "step": 420 + }, + { + "entropy": 0.5886123344302178, + "epoch": 2.2756792577866136, + "grad_norm": 7.65625, + "learning_rate": 6.333333333333333e-06, + "loss": 0.577965784072876, + "mean_token_accuracy": 0.8081556148827076, + "num_tokens": 1335327.0, + "step": 430 + }, + { + "entropy": 0.5796562805771828, + "epoch": 2.3286944996686545, + "grad_norm": 8.125, + "learning_rate": 5.962962962962963e-06, + "loss": 0.5881845951080322, + "mean_token_accuracy": 0.8047773309051991, + "num_tokens": 1366019.0, + "step": 440 + }, + { + "entropy": 0.590113490819931, + "epoch": 2.381709741550696, + "grad_norm": 6.96875, + "learning_rate": 5.5925925925925926e-06, + "loss": 0.593454647064209, + "mean_token_accuracy": 0.809637114405632, + "num_tokens": 1397101.0, + "step": 450 + }, + { + "entropy": 0.570321362093091, + "epoch": 2.434724983432737, + "grad_norm": 7.34375, + "learning_rate": 5.2222222222222226e-06, + "loss": 0.5790628910064697, + "mean_token_accuracy": 0.809483564645052, + "num_tokens": 1428115.0, + "step": 460 + }, + { + "entropy": 0.6001236628741026, + "epoch": 2.487740225314778, + "grad_norm": 7.21875, + "learning_rate": 4.851851851851852e-06, + "loss": 0.6089425086975098, + "mean_token_accuracy": 0.8031827084720135, + "num_tokens": 1460337.0, + "step": 470 + }, + { + "entropy": 0.5818770505487919, + "epoch": 2.540755467196819, + "grad_norm": 6.5625, + "learning_rate": 4.481481481481482e-06, + "loss": 0.5913994789123536, + "mean_token_accuracy": 0.8058472029864788, + "num_tokens": 1491465.0, + "step": 480 + }, + { + "entropy": 0.5761014901101589, + "epoch": 2.59377070907886, + "grad_norm": 7.25, + "learning_rate": 4.111111111111111e-06, + "loss": 0.573754072189331, + "mean_token_accuracy": 0.8119833193719387, + "num_tokens": 1522283.0, + "step": 490 + }, + { + "entropy": 0.5881835252046586, + "epoch": 2.6467859509609015, + "grad_norm": 7.8125, + "learning_rate": 3.740740740740741e-06, + "loss": 0.5855993270874024, + "mean_token_accuracy": 0.8061533674597741, + "num_tokens": 1553066.0, + "step": 500 + }, + { + "entropy": 0.5923898722976446, + "epoch": 2.6998011928429424, + "grad_norm": 7.25, + "learning_rate": 3.3703703703703705e-06, + "loss": 0.5783270835876465, + "mean_token_accuracy": 0.8092506147921086, + "num_tokens": 1583785.0, + "step": 510 + }, + { + "entropy": 0.5890947930514813, + "epoch": 2.7528164347249833, + "grad_norm": 7.09375, + "learning_rate": 3e-06, + "loss": 0.5847487449645996, + "mean_token_accuracy": 0.8107674546539784, + "num_tokens": 1615166.0, + "step": 520 + }, + { + "entropy": 0.5714535418897867, + "epoch": 2.8058316766070246, + "grad_norm": 7.0, + "learning_rate": 2.6296296296296297e-06, + "loss": 0.5737119197845459, + "mean_token_accuracy": 0.8139248043298721, + "num_tokens": 1646166.0, + "step": 530 + }, + { + "entropy": 0.5585082449018955, + "epoch": 2.8588469184890655, + "grad_norm": 8.75, + "learning_rate": 2.2592592592592592e-06, + "loss": 0.5831988334655762, + "mean_token_accuracy": 0.8078319534659386, + "num_tokens": 1675970.0, + "step": 540 + }, + { + "entropy": 0.5816058319061994, + "epoch": 2.911862160371107, + "grad_norm": 6.6875, + "learning_rate": 1.888888888888889e-06, + "loss": 0.585014820098877, + "mean_token_accuracy": 0.8070621207356453, + "num_tokens": 1707365.0, + "step": 550 + }, + { + "entropy": 0.5718538902699948, + "epoch": 2.9648774022531477, + "grad_norm": 7.4375, + "learning_rate": 1.5185185185185186e-06, + "loss": 0.5958520889282226, + "mean_token_accuracy": 0.8062198340892792, + "num_tokens": 1737306.0, + "step": 560 + }, + { + "entropy": 0.5680852636888429, + "epoch": 3.0159045725646125, + "grad_norm": 7.28125, + "learning_rate": 1.1481481481481482e-06, + "loss": 0.5883944511413575, + "mean_token_accuracy": 0.8095385347093854, + "num_tokens": 1766093.0, + "step": 570 + }, + { + "entropy": 0.5735234707593918, + "epoch": 3.0689198144466534, + "grad_norm": 6.1875, + "learning_rate": 7.777777777777779e-07, + "loss": 0.5706701755523682, + "mean_token_accuracy": 0.8123805336654186, + "num_tokens": 1796844.0, + "step": 580 + }, + { + "entropy": 0.575826159119606, + "epoch": 3.1219350563286943, + "grad_norm": 6.9375, + "learning_rate": 4.074074074074075e-07, + "loss": 0.5520034790039062, + "mean_token_accuracy": 0.8180661156773568, + "num_tokens": 1828137.0, + "step": 590 + }, + { + "entropy": 0.5985390111804009, + "epoch": 3.1749502982107356, + "grad_norm": 7.34375, + "learning_rate": 3.703703703703704e-08, + "loss": 0.5971494674682617, + "mean_token_accuracy": 0.8037057816982269, + "num_tokens": 1859725.0, + "step": 600 + } + ], + "logging_steps": 10, + "max_steps": 600, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.7956887511474176e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-600/training_args.bin b/checkpoint-600/training_args.bin new file mode 100644 index 0000000..6618e60 --- /dev/null +++ b/checkpoint-600/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c0557c0612cd65ef8abf2a6c3d767d5aa1d732f9c315d62b560fc3d0f5beafe +size 5713 diff --git a/checkpoint-63/chat_template.jinja b/checkpoint-63/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-63/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-63/config.json b/checkpoint-63/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-63/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-63/generation_config.json b/checkpoint-63/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-63/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-63/model.safetensors b/checkpoint-63/model.safetensors new file mode 100644 index 0000000..681b214 --- /dev/null +++ b/checkpoint-63/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b861c0346017062303d33517df827a8b2719243b9b4ab74727ea3b4a999289bc +size 3441185608 diff --git a/checkpoint-63/optimizer.pt b/checkpoint-63/optimizer.pt new file mode 100644 index 0000000..d320cf9 --- /dev/null +++ b/checkpoint-63/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1c505c1716a82ca3d2e1cd8cd1877b1f717f4453be73242b57922e6a4942493 +size 6882572207 diff --git a/checkpoint-63/rng_state.pth b/checkpoint-63/rng_state.pth new file mode 100644 index 0000000..5a8f17a --- /dev/null +++ b/checkpoint-63/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4 +size 14645 diff --git a/checkpoint-63/scheduler.pt b/checkpoint-63/scheduler.pt new file mode 100644 index 0000000..c3e6563 --- /dev/null +++ b/checkpoint-63/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8baf2b5a46fac85f95c0a15caf2414a92c036d41b6ccf016dc0542ed26e499c2 +size 1465 diff --git a/checkpoint-63/tokenizer.json b/checkpoint-63/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-63/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-63/tokenizer_config.json b/checkpoint-63/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-63/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-63/trainer_state.json b/checkpoint-63/trainer_state.json new file mode 100644 index 0000000..5cd7f04 --- /dev/null +++ b/checkpoint-63/trainer_state.json @@ -0,0 +1,94 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 63, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.1056655953405425, + "epoch": 0.16, + "grad_norm": 12.0625, + "learning_rate": 1.894736842105263e-05, + "loss": 3.4882164001464844, + "mean_token_accuracy": 0.7255681823939085, + "num_tokens": 40047.0, + "step": 10 + }, + { + "entropy": 0.06432699970901012, + "epoch": 0.32, + "grad_norm": 2.453125, + "learning_rate": 1.543859649122807e-05, + "loss": 0.06028715968132019, + "mean_token_accuracy": 0.9670454517006875, + "num_tokens": 79520.0, + "step": 20 + }, + { + "entropy": 0.0531476724660024, + "epoch": 0.48, + "grad_norm": 3.5, + "learning_rate": 1.192982456140351e-05, + "loss": 0.05689932703971863, + "mean_token_accuracy": 0.9693181797862053, + "num_tokens": 120623.0, + "step": 30 + }, + { + "entropy": 0.054595551500096914, + "epoch": 0.64, + "grad_norm": 4.0625, + "learning_rate": 8.421052631578948e-06, + "loss": 0.06269298195838928, + "mean_token_accuracy": 0.961363635212183, + "num_tokens": 162158.0, + "step": 40 + }, + { + "entropy": 0.056087028328329326, + "epoch": 0.8, + "grad_norm": 2.953125, + "learning_rate": 4.912280701754386e-06, + "loss": 0.05635017156600952, + "mean_token_accuracy": 0.9698863588273525, + "num_tokens": 202787.0, + "step": 50 + }, + { + "entropy": 0.05579588690306991, + "epoch": 0.96, + "grad_norm": 3.125, + "learning_rate": 1.4035087719298246e-06, + "loss": 0.05594708323478699, + "mean_token_accuracy": 0.965340905636549, + "num_tokens": 243390.0, + "step": 60 + } + ], + "logging_steps": 10, + "max_steps": 63, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2576514320547840.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-63/training_args.bin b/checkpoint-63/training_args.bin new file mode 100644 index 0000000..16afdf1 --- /dev/null +++ b/checkpoint-63/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0c6b5ffa9e62ba1a8e5094c99c37fad8b460397179909c485bbecc3b7bcf0dd +size 5713 diff --git a/checkpoint-950/chat_template.jinja b/checkpoint-950/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/checkpoint-950/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-950/config.json b/checkpoint-950/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/checkpoint-950/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-950/generation_config.json b/checkpoint-950/generation_config.json new file mode 100644 index 0000000..5ec133d --- /dev/null +++ b/checkpoint-950/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.7.0" +} diff --git a/checkpoint-950/model.safetensors b/checkpoint-950/model.safetensors new file mode 100644 index 0000000..febc610 --- /dev/null +++ b/checkpoint-950/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07855dca909d308e2784d7676d0d2657e5764305c4cf1ccf175a187195feee05 +size 3441185608 diff --git a/checkpoint-950/optimizer.pt b/checkpoint-950/optimizer.pt new file mode 100644 index 0000000..c3e6e62 --- /dev/null +++ b/checkpoint-950/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaa00141bbb9106c994bfd47575986a0125627a6bb25c191ab66699fd85a356c +size 6882572207 diff --git a/checkpoint-950/rng_state.pth b/checkpoint-950/rng_state.pth new file mode 100644 index 0000000..1feba1a --- /dev/null +++ b/checkpoint-950/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878 +size 14645 diff --git a/checkpoint-950/scheduler.pt b/checkpoint-950/scheduler.pt new file mode 100644 index 0000000..bfb8f8b --- /dev/null +++ b/checkpoint-950/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4adf707525d5cbe09ee83650f76c0ed10dc8b570230201ecf46eef05b98e71ca +size 1465 diff --git a/checkpoint-950/tokenizer.json b/checkpoint-950/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-950/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-950/tokenizer_config.json b/checkpoint-950/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/checkpoint-950/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-950/trainer_state.json b/checkpoint-950/trainer_state.json new file mode 100644 index 0000000..f0f5be9 --- /dev/null +++ b/checkpoint-950/trainer_state.json @@ -0,0 +1,984 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 950, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.8691443920135498, + "epoch": 0.021085925144965736, + "grad_norm": 47.5, + "learning_rate": 7.2e-07, + "loss": 2.1268569946289064, + "mean_token_accuracy": 0.5856592752039432, + "num_tokens": 53385.0, + "step": 10 + }, + { + "entropy": 0.9018128983676433, + "epoch": 0.04217185028993147, + "grad_norm": 49.25, + "learning_rate": 1.52e-06, + "loss": 2.1235822677612304, + "mean_token_accuracy": 0.5826416034251451, + "num_tokens": 108949.0, + "step": 20 + }, + { + "entropy": 0.9313402399420738, + "epoch": 0.0632577754348972, + "grad_norm": 38.25, + "learning_rate": 2.3200000000000002e-06, + "loss": 1.9997358322143555, + "mean_token_accuracy": 0.5817345686256885, + "num_tokens": 162034.0, + "step": 30 + }, + { + "entropy": 0.9819381944835186, + "epoch": 0.08434370057986294, + "grad_norm": 28.25, + "learning_rate": 3.12e-06, + "loss": 1.8760208129882812, + "mean_token_accuracy": 0.5874959267675877, + "num_tokens": 214165.0, + "step": 40 + }, + { + "entropy": 1.0624087303876877, + "epoch": 0.10542962572482868, + "grad_norm": 14.1875, + "learning_rate": 3.920000000000001e-06, + "loss": 1.6169570922851562, + "mean_token_accuracy": 0.601640059798956, + "num_tokens": 265527.0, + "step": 50 + }, + { + "entropy": 1.1721510097384453, + "epoch": 0.1265155508697944, + "grad_norm": 11.3125, + "learning_rate": 4.7200000000000005e-06, + "loss": 1.4622878074645995, + "mean_token_accuracy": 0.6209174185991287, + "num_tokens": 319341.0, + "step": 60 + }, + { + "entropy": 1.2009219758212566, + "epoch": 0.14760147601476015, + "grad_norm": 9.0625, + "learning_rate": 5.5200000000000005e-06, + "loss": 1.2933337211608886, + "mean_token_accuracy": 0.6467074818909169, + "num_tokens": 373709.0, + "step": 70 + }, + { + "entropy": 1.1439887225627898, + "epoch": 0.16868740115972589, + "grad_norm": 9.3125, + "learning_rate": 6.3200000000000005e-06, + "loss": 1.2084498405456543, + "mean_token_accuracy": 0.6660686552524566, + "num_tokens": 425923.0, + "step": 80 + }, + { + "entropy": 1.0713621236383915, + "epoch": 0.18977332630469163, + "grad_norm": 7.84375, + "learning_rate": 7.1200000000000004e-06, + "loss": 1.1175435066223145, + "mean_token_accuracy": 0.6812180563807487, + "num_tokens": 480012.0, + "step": 90 + }, + { + "entropy": 1.0369222231209279, + "epoch": 0.21085925144965736, + "grad_norm": 7.09375, + "learning_rate": 7.92e-06, + "loss": 1.062850570678711, + "mean_token_accuracy": 0.6899570092558861, + "num_tokens": 533890.0, + "step": 100 + }, + { + "entropy": 0.9894454136490822, + "epoch": 0.23194517659462308, + "grad_norm": 7.75, + "learning_rate": 8.720000000000001e-06, + "loss": 1.0150799751281738, + "mean_token_accuracy": 0.7009239844977856, + "num_tokens": 590654.0, + "step": 110 + }, + { + "entropy": 0.9410129584372043, + "epoch": 0.2530311017395888, + "grad_norm": 7.28125, + "learning_rate": 9.52e-06, + "loss": 0.9810515403747558, + "mean_token_accuracy": 0.709618617594242, + "num_tokens": 643080.0, + "step": 120 + }, + { + "entropy": 0.9265115469694137, + "epoch": 0.2741170268845546, + "grad_norm": 7.46875, + "learning_rate": 1.0320000000000001e-05, + "loss": 0.9485825538635254, + "mean_token_accuracy": 0.7162063598632813, + "num_tokens": 697254.0, + "step": 130 + }, + { + "entropy": 0.9009825974702835, + "epoch": 0.2952029520295203, + "grad_norm": 6.96875, + "learning_rate": 1.1120000000000002e-05, + "loss": 0.9119878768920898, + "mean_token_accuracy": 0.7238413639366627, + "num_tokens": 751774.0, + "step": 140 + }, + { + "entropy": 0.9040703341364861, + "epoch": 0.316288877174486, + "grad_norm": 7.21875, + "learning_rate": 1.1920000000000001e-05, + "loss": 0.9283308029174805, + "mean_token_accuracy": 0.7166316144168376, + "num_tokens": 808927.0, + "step": 150 + }, + { + "entropy": 0.8878034949302673, + "epoch": 0.33737480231945177, + "grad_norm": 7.375, + "learning_rate": 1.2720000000000002e-05, + "loss": 0.9068665504455566, + "mean_token_accuracy": 0.7254431843757629, + "num_tokens": 858559.0, + "step": 160 + }, + { + "entropy": 0.887480016052723, + "epoch": 0.3584607274644175, + "grad_norm": 7.65625, + "learning_rate": 1.3520000000000003e-05, + "loss": 0.8971159934997559, + "mean_token_accuracy": 0.7258390121161937, + "num_tokens": 912285.0, + "step": 170 + }, + { + "entropy": 0.8527634374797344, + "epoch": 0.37954665260938325, + "grad_norm": 7.6875, + "learning_rate": 1.432e-05, + "loss": 0.8586221694946289, + "mean_token_accuracy": 0.7359919100999832, + "num_tokens": 969385.0, + "step": 180 + }, + { + "entropy": 0.8445627830922604, + "epoch": 0.40063257775434896, + "grad_norm": 7.03125, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.855714225769043, + "mean_token_accuracy": 0.7341729424893856, + "num_tokens": 1026106.0, + "step": 190 + }, + { + "entropy": 0.841130618005991, + "epoch": 0.42171850289931473, + "grad_norm": 7.21875, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.8620488166809082, + "mean_token_accuracy": 0.7361006513237953, + "num_tokens": 1079743.0, + "step": 200 + }, + { + "entropy": 0.831258487701416, + "epoch": 0.44280442804428044, + "grad_norm": 7.0, + "learning_rate": 1.672e-05, + "loss": 0.8441590309143067, + "mean_token_accuracy": 0.7379129216074943, + "num_tokens": 1131453.0, + "step": 210 + }, + { + "entropy": 0.79731200709939, + "epoch": 0.46389035318924615, + "grad_norm": 7.875, + "learning_rate": 1.752e-05, + "loss": 0.8135190963745117, + "mean_token_accuracy": 0.7497627973556519, + "num_tokens": 1182486.0, + "step": 220 + }, + { + "entropy": 0.8269097700715065, + "epoch": 0.4849762783342119, + "grad_norm": 7.5625, + "learning_rate": 1.832e-05, + "loss": 0.8399410247802734, + "mean_token_accuracy": 0.7389348305761814, + "num_tokens": 1238136.0, + "step": 230 + }, + { + "entropy": 0.8093138128519058, + "epoch": 0.5060622034791776, + "grad_norm": 7.40625, + "learning_rate": 1.912e-05, + "loss": 0.810025691986084, + "mean_token_accuracy": 0.7460841409862041, + "num_tokens": 1294939.0, + "step": 240 + }, + { + "entropy": 0.8069386176764966, + "epoch": 0.5271481286241434, + "grad_norm": 6.84375, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8215888023376465, + "mean_token_accuracy": 0.7438369721174241, + "num_tokens": 1346462.0, + "step": 250 + }, + { + "entropy": 0.8088659413158894, + "epoch": 0.5482340537691092, + "grad_norm": 6.75, + "learning_rate": 1.9920000000000002e-05, + "loss": 0.8201449394226075, + "mean_token_accuracy": 0.7393839374184609, + "num_tokens": 1397444.0, + "step": 260 + }, + { + "entropy": 0.7969542622566224, + "epoch": 0.5693199789140748, + "grad_norm": 6.59375, + "learning_rate": 1.9831111111111113e-05, + "loss": 0.8009759902954101, + "mean_token_accuracy": 0.7512488305568695, + "num_tokens": 1453900.0, + "step": 270 + }, + { + "entropy": 0.780416651815176, + "epoch": 0.5904059040590406, + "grad_norm": 7.21875, + "learning_rate": 1.9742222222222223e-05, + "loss": 0.7874263763427735, + "mean_token_accuracy": 0.7526524804532528, + "num_tokens": 1505529.0, + "step": 280 + }, + { + "entropy": 0.7752278268337249, + "epoch": 0.6114918292040064, + "grad_norm": 6.875, + "learning_rate": 1.9653333333333334e-05, + "loss": 0.8075678825378418, + "mean_token_accuracy": 0.746569037437439, + "num_tokens": 1562473.0, + "step": 290 + }, + { + "entropy": 0.7895354829728604, + "epoch": 0.632577754348972, + "grad_norm": 6.6875, + "learning_rate": 1.9564444444444444e-05, + "loss": 0.7880948543548584, + "mean_token_accuracy": 0.7494045957922936, + "num_tokens": 1613320.0, + "step": 300 + }, + { + "entropy": 0.7653828673064709, + "epoch": 0.6536636794939378, + "grad_norm": 6.78125, + "learning_rate": 1.9475555555555558e-05, + "loss": 0.7780828475952148, + "mean_token_accuracy": 0.7556171745061875, + "num_tokens": 1662847.0, + "step": 310 + }, + { + "entropy": 0.7652887552976608, + "epoch": 0.6747496046389035, + "grad_norm": 6.375, + "learning_rate": 1.938666666666667e-05, + "loss": 0.7539787292480469, + "mean_token_accuracy": 0.7582654722034932, + "num_tokens": 1718553.0, + "step": 320 + }, + { + "entropy": 0.7076033115386963, + "epoch": 0.6958355297838693, + "grad_norm": 7.15625, + "learning_rate": 1.929777777777778e-05, + "loss": 0.7396605491638184, + "mean_token_accuracy": 0.7659220136702061, + "num_tokens": 1771024.0, + "step": 330 + }, + { + "entropy": 0.7509547933936119, + "epoch": 0.716921454928835, + "grad_norm": 6.15625, + "learning_rate": 1.920888888888889e-05, + "loss": 0.7459388256072998, + "mean_token_accuracy": 0.7644543245434761, + "num_tokens": 1821303.0, + "step": 340 + }, + { + "entropy": 0.7591827407479286, + "epoch": 0.7380073800738007, + "grad_norm": 7.03125, + "learning_rate": 1.912e-05, + "loss": 0.7627717971801757, + "mean_token_accuracy": 0.7567056350409984, + "num_tokens": 1868674.0, + "step": 350 + }, + { + "entropy": 0.7316406652331352, + "epoch": 0.7590933052187665, + "grad_norm": 6.59375, + "learning_rate": 1.9031111111111114e-05, + "loss": 0.7425707340240478, + "mean_token_accuracy": 0.7653280735015869, + "num_tokens": 1924619.0, + "step": 360 + }, + { + "entropy": 0.7539067789912224, + "epoch": 0.7801792303637322, + "grad_norm": 7.0625, + "learning_rate": 1.8942222222222224e-05, + "loss": 0.7716987609863282, + "mean_token_accuracy": 0.7524559155106545, + "num_tokens": 1978795.0, + "step": 370 + }, + { + "entropy": 0.7687754213809967, + "epoch": 0.8012651555086979, + "grad_norm": 7.1875, + "learning_rate": 1.8853333333333335e-05, + "loss": 0.7734373569488525, + "mean_token_accuracy": 0.7497503250837326, + "num_tokens": 2029994.0, + "step": 380 + }, + { + "entropy": 0.7508607231080532, + "epoch": 0.8223510806536637, + "grad_norm": 7.09375, + "learning_rate": 1.8764444444444445e-05, + "loss": 0.7575721740722656, + "mean_token_accuracy": 0.7572125047445297, + "num_tokens": 2082653.0, + "step": 390 + }, + { + "entropy": 0.7252861328423024, + "epoch": 0.8434370057986295, + "grad_norm": 6.5625, + "learning_rate": 1.867555555555556e-05, + "loss": 0.7381186485290527, + "mean_token_accuracy": 0.7612978041172027, + "num_tokens": 2135276.0, + "step": 400 + }, + { + "entropy": 0.7339485622942448, + "epoch": 0.8645229309435951, + "grad_norm": 6.625, + "learning_rate": 1.858666666666667e-05, + "loss": 0.7348906993865967, + "mean_token_accuracy": 0.7663561321794987, + "num_tokens": 2187466.0, + "step": 410 + }, + { + "entropy": 0.7272624362260103, + "epoch": 0.8856088560885609, + "grad_norm": 6.90625, + "learning_rate": 1.849777777777778e-05, + "loss": 0.749025821685791, + "mean_token_accuracy": 0.7617379955947399, + "num_tokens": 2237434.0, + "step": 420 + }, + { + "entropy": 0.7477380312979222, + "epoch": 0.9066947812335266, + "grad_norm": 6.34375, + "learning_rate": 1.840888888888889e-05, + "loss": 0.7498275756835937, + "mean_token_accuracy": 0.7604753144085408, + "num_tokens": 2288985.0, + "step": 430 + }, + { + "entropy": 0.7346535548567772, + "epoch": 0.9277807063784923, + "grad_norm": 6.9375, + "learning_rate": 1.832e-05, + "loss": 0.739972448348999, + "mean_token_accuracy": 0.7658773727715016, + "num_tokens": 2341187.0, + "step": 440 + }, + { + "entropy": 0.7167556572705507, + "epoch": 0.9488666315234581, + "grad_norm": 6.4375, + "learning_rate": 1.823111111111111e-05, + "loss": 0.7350784778594971, + "mean_token_accuracy": 0.7644179269671441, + "num_tokens": 2393385.0, + "step": 450 + }, + { + "entropy": 0.7537519961595536, + "epoch": 0.9699525566684238, + "grad_norm": 7.28125, + "learning_rate": 1.814222222222222e-05, + "loss": 0.757652759552002, + "mean_token_accuracy": 0.7587068513035774, + "num_tokens": 2442248.0, + "step": 460 + }, + { + "entropy": 0.72038439437747, + "epoch": 0.9910384818133896, + "grad_norm": 7.40625, + "learning_rate": 1.8053333333333332e-05, + "loss": 0.7361824035644531, + "mean_token_accuracy": 0.7609050884842873, + "num_tokens": 2498394.0, + "step": 470 + }, + { + "entropy": 0.735570513718837, + "epoch": 1.0105429625724829, + "grad_norm": 6.875, + "learning_rate": 1.7964444444444446e-05, + "loss": 0.730868148803711, + "mean_token_accuracy": 0.7710521068121936, + "num_tokens": 2545422.0, + "step": 480 + }, + { + "entropy": 0.6775223582983017, + "epoch": 1.0316288877174487, + "grad_norm": 7.125, + "learning_rate": 1.7875555555555556e-05, + "loss": 0.6740085124969483, + "mean_token_accuracy": 0.7795436836779117, + "num_tokens": 2597144.0, + "step": 490 + }, + { + "entropy": 0.6628748029470444, + "epoch": 1.0527148128624144, + "grad_norm": 6.84375, + "learning_rate": 1.7786666666666667e-05, + "loss": 0.6774313926696778, + "mean_token_accuracy": 0.7789938881993294, + "num_tokens": 2651938.0, + "step": 500 + }, + { + "entropy": 0.6695188779383898, + "epoch": 1.07380073800738, + "grad_norm": 7.0, + "learning_rate": 1.7697777777777777e-05, + "loss": 0.664512825012207, + "mean_token_accuracy": 0.7832319870591163, + "num_tokens": 2704948.0, + "step": 510 + }, + { + "entropy": 0.6812492057681083, + "epoch": 1.0948866631523457, + "grad_norm": 7.0, + "learning_rate": 1.760888888888889e-05, + "loss": 0.7036191940307617, + "mean_token_accuracy": 0.7707728065550328, + "num_tokens": 2757576.0, + "step": 520 + }, + { + "entropy": 0.6861097872257232, + "epoch": 1.1159725882973115, + "grad_norm": 7.3125, + "learning_rate": 1.752e-05, + "loss": 0.6651991844177246, + "mean_token_accuracy": 0.7841649115085602, + "num_tokens": 2808989.0, + "step": 530 + }, + { + "entropy": 0.6694958411157131, + "epoch": 1.1370585134422773, + "grad_norm": 6.78125, + "learning_rate": 1.7431111111111112e-05, + "loss": 0.6833772659301758, + "mean_token_accuracy": 0.7775424145162105, + "num_tokens": 2864507.0, + "step": 540 + }, + { + "entropy": 0.6568067740648985, + "epoch": 1.158144438587243, + "grad_norm": 6.21875, + "learning_rate": 1.7342222222222226e-05, + "loss": 0.6552294254302978, + "mean_token_accuracy": 0.7832994520664215, + "num_tokens": 2919201.0, + "step": 550 + }, + { + "entropy": 0.6432577680796385, + "epoch": 1.1792303637322088, + "grad_norm": 6.65625, + "learning_rate": 1.7253333333333336e-05, + "loss": 0.6560451030731201, + "mean_token_accuracy": 0.7863748580217361, + "num_tokens": 2972434.0, + "step": 560 + }, + { + "entropy": 0.6655397050082683, + "epoch": 1.2003162888771746, + "grad_norm": 6.09375, + "learning_rate": 1.7164444444444447e-05, + "loss": 0.6878513336181641, + "mean_token_accuracy": 0.7753379993140698, + "num_tokens": 3025099.0, + "step": 570 + }, + { + "entropy": 0.686093359068036, + "epoch": 1.2214022140221403, + "grad_norm": 7.09375, + "learning_rate": 1.7075555555555557e-05, + "loss": 0.6911579132080078, + "mean_token_accuracy": 0.7787310972809791, + "num_tokens": 3076307.0, + "step": 580 + }, + { + "entropy": 0.6812980778515338, + "epoch": 1.2424881391671059, + "grad_norm": 7.125, + "learning_rate": 1.6986666666666668e-05, + "loss": 0.6704156398773193, + "mean_token_accuracy": 0.7793935135006904, + "num_tokens": 3130053.0, + "step": 590 + }, + { + "entropy": 0.6409769468009472, + "epoch": 1.2635740643120716, + "grad_norm": 7.4375, + "learning_rate": 1.6897777777777778e-05, + "loss": 0.661864185333252, + "mean_token_accuracy": 0.7865670621395111, + "num_tokens": 3183331.0, + "step": 600 + }, + { + "entropy": 0.6799454756081105, + "epoch": 1.2846599894570374, + "grad_norm": 6.65625, + "learning_rate": 1.680888888888889e-05, + "loss": 0.6818383693695068, + "mean_token_accuracy": 0.778287123888731, + "num_tokens": 3239561.0, + "step": 610 + }, + { + "entropy": 0.6814555354416371, + "epoch": 1.3057459146020032, + "grad_norm": 6.3125, + "learning_rate": 1.672e-05, + "loss": 0.679498815536499, + "mean_token_accuracy": 0.7788635492324829, + "num_tokens": 3292996.0, + "step": 620 + }, + { + "entropy": 0.667108029127121, + "epoch": 1.326831839746969, + "grad_norm": 6.90625, + "learning_rate": 1.6631111111111113e-05, + "loss": 0.6758651733398438, + "mean_token_accuracy": 0.7792007848620415, + "num_tokens": 3345736.0, + "step": 630 + }, + { + "entropy": 0.6692709110677242, + "epoch": 1.3479177648919347, + "grad_norm": 6.96875, + "learning_rate": 1.6542222222222223e-05, + "loss": 0.6796128273010253, + "mean_token_accuracy": 0.7766051791608334, + "num_tokens": 3398076.0, + "step": 640 + }, + { + "entropy": 0.6564048878848553, + "epoch": 1.3690036900369003, + "grad_norm": 6.9375, + "learning_rate": 1.6453333333333334e-05, + "loss": 0.6633264064788819, + "mean_token_accuracy": 0.7854946702718735, + "num_tokens": 3450796.0, + "step": 650 + }, + { + "entropy": 0.6611979272216558, + "epoch": 1.390089615181866, + "grad_norm": 6.65625, + "learning_rate": 1.6364444444444444e-05, + "loss": 0.6661429405212402, + "mean_token_accuracy": 0.7835663430392742, + "num_tokens": 3503863.0, + "step": 660 + }, + { + "entropy": 0.6534961331635714, + "epoch": 1.4111755403268318, + "grad_norm": 6.9375, + "learning_rate": 1.6275555555555558e-05, + "loss": 0.6586530208587646, + "mean_token_accuracy": 0.7844067431986332, + "num_tokens": 3556265.0, + "step": 670 + }, + { + "entropy": 0.6516873501241207, + "epoch": 1.4322614654717976, + "grad_norm": 6.40625, + "learning_rate": 1.618666666666667e-05, + "loss": 0.6522529125213623, + "mean_token_accuracy": 0.7855845786631107, + "num_tokens": 3611423.0, + "step": 680 + }, + { + "entropy": 0.6764013767242432, + "epoch": 1.4533473906167633, + "grad_norm": 6.71875, + "learning_rate": 1.609777777777778e-05, + "loss": 0.6884442806243897, + "mean_token_accuracy": 0.7756352722644806, + "num_tokens": 3666630.0, + "step": 690 + }, + { + "entropy": 0.6554486058652401, + "epoch": 1.474433315761729, + "grad_norm": 7.40625, + "learning_rate": 1.600888888888889e-05, + "loss": 0.6563316822052002, + "mean_token_accuracy": 0.7854672558605671, + "num_tokens": 3718719.0, + "step": 700 + }, + { + "entropy": 0.6385185327380896, + "epoch": 1.4955192409066949, + "grad_norm": 6.59375, + "learning_rate": 1.5920000000000003e-05, + "loss": 0.6513954639434815, + "mean_token_accuracy": 0.7851097173988819, + "num_tokens": 3771822.0, + "step": 710 + }, + { + "entropy": 0.654766371101141, + "epoch": 1.5166051660516606, + "grad_norm": 6.5, + "learning_rate": 1.5831111111111114e-05, + "loss": 0.6622748851776123, + "mean_token_accuracy": 0.7847663886845112, + "num_tokens": 3823795.0, + "step": 720 + }, + { + "entropy": 0.6597310703247785, + "epoch": 1.5376910911966264, + "grad_norm": 6.375, + "learning_rate": 1.5742222222222224e-05, + "loss": 0.6634318351745605, + "mean_token_accuracy": 0.7861979939043522, + "num_tokens": 3875784.0, + "step": 730 + }, + { + "entropy": 0.6637851595878601, + "epoch": 1.558777016341592, + "grad_norm": 6.9375, + "learning_rate": 1.5653333333333335e-05, + "loss": 0.6621220111846924, + "mean_token_accuracy": 0.7886320613324642, + "num_tokens": 3929412.0, + "step": 740 + }, + { + "entropy": 0.637692479044199, + "epoch": 1.5798629414865577, + "grad_norm": 6.59375, + "learning_rate": 1.5564444444444445e-05, + "loss": 0.6402386665344239, + "mean_token_accuracy": 0.7895873270928859, + "num_tokens": 3982472.0, + "step": 750 + }, + { + "entropy": 0.6578179821372032, + "epoch": 1.6009488666315235, + "grad_norm": 7.25, + "learning_rate": 1.5475555555555556e-05, + "loss": 0.6751936435699463, + "mean_token_accuracy": 0.7776105716824532, + "num_tokens": 4037829.0, + "step": 760 + }, + { + "entropy": 0.6653612300753593, + "epoch": 1.6220347917764892, + "grad_norm": 6.9375, + "learning_rate": 1.5386666666666666e-05, + "loss": 0.6717413425445556, + "mean_token_accuracy": 0.777862798422575, + "num_tokens": 4093735.0, + "step": 770 + }, + { + "entropy": 0.6561707239598036, + "epoch": 1.6431207169214548, + "grad_norm": 7.03125, + "learning_rate": 1.5297777777777777e-05, + "loss": 0.6501868724822998, + "mean_token_accuracy": 0.7863908343017101, + "num_tokens": 4148410.0, + "step": 780 + }, + { + "entropy": 0.6655665650963783, + "epoch": 1.6642066420664205, + "grad_norm": 6.59375, + "learning_rate": 1.520888888888889e-05, + "loss": 0.679684829711914, + "mean_token_accuracy": 0.778567586839199, + "num_tokens": 4202907.0, + "step": 790 + }, + { + "entropy": 0.6781473312526941, + "epoch": 1.6852925672113863, + "grad_norm": 6.875, + "learning_rate": 1.5120000000000001e-05, + "loss": 0.6722404479980468, + "mean_token_accuracy": 0.7763243250548839, + "num_tokens": 4256213.0, + "step": 800 + }, + { + "entropy": 0.6321199681609869, + "epoch": 1.706378492356352, + "grad_norm": 6.78125, + "learning_rate": 1.5031111111111111e-05, + "loss": 0.6404708385467529, + "mean_token_accuracy": 0.7903632469475269, + "num_tokens": 4309268.0, + "step": 810 + }, + { + "entropy": 0.6424700867384672, + "epoch": 1.7274644175013179, + "grad_norm": 7.15625, + "learning_rate": 1.4942222222222222e-05, + "loss": 0.6444578170776367, + "mean_token_accuracy": 0.7881530813872815, + "num_tokens": 4360761.0, + "step": 820 + }, + { + "entropy": 0.6615507304668427, + "epoch": 1.7485503426462836, + "grad_norm": 6.0625, + "learning_rate": 1.4853333333333336e-05, + "loss": 0.6604718208312989, + "mean_token_accuracy": 0.782805722206831, + "num_tokens": 4411678.0, + "step": 830 + }, + { + "entropy": 0.6196760050952435, + "epoch": 1.7696362677912494, + "grad_norm": 6.6875, + "learning_rate": 1.4764444444444446e-05, + "loss": 0.6261836051940918, + "mean_token_accuracy": 0.7926589228212834, + "num_tokens": 4467254.0, + "step": 840 + }, + { + "entropy": 0.6139154363423586, + "epoch": 1.7907221929362152, + "grad_norm": 7.21875, + "learning_rate": 1.4675555555555557e-05, + "loss": 0.6301932811737061, + "mean_token_accuracy": 0.7918383464217186, + "num_tokens": 4517383.0, + "step": 850 + }, + { + "entropy": 0.6410171519964933, + "epoch": 1.811808118081181, + "grad_norm": 7.1875, + "learning_rate": 1.4586666666666667e-05, + "loss": 0.655810022354126, + "mean_token_accuracy": 0.7856725618243218, + "num_tokens": 4569980.0, + "step": 860 + }, + { + "entropy": 0.6666022066026926, + "epoch": 1.8328940432261467, + "grad_norm": 6.5625, + "learning_rate": 1.449777777777778e-05, + "loss": 0.6677482604980469, + "mean_token_accuracy": 0.7798060193657875, + "num_tokens": 4622612.0, + "step": 870 + }, + { + "entropy": 0.6398684818297624, + "epoch": 1.8539799683711122, + "grad_norm": 7.84375, + "learning_rate": 1.440888888888889e-05, + "loss": 0.6501501083374024, + "mean_token_accuracy": 0.7857734531164169, + "num_tokens": 4676525.0, + "step": 880 + }, + { + "entropy": 0.6285852700471878, + "epoch": 1.875065893516078, + "grad_norm": 7.0625, + "learning_rate": 1.432e-05, + "loss": 0.6422877311706543, + "mean_token_accuracy": 0.7884316690266132, + "num_tokens": 4734090.0, + "step": 890 + }, + { + "entropy": 0.6409385818988085, + "epoch": 1.8961518186610438, + "grad_norm": 7.9375, + "learning_rate": 1.4231111111111114e-05, + "loss": 0.6352181434631348, + "mean_token_accuracy": 0.7952412232756615, + "num_tokens": 4788537.0, + "step": 900 + }, + { + "entropy": 0.6456746399402619, + "epoch": 1.9172377438060095, + "grad_norm": 6.90625, + "learning_rate": 1.4142222222222224e-05, + "loss": 0.6564863204956055, + "mean_token_accuracy": 0.789187741279602, + "num_tokens": 4840342.0, + "step": 910 + }, + { + "entropy": 0.6360713161528111, + "epoch": 1.938323668950975, + "grad_norm": 6.3125, + "learning_rate": 1.4053333333333335e-05, + "loss": 0.6300373554229737, + "mean_token_accuracy": 0.7876299776136875, + "num_tokens": 4891524.0, + "step": 920 + }, + { + "entropy": 0.6285211775451899, + "epoch": 1.9594095940959408, + "grad_norm": 7.78125, + "learning_rate": 1.3964444444444445e-05, + "loss": 0.6337003231048584, + "mean_token_accuracy": 0.7889726974070073, + "num_tokens": 4939573.0, + "step": 930 + }, + { + "entropy": 0.6301535632461309, + "epoch": 1.9804955192409066, + "grad_norm": 6.84375, + "learning_rate": 1.3875555555555557e-05, + "loss": 0.6374162673950196, + "mean_token_accuracy": 0.7898242607712745, + "num_tokens": 4991267.0, + "step": 940 + }, + { + "entropy": 0.6537058566873138, + "epoch": 2.0, + "grad_norm": 13.3125, + "learning_rate": 1.3786666666666668e-05, + "loss": 0.641156530380249, + "mean_token_accuracy": 0.7822844466647586, + "num_tokens": 5036304.0, + "step": 950 + } + ], + "logging_steps": 10, + "max_steps": 2500, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.159714318599373e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-950/training_args.bin b/checkpoint-950/training_args.bin new file mode 100644 index 0000000..f138a50 --- /dev/null +++ b/checkpoint-950/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7103b0ce3340ea41aca3b7441d01d7c38791e3d3ce8c9b28c906c9b1d49bbee9 +size 5713 diff --git a/config.json b/config.json new file mode 100644 index 0000000..1d219d6 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..6e9e36c --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "4.51.0" +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..febc610 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07855dca909d308e2784d7676d0d2657e5764305c4cf1ccf175a187195feee05 +size 3441185608 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..770e41d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..f138a50 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7103b0ce3340ea41aca3b7441d01d7c38791e3d3ce8c9b28c906c9b1d49bbee9 +size 5713